tahamajs/RR / Docs /visualizer /src /data /module_math.json
tahamajs's picture
download
raw
169 kB
{ "SOAFusedAttention": { "id": "SOAFusedAttention", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "adaptive_span_attention": { "id": "adaptive_span_attention", "category": "C", "primary_formula": "S_{ij} = \\frac{Q_i K_j^\\top}{\\sqrt{d}}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "adaptive_span_inject": { "id": "adaptive_span_inject", "category": "C", "primary_formula": "S_{ij} = \\frac{Q_i K_j^\\top}{\\sqrt{d}}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "adaptive_sparse_attention": { "id": "adaptive_sparse_attention", "category": "E", "primary_formula": "\\gamma_h = \\sigma(\\text{logit}_h) \\in (0, 1)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "adaptive_sparse_inject": { "id": "adaptive_sparse_inject", "category": "B", "primary_formula": "\\gamma_h = \\sigma(\\text{logit}_h) \\in (0, 1)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "adaptive_svd_attention": { "id": "adaptive_svd_attention", "category": "E", "primary_formula": "W = U \\Sigma V^\\top = \\sum_{i=1}^{\\min(d_{\\text{out}}, d_{\\text{in}})} \\sigma_i u_i v_i^\\top", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "adasplash_attention": { "id": "adasplash_attention", "category": "C", "primary_formula": "S_{ij} = \\frac{Q_i K_j^\\top}{\\sqrt{d_h}}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "adasplash_inject": { "id": "adasplash_inject", "category": "B", "primary_formula": "S_{ij} = \\frac{Q_i K_j^\\top}{\\sqrt{d_h}}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "aegis_hybrid": { "id": "aegis_hybrid", "category": "E", "primary_formula": "O_t = \\alpha_t \\odot O_t^{\\text{local}} + (1 - \\alpha_t) \\odot O_t^{\\text{global}}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "aegis_hybrid_attention": { "id": "aegis_hybrid_attention", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "aegis_spectron_fp8": { "id": "aegis_spectron_fp8", "category": "E", "primary_formula": "p_{t, d} = \\frac{x_{t, d}^2}{\\sum_{k=1}^D x_{t, k}^2 + \\epsilon}, \\quad d \\in \\{1, \\dots, D\\}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "aegis_v1": { "id": "aegis_v1", "category": "E", "primary_formula": "W = U \\Sigma V^\\top = \\sum_{k=1}^{\\min(d_{\\text{in}}, d_{\\text{out}})} \\sigma_k u_k v_k^\\top", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "aegis_v2": { "id": "aegis_v2", "category": "E", "primary_formula": "Q = X W_{Q1} W_{Q2}, \\quad K = X W_{K1} W_{K2}, \\quad V = X W_{V1} W_{V2}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "aegis_v3": { "id": "aegis_v3", "category": "E", "primary_formula": "e_j = \\exp\\left(\\frac{x_j - m}{\\tau}\\right), \\quad m_{\\text{scaled}} = \\operatorname{clamp}\\left(-\\frac{m}{\\tau}, -\\infty, 80.0\\right)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "aegis_v4": { "id": "aegis_v4", "category": "E", "primary_formula": "Q = \\text{FactorisedLinear}_Q(X), \\quad K = \\text{FactorisedLinear}_K(X), \\quad V = \\text{FactorisedLinear}_V(X)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "aegis_v5": { "id": "aegis_v5", "category": "E", "primary_formula": "Q_{\\text{feat}} = \\operatorname{ELU}(Q_g \\Omega) + 1.0 \\in \\mathbb{R}^{B \\times H_g \\times L \\times F}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "aegis_v6": { "id": "aegis_v6", "category": "E", "primary_formula": "y = x (V \\Sigma U^\\top) + x (B^\\top A^\\top) + b", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "asvd_attention": { "id": "asvd_attention", "category": "E", "primary_formula": "y = ((x V^\\top) \\odot S) U^\\top + (x A^\\top) B^\\top + b", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "bam_umoe": { "id": "bam_umoe", "category": "F", "primary_formula": "\\mathbf{Q} = \\mathbf{X} \\mathbf{W}_Q, \\quad \\mathbf{K} = \\mathbf{X} \\mathbf{W}_K, \\quad \\mathbf{V}_{\\text{base}} = \\mathbf{X} \\mathbf{W}_V", "steps": [ { "id": 1, "name": "Router Gating", "formula": "G(x) = \text{Softmax}(\text{TopK}(x W_g, k))", "flops": 12288 }, { "id": 2, "name": "Expert Execution", "formula": "Y = \\sum_{e \\in \text{TopK}} G(x)_e E_e(x)", "flops": 2359296 } ], "total_flops_per_token": 2371584 }, "block_transformer_hiba": { "id": "block_transformer_hiba", "category": "A", "primary_formula": "\\mathbf{Q}_b = \\mathbf{Q}[:, :, \\operatorname{start}_b:\\operatorname{end}_b, :], \\quad \\mathbf{K}_b = \\mathbf{K}[:, :, \\operatorname{start}_b:\\operatorname{end}_b, :], \\quad \\mathbf{V}_b = \\mathbf{V}[:, :, \\operatorname{start}_b:\\operatorname{end}_b, :]", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "chela_slay": { "id": "chela_slay", "category": "B", "primary_formula": "\\hat{Q} = \\frac{Q}{\\|Q\\|_2}, \\quad \\hat{K} = \\frac{K}{\\|K\\|_2}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "cluster_attention": { "id": "cluster_attention", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "cluster_inject": { "id": "cluster_inject", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "compressed_attention": { "id": "compressed_attention", "category": "E", "primary_formula": "\\mathbf{W} \\approx \\mathbf{U}_r \\mathbf{\\Sigma}_r \\mathbf{V}_r^T", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "compressed_mha": { "id": "compressed_mha", "category": "B", "primary_formula": "\\mathbf{W}_{QKV} = [\\mathbf{W}_Q; \\mathbf{W}_K; \\mathbf{W}_V] \\in \\mathbb{R}^{3D \\times D}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "compressed_mha_attention": { "id": "compressed_mha_attention", "category": "E", "primary_formula": "\\mathbf{W}_{QKV} = [\\mathbf{W}_Q; \\mathbf{W}_K; \\mathbf{W}_V] \\in \\mathbb{R}^{3D \\times D}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "cosformer": { "id": "cosformer", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "cosformer_inject": { "id": "cosformer_inject", "category": "A", "primary_formula": "h_t = \\bar{A} h_{t-1} + \\bar{B} x_t, \\quad y_t = C h_t + D x_t, \\quad \\bar{A} = \\exp(\\Delta A)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "decay_linear_attention": { "id": "decay_linear_attention", "category": "D", "primary_formula": "Q = X W_q, \\quad K = X W_k, \\quad V = X W_v", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "eat": { "id": "eat", "category": "G", "primary_formula": "O = \\operatorname{Softmax}\\left(\\frac{Q K^T}{\\sqrt{d_h}} + M\\right) V W_O", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "expert_choice_moe": { "id": "expert_choice_moe", "category": "E", "primary_formula": "\\mathbf{Q} = \\mathbf{X} \\mathbf{W}_Q, \\quad \\mathbf{K} = \\mathbf{X} \\mathbf{W}_K, \\quad \\mathbf{V} = \\mathbf{X} \\mathbf{W}_V", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "factorised_matrix_state": { "id": "factorised_matrix_state", "category": "E", "primary_formula": "Q = X W_q, \\quad K = X W_k, \\quad V = X W_v \\quad (W_q, W_k, W_v \\in \\mathbb{R}^{D \\times D})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "flash_attention3": { "id": "flash_attention3", "category": "E", "primary_formula": "S = \\frac{Q K^\\top}{\\sqrt{d_h}} + M", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "flash_attention3_fp8": { "id": "flash_attention3_fp8", "category": "E", "primary_formula": "Q_{\\text{fp8}} = \\text{cast}_{\\text{fp8\\_e4m3}}(Q) \\in \\mathbb{F}_{p8}^{B \\times L \\times H \\times d_h}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "full_attention": { "id": "full_attention", "category": "E", "primary_formula": "\\text{Attention}(Q, K, V) = \\text{softmax}\\left(\\frac{Q K^\\top}{\\sqrt{d_k}} + M\\right) V", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "fused_svd_kernel": { "id": "fused_svd_kernel", "category": "E", "primary_formula": "\\min_{\\text{rank}(\\widehat{W}) \\le r} \\|W - \\widehat{W}\\|_F^2 = \\sum_{i=r+1}^{D} \\sigma_i^2", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "fusion_attn": { "id": "fusion_attn", "category": "E", "primary_formula": "\\mathbf{S}_{\\text{blk}} = \\frac{\\mathbf{G}_Q(\\mathbf{Q}_{\\text{blk}}) \\mathbf{G}_K(\\mathbf{K}_{\\text{blk}})^T}{\\tau}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "galerkin": { "id": "galerkin", "category": "E", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "galerkin_attention": { "id": "galerkin_attention", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "galerkin_inject": { "id": "galerkin_inject", "category": "E", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "gated_delta_net": { "id": "gated_delta_net", "category": "D", "primary_formula": "Q = X W_q \\in \\mathbb{R}^{B \\times L \\times D}, \\quad K = X W_k \\in \\mathbb{R}^{B \\times L \\times D}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "gated_deltanet_block": { "id": "gated_deltanet_block", "category": "D", "primary_formula": "\\text{Output} = \\begin{cases}\n \\text{LayerNorm}(X + \\text{MHA}(X, X, X)), & \\text{if } \\text{use\\_full\\_attention} \\\\\n \\text{LayerNorm}(X + \\text{GatedDeltaNet}(X)), & \\text{otherwise}\n\\end{cases}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "gated_linear_attention": { "id": "gated_linear_attention", "category": "D", "primary_formula": "Q = X W_q, \\quad K = X W_k, \\quad V = X W_v, \\quad Z = X W_z", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "goldfinch_ahn": { "id": "goldfinch_ahn", "category": "D", "primary_formula": "h_t = \\bar{A} h_{t-1} + \\bar{B} x_t, \\quad y_t = C h_t + D x_t, \\quad \\bar{A} = \\exp(\\Delta A)", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "gqa": { "id": "gqa", "category": "B", "primary_formula": "K_{\\text{shared}} = \\text{repeat\\_interleave}(K, H_Q / H_{KV}, \\text{dim}=\\text{heads})", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "gqa_attention": { "id": "gqa_attention", "category": "A", "primary_formula": "K_{\\text{shared}} = \\text{repeat\\_interleave}(K, H_Q / H_{KV}, \\text{dim}=\\text{heads})", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "granite_hybrid": { "id": "granite_hybrid", "category": "D", "primary_formula": "[X_{\\text{ssm}}, Z] = X W_{\\text{in}} \\in \\mathbb{R}^{B \\times L \\times 2 D_{\\text{inner}}}, \\quad D_{\\text{inner}} = D \\cdot \\text{expand}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "griffin_ssd": { "id": "griffin_ssd", "category": "E", "primary_formula": "S^{\\text{local}}_{b, h, i, j} = \\frac{\\langle q_{b, h, i}, k_{b, h, j} \\rangle}{\\sqrt{d}} + M^{\\text{window}}_{i, j} + M^{\\text{pad}}_{b, j}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "halo_regla": { "id": "halo_regla", "category": "B", "primary_formula": "Q_{\\text{rope}} = \\operatorname{RoPE}(Q), \\quad K_{\\text{rope}} = \\operatorname{RoPE}(K)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "hamming_attention": { "id": "hamming_attention", "category": "E", "primary_formula": "Q_b = \\tanh(5.0 \\cdot \\gamma_Q \\odot Q) \\in [-1, 1]^{B \\times H \\times L \\times d_k}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "hgrn2_gated_deltanet": { "id": "hgrn2_gated_deltanet", "category": "D", "primary_formula": "\\begin{aligned}\nY_{\\text{hgrn}} &= \\text{HGRN2}(X) \\\\\nY_{\\text{deltanet}} &= \\text{GatedDeltaNet}(X) \\\\\n\\gamma &= \\sigma(W_g X) \\\\\nY &= \\gamma \\odot Y_{\\text{hgrn}} + (1 - \\gamma) \\odot Y_{\\text{deltanet}} \\\\\n\\text{Output} &= Y W_o\n\\end{aligned}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "hgrn2_layer": { "id": "hgrn2_layer", "category": "D", "primary_formula": "f_t = \\exp(-\\operatorname{softplus}(W_f x_t + b_f)) \\in (0, 1)^D, \\quad b_f = 1.5", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "hgrn2_outer": { "id": "hgrn2_outer", "category": "D", "primary_formula": "Q = X W_q, \\quad K = X W_k, \\quad V = X W_v", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "hla_attention": { "id": "hla_attention", "category": "D", "primary_formula": "\\phi(Q) = \\text{ELU}(X W_q) + 1, \\quad \\phi(K) = \\text{ELU}(X W_k) + 1, \\quad V = X W_v", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "hla_inject": { "id": "hla_inject", "category": "B", "primary_formula": "\\phi(Q) = \\text{ELU}(X W_q) + 1, \\quad \\phi(K) = \\text{ELU}(X W_k) + 1, \\quad V = X W_v", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "hymba_transx": { "id": "hymba_transx", "category": "E", "primary_formula": "\\mathbf{R}_{\\Theta, m}^{d_k} \\mathbf{x}_m = \\begin{pmatrix} x_{m, 0} \\cos(m \\theta_0) - x_{m, 1} \\sin(m \\theta_0) \\\\ x_{m, 0} \\sin(m \\theta_0) + x_{m, 1} \\cos(m \\theta_0) \\\\ \\vdots \\end{pmatrix}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "infini_attention": { "id": "infini_attention", "category": "E", "primary_formula": "\\mathbf{Y} = \\mathbf{Y}_{\\text{local}} + \\sigma(\\beta) \\mathbf{Y}_{\\text{mem}}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "jamba": { "id": "jamba", "category": "D", "primary_formula": "\\mathbf{X}_{\\text{proj}} = \\mathbf{X} \\mathbf{W}_{\\text{in}}, \\quad \\mathbf{W}_{\\text{in}} \\in \\mathbb{R}^{D \\times D}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "jamba_info": { "id": "jamba_info", "category": "E", "primary_formula": "\\mathbf{Y} = \\mathbf{W}_O \\left(\\mathbf{A}_{\\text{diff}} \\mathbf{V} + 0.1 \\cdot \\text{SimpleSSM}(\\text{MLP}_{\\text{down-up}}(\\mathbf{X}))\\right)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "kda_attention": { "id": "kda_attention", "category": "D", "primary_formula": "Q = \\text{normalize}(X W_q, \\text{dim}=-1), \\quad K = \\frac{\\text{normalize}(X W_k, \\text{dim}=-1)}{\\sqrt{d_k}}, \\quad V = X W_v", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "kda_inject": { "id": "kda_inject", "category": "B", "primary_formula": "Q = \\text{normalize}(X W_q, \\text{dim}=-1), \\quad K = \\frac{\\text{normalize}(X W_k, \\text{dim}=-1)}{\\sqrt{d_k}}, \\quad V = X W_v", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "leka_v1": { "id": "leka_v1", "category": "E", "primary_formula": "\\mathbf{O}_i = \\sum_{j=1}^N \\frac{\\exp\\left(\\frac{\\mathbf{q}_i \\mathbf{k}_j^\\top}{\\sqrt{d_k}}\\right)}{\\sum_{m=1}^N \\exp\\left(\\frac{\\mathbf{q}_i \\mathbf{k}_m^\\top}{\\sqrt{d_k}}\\right)} \\mathbf{v}_j", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "leka_v1_inject": { "id": "leka_v1_inject", "category": "B", "primary_formula": "\\mathbf{O}_i = \\sum_{j=1}^N \\frac{\\exp\\left(\\frac{\\mathbf{q}_i \\mathbf{k}_j^\\top}{\\sqrt{d_k}}\\right)}{\\sum_{m=1}^N \\exp\\left(\\frac{\\mathbf{q}_i \\mathbf{k}_m^\\top}{\\sqrt{d_k}}\\right)} \\mathbf{v}_j", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "leka_v2": { "id": "leka_v2", "category": "E", "primary_formula": "\\phi_1(\\mathbf{x}) = \\frac{\\left[\\cos(\\mathbf{x} \\mathbf{W}_1), \\; \\sin(\\mathbf{x} \\mathbf{W}_1)\\right]}{\\sqrt{F_1}}, \\quad \\phi_2(\\mathbf{x}) = \\frac{\\left[\\cos(\\mathbf{x} \\mathbf{W}_2), \\; \\sin(\\mathbf{x} \\mathbf{W}_2)\\right]}{\\sqrt{F_2}}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "leka_v2_inject": { "id": "leka_v2_inject", "category": "B", "primary_formula": "\\phi_1(\\mathbf{x}) = \\frac{\\left[\\cos(\\mathbf{x} \\mathbf{W}_1), \\; \\sin(\\mathbf{x} \\mathbf{W}_1)\\right]}{\\sqrt{F_1}}, \\quad \\phi_2(\\mathbf{x}) = \\frac{\\left[\\cos(\\mathbf{x} \\mathbf{W}_2), \\; \\sin(\\mathbf{x} \\mathbf{W}_2)\\right]}{\\sqrt{F_2}}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "leka_v3": { "id": "leka_v3", "category": "E", "primary_formula": "\\mathbf{s} = \\operatorname{MLP}_{\\text{scorer}}(\\mathbf{V}) = \\mathbf{W}_2 \\operatorname{GELU}(\\mathbf{V} \\mathbf{W}_1 + \\mathbf{b}_1) + \\mathbf{b}_2 \\in \\mathbb{R}^{B \\times H \\times N}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "leka_v3_inject": { "id": "leka_v3_inject", "category": "B", "primary_formula": "\\mathbf{s} = \\operatorname{MLP}_{\\text{scorer}}(\\mathbf{V}) = \\mathbf{W}_2 \\operatorname{GELU}(\\mathbf{V} \\mathbf{W}_1 + \\mathbf{b}_1) + \\mathbf{b}_2 \\in \\mathbb{R}^{B \\times H \\times N}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "linformer_attention": { "id": "linformer_attention", "category": "E", "primary_formula": "c^{KV}_t = W^{DKV} x_t, \\quad k^C_t = W^{UK} c^{KV}_t, \\quad v^C_t = W^{UV} c^{KV}_t", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "lola_attention": { "id": "lola_attention", "category": "E", "primary_formula": "W \\approx U V, \\quad U \\in \\mathbb{R}^{d_{\\text{out}} \\times r}, \\; V \\in \\mathbb{R}^{r \\times d_{\\text{in}}}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "lora_attention": { "id": "lora_attention", "category": "E", "primary_formula": "y = W_0 x + \\frac{\\alpha}{r} B A x", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "lsh_attention": { "id": "lsh_attention", "category": "E", "primary_formula": "\\mathbf{Q} = \\mathbf{X} \\mathbf{W}_Q, \\quad \\mathbf{K} = \\mathbf{X} \\mathbf{W}_K, \\quad \\mathbf{V} = \\mathbf{X} \\mathbf{W}_V \\quad \\in \\mathbb{R}^{B \\times H \\times N \\times d_k}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "mamba": { "id": "mamba", "category": "D", "primary_formula": "[U, Z] = \\operatorname{chunk}(X W_{\\text{in}}, 2) \\in \\mathbb{R}^{B \\times L \\times D}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "mamba_inject": { "id": "mamba_inject", "category": "D", "primary_formula": "h_t = \\bar{A} h_{t-1} + \\bar{B} x_t, \\quad y_t = C h_t + D x_t, \\quad \\bar{A} = \\exp(\\Delta A)", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "mamba_svd_flash": { "id": "mamba_svd_flash", "category": "D", "primary_formula": "U_t = \\text{proj}_u(x_t) \\in \\mathbb{R}^{D \\times r}, \\quad V_t = \\text{proj}_v(x_t) \\in \\mathbb{R}^{r \\times D}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "minimal_aurora": { "id": "minimal_aurora", "category": "E", "primary_formula": "W_0 = U \\Sigma V^T, \\quad \\Sigma = \\operatorname{diag}(\\sigma_1, \\sigma_2, \\dots, \\sigma_{\\min(d_{\\text{out}}, d_{\\text{in}})})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "minimal_nova": { "id": "minimal_nova", "category": "E", "primary_formula": "W_0 = U \\Sigma V^T, \\quad \\Sigma = \\operatorname{diag}(\\sigma_1, \\sigma_2, \\dots, \\sigma_{\\min(d_{\\text{out}}, d_{\\text{in}})})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "minimal_prism": { "id": "minimal_prism", "category": "E", "primary_formula": "W_0 = U \\Sigma V^T, \\quad \\Sigma = \\operatorname{diag}(\\sigma_1, \\sigma_2, \\dots, \\sigma_{\\min(d_{\\text{out}}, d_{\\text{in}})})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "minimal_svd": { "id": "minimal_svd", "category": "E", "primary_formula": "A = U_{:, :r} \\text{diag}(\\sqrt{S_{:r}}) \\in \\mathbb{R}^{\\text{out} \\times r}, \\quad B = \\text{diag}(\\sqrt{S_{:r}}) V_{:r, :} \\in \\mathbb{R}^{r \\times \\text{in}}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "minimal_svd_v2": { "id": "minimal_svd_v2", "category": "E", "primary_formula": "W = U \\Sigma V^\\top \\approx U_{:, :r} \\Sigma_{:r} V_{:r, :}^\\top = A B", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "mixattention_moska": { "id": "mixattention_moska", "category": "B", "primary_formula": "\\operatorname{Var}(\\mathbf{K}_{b, h, t}) = \\frac{1}{d_k} \\sum_{c=1}^{d_k} \\left(K_{b, h, t, c} - \\bar{K}_{b, h, t}\\right)^2", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "mla_attention": { "id": "mla_attention", "category": "E", "primary_formula": "\\mathbf{c}_{KV} = W_{DKV} X \\in \\mathbb{R}^{B \\times N \\times d_c}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "moba": { "id": "moba", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "moba_attention": { "id": "moba_attention", "category": "B", "primary_formula": "\\mathbf{Q} = \\mathbf{X} \\mathbf{W}_Q, \\quad \\mathbf{K} = \\mathbf{X} \\mathbf{W}_K, \\quad \\mathbf{V} = \\mathbf{X} \\mathbf{W}_V", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "moe_deltakv_paged": { "id": "moe_deltakv_paged", "category": "E", "primary_formula": "\\Delta \\mathbf{K}_t = \\mathbf{K}_t - \\mathbf{K}_{t-1}, \\quad \\forall t \\in \\{1, \\dots, N-1\\}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "nasc_attention": { "id": "nasc_attention", "category": "E", "primary_formula": "Q_{\\text{loc}}, K_{\\text{loc}}, V_{\\text{loc}} = \\text{SVD-LoRA}(X)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "nexus": { "id": "nexus", "category": "B", "primary_formula": "\\mathbf{Q}_{\\text{sample}} = \\mathbf{Q}[:, :, ::\\sigma, :], \\quad \\mathbf{K}_{\\text{sample}} = \\mathbf{K}[:, :, ::\\sigma, :]", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "nha_attention": { "id": "nha_attention", "category": "E", "primary_formula": "\\mathbf{Y} = \\gamma(\\mathbf{x}) \\mathbf{Y}_{\\text{local}} + (1 - \\gamma(\\mathbf{x})) \\mathbf{Y}_{\\text{global}}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "nha_compressor": { "id": "nha_compressor", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "nsa_attention": { "id": "nsa_attention", "category": "E", "primary_formula": "\\mathbf{Q} = \\mathbf{X} \\mathbf{W}_Q, \\quad \\mathbf{K} = \\mathbf{X} \\mathbf{W}_K, \\quad \\mathbf{V} = \\mathbf{X} \\mathbf{W}_V \\quad \\in \\mathbb{R}^{B \\times H \\times L \\times d_h}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "nystrom_attention": { "id": "nystrom_attention", "category": "E", "primary_formula": "\\hat{A} = \\hat{F}_1 \\hat{F}_2^+ \\hat{F}_3", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "optimized_hybrid_attention": { "id": "optimized_hybrid_attention", "category": "E", "primary_formula": "Q = X W_Q, \\quad K = X W_K, \\quad V = X W_V \\in \\mathbb{R}^{B \\times N \\times H \\cdot d_h}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "orion": { "id": "orion", "category": "E", "primary_formula": "W = U \\Sigma V^T, \\quad \\Sigma = \\text{diag}(\\sigma_1, \\sigma_2, \\dots, \\sigma_{\\min(d_{\\text{out}}, d_{\\text{in}})})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "performer_attention": { "id": "performer_attention", "category": "E", "primary_formula": "\\text{Attn}(Q, K, V) = D^{-1} (\\phi(Q) (\\phi(K)^\\top V)), \\quad \\phi(x) = \\frac{h(x)}{\\sqrt{m}} \\exp\\left(\\omega^\\top x - \\frac{\\|x\\|^2}{2}\\right)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "phantom": { "id": "phantom", "category": "E", "primary_formula": "\\ell_{\\text{row}}(i) = \\frac{\\|U_{i, :}\\|_2^2}{\\sum_{i'=1}^{d_{\\text{out}}} \\|U_{i', :}\\|_2^2}, \\quad \\ell_{\\text{col}}(j) = \\frac{\\|V_{:, j}\\|_2^2}{\\sum_{j'=1}^{d_{\\text{in}}} \\|V_{:, j'}\\|_2^2}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "primal_attention": { "id": "primal_attention", "category": "E", "primary_formula": "F_1 = \\frac{Q \\widetilde{K}^T}{\\sqrt{d}} \\in \\mathbb{R}^{B \\times H \\times N \\times r}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "progressive_token_pruning": { "id": "progressive_token_pruning", "category": "E", "primary_formula": "\\min_{\\text{rank}(\\widehat{W}) \\le r} \\|W - \\widehat{W}\\|_F^2 = \\sum_{i=r+1}^{D} \\sigma_i^2", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "projection_factorized_attn": { "id": "projection_factorized_attn", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "qcqa_attention": { "id": "qcqa_attention", "category": "A", "primary_formula": "\\pi_{h, j} = \\frac{\\exp\\left(\\frac{q_h W_{\\text{router}, j}}{\\tau}\\right)}{\\sum_{j'=1}^{H_{kv}} \\exp\\left(\\frac{q_h W_{\\text{router}, j'}}{\\tau}\\right)}, \\quad \\forall j \\in \\{1, \\dots, H_{kv}\\}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "rala_attention": { "id": "rala_attention", "category": "E", "primary_formula": "\\text{Attention}_{\\text{RALA}}(Q, K, V) = \\frac{\\phi(Q) (\\phi(K)^\\top V)}{\\phi(Q) (\\phi(K)^\\top \\mathbf{1})} + \\alpha \\frac{\\phi(Q) (U V^\\top)}{\\phi(Q) (\\phi(K)^\\top \\mathbf{1})}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "rala_compressor": { "id": "rala_compressor", "category": "E", "primary_formula": "S_{\\text{KV}} = \\sum_{j=1}^N \\phi(K_j) V_j^T \\in \\mathbb{R}^{B \\times H \\times d \\times d}, \\quad z = \\sum_{j=1}^N \\phi(K_j) \\in \\mathbb{R}^{B \\times H \\times d}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "reap_moe": { "id": "reap_moe", "category": "E", "primary_formula": "\\mathbf{W}_e = \\mathbf{U}_e \\mathbf{\\Sigma}_e \\mathbf{V}_e^\\top, \\quad \\operatorname{rank}(\\mathbf{W}_e) \\le r \\ll \\min(d_{\\text{in}}, d_{\\text{out}})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "regla_attention": { "id": "regla_attention", "category": "E", "primary_formula": "Q = X W_Q, \\quad K = X W_K, \\quad V = X W_V, \\quad G = X W_G", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "replace": { "id": "replace", "category": "B", "primary_formula": "\\min_{\\Theta_{\\text{new}}} \\|f_{\\text{new}}(X; \\Theta_{\\text{new}}) - f_{\\text{old}}(X; \\Theta_{\\text{old}})\\|_F", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "res_svd": { "id": "res_svd", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "routing_attention": { "id": "routing_attention", "category": "B", "primary_formula": "\\mathbf{Q} = \\mathbf{X} \\mathbf{W}_Q, \\quad \\mathbf{K} = \\mathbf{X} \\mathbf{W}_K, \\quad \\mathbf{V} = \\mathbf{X} \\mathbf{W}_V \\quad \\in \\mathbb{R}^{B \\times H \\times L \\times d_h}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "rpc": { "id": "rpc", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "rpc_attention": { "id": "rpc_attention", "category": "A", "primary_formula": "\\widetilde{Q} = Q P \\in \\mathbb{R}^{B \\times H \\times L \\times d_p}, \\quad \\widetilde{K} = K P \\in \\mathbb{R}^{B \\times H \\times L \\times d_p}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "salt": { "id": "salt", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "salt_attention": { "id": "salt_attention", "category": "E", "primary_formula": "W = U_{:, :r} \\operatorname{diag}(\\Sigma_{1:r} \\odot \\gamma) V_{1:r, :}^\\top + A B", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "samba_m3": { "id": "samba_m3", "category": "D", "primary_formula": "\\mathbf{M}^{\\text{window}}_{i, j} = \\begin{cases} 0 & \\text{if } 0 \\le i - j < W \\\\ -\\infty & \\text{otherwise} \\end{cases}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "samba_moe": { "id": "samba_moe", "category": "D", "primary_formula": "\\mathbf{h}_t' = \\mathbf{A} \\mathbf{h}_{t-1}' + \\mathbf{B} \\mathbf{x}_t, \\quad \\mathbf{y}_{\\text{ssm}, t} = \\mathbf{C} \\mathbf{h}_t' + \\mathbf{D} \\mathbf{x}_t", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "simple_ssm": { "id": "simple_ssm", "category": "A", "primary_formula": "[X_{\\text{ssm}}, Z] = \\operatorname{chunk}(X W_{\\text{in}}, 2) \\in \\mathbb{R}^{B \\times L \\times D}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "smart_weighted_ensemble": { "id": "smart_weighted_ensemble", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "soa_attn": { "id": "soa_attn", "category": "E", "primary_formula": "W \\approx U_{:, :r} \\Sigma_{:r, :r} V_{:, :r}^T, \\quad r=64", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "soa_fused_module": { "id": "soa_fused_module", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "soa_module": { "id": "soa_module", "category": "E", "primary_formula": "W \\approx U_{:, :r} \\Sigma_{:r, :r} V_{:, :r}^T, \\quad r=64", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "span_expanded_attention": { "id": "span_expanded_attention", "category": "E", "primary_formula": "A = \\operatorname{Softmax}\\left(\\frac{Q K^\\top}{\\sqrt{d}} + M\\right)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "spectra_v2": { "id": "spectra_v2", "category": "B", "primary_formula": "M^{\\text{span}}_{h, i, j} = \\ln\\left( \\sigma\\left( 5.0 \\cdot (S_h - d_{ij}) \\right) + 10^{-5} \\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "spectra_v2_attention": { "id": "spectra_v2_attention", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "spectral_mamba": { "id": "spectral_mamba", "category": "D", "primary_formula": "Q = X W_Q, \\quad K = X W_K, \\quad V = X W_V", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "spectral_mamba_attention": { "id": "spectral_mamba_attention", "category": "D", "primary_formula": "h_t = \\bar{A} h_{t-1} + \\bar{B} x_t, \\quad y_t = C h_t + D x_t, \\quad \\bar{A} = \\exp(\\Delta A)", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "spectron": { "id": "spectron", "category": "E", "primary_formula": "\\min_{\\{r_i\\}} \\sum_{i=1}^M \\left(1 - \\frac{\\sum_{k=1}^{r_i} \\sigma_{i, k}^2}{\\sum_k \\sigma_{i, k}^2}\\right) \\quad \\text{s.t.} \\quad \\sum_{i=1}^M r_i (d_{i, \\text{in}} + d_{i, \\text{out}}) \\le B_{\\text{target}}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "spectron_attention": { "id": "spectron_attention", "category": "H", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "squeezing_heads": { "id": "squeezing_heads", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "squeezing_heads_distill": { "id": "squeezing_heads_distill", "category": "B", "primary_formula": "Q = X W_Q, \\quad K = X W_K, \\quad V = X W_V \\in \\mathbb{R}^{B \\times H \\times L \\times d}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "squeezing_heads_distiller": { "id": "squeezing_heads_distiller", "category": "B", "primary_formula": "\\mathcal{L} = \\alpha D_{\\text{KL}}(A_{\\text{teacher}} \\parallel A_{\\text{student}}) + \\beta \\|V_{\\text{teacher}} - V_{\\text{student}} W_{\\text{proj}}\\|_F^2", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "svd_attention": { "id": "svd_attention", "category": "E", "primary_formula": "X_{\\text{valid}} = X \\odot M_{\\text{valid}}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "svd_compressed": { "id": "svd_compressed", "category": "E", "primary_formula": "E_i(r) = \\frac{\\sum_{j=1}^r \\sigma_{i, j}^2}{\\sum_{j=1}^{\\min(d_{\\text{in}}, d_{\\text{out}})} \\sigma_{i, j}^2}, \\quad \\operatorname{Cost}_i(r) = 1 - E_i(r)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "svd_compressed_attention": { "id": "svd_compressed_attention", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "svd_linear_attention": { "id": "svd_linear_attention", "category": "D", "primary_formula": "\\\\text{Attention}(Q, K, V) = \\\\frac{\\\\phi(Q) (\\\\phi(K)^\\\\top V)}{\\\\phi(Q) \\\\sum_j \\\\phi(k_j)^\\\\top + \\\\epsilon}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "svd_linear_decay": { "id": "svd_linear_decay", "category": "D", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "svd_llm_v2": { "id": "svd_llm_v2", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "svd_llm_v2_attention": { "id": "svd_llm_v2_attention", "category": "E", "primary_formula": "\\frac{\\sum_{i=1}^{r^*} \\sigma_i^2}{\\sum_{j=1}^d \\sigma_j^2} \\ge \\tau \\quad (\\tau = 0.92)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "svd_mamba": { "id": "svd_mamba", "category": "D", "primary_formula": "W = U \\Sigma V^\\top \\approx W_{\\text{down}} W_{\\text{up}}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "svd_proj_attn": { "id": "svd_proj_attn", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "svd_projection_attention": { "id": "svd_projection_attention", "category": "E", "primary_formula": "W = U V \\in \\mathbb{R}^{d_{\\text{out}} \\times d_{\\text{in}}}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "svd_sequence_attention": { "id": "svd_sequence_attention", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "template_attention": { "id": "template_attention", "category": "B", "primary_formula": "Q = X W_Q, \\quad K = X W_K, \\quad V = X W_V", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "token_compressed": { "id": "token_compressed", "category": "B", "primary_formula": "\\min_{\\text{rank}(\\widehat{W}) \\le r} \\|W - \\widehat{W}\\|_F^2 = \\sum_{i=r+1}^{D} \\sigma_i^2", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "token_compressed_attention": { "id": "token_compressed_attention", "category": "G", "primary_formula": "\\text{energy}_i = \\|x_i\\|_2, \\quad \\text{score}_i = \\text{ScoreNet}(x_i)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "tpa_attention": { "id": "tpa_attention", "category": "E", "primary_formula": "q_1 = \\text{LoRALinear}_{Q1}(X) \\in \\mathbb{R}^{B \\times L \\times (H \\cdot r)}, \\quad q_2 = \\text{LoRALinear}_{Q2}(X) \\in \\mathbb{R}^{B \\times L \\times (H \\cdot r)}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "tt": { "id": "tt", "category": "B", "primary_formula": "\\text{Output} = \\text{BaseAttn}(X, M)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "unified_rope": { "id": "unified_rope", "category": "B", "primary_formula": "Q = \\operatorname{LoRALinear}_Q(X) \\in \\mathbb{R}^{B \\times L \\times (H \\cdot d_{\\text{head}})}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "upi_mamba": { "id": "upi_mamba", "category": "D", "primary_formula": "[X_{\\text{ssm}}, Z] = X W_{\\text{in}} \\in \\mathbb{R}^{B \\times L \\times 2 D}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "upi_stuffed_mamba": { "id": "upi_stuffed_mamba", "category": "D", "primary_formula": "[X_{\\text{ssm}}, Z] = X W_{\\text{in}} \\in \\mathbb{R}^{B \\times L \\times 2 D}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "vig_attention": { "id": "vig_attention", "category": "B", "primary_formula": "\\text{Total MACs} \\approx 4 B N D^2 + 4 B N D d_k", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "vspec": { "id": "vspec", "category": "E", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "weighted_ensemble": { "id": "weighted_ensemble", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "wgqa_attention": { "id": "wgqa_attention", "category": "A", "primary_formula": "c^{KV}_t = W^{DKV} x_t, \\quad k^C_t = W^{UK} c^{KV}_t, \\quad v^C_t = W^{UV} c^{KV}_t", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "xattention": { "id": "xattention", "category": "E", "primary_formula": "y = \\sum_{i=1}^K G(x)_i \\cdot E_i(x), \\quad G(x) = \\text{Softmax}(\\text{TopK}(x W_g, K=2))", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "zeros_attention": { "id": "zeros_attention", "category": "E", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "a3_compressor": { "id": "a3_compressor", "category": "E", "primary_formula": "\\text{FLOPs}_{\\text{dense}} = 2 \\cdot m \\cdot n \\quad \\longrightarrow \\quad \\text{FLOPs}_{\\text{factorized}} = 2 \\cdot r \\cdot (m + n)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "aa_svd_compressor": { "id": "aa_svd_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "aasvd": { "id": "aasvd", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top, \\quad \\min_{U, \\Sigma, V} \\left\\| (W - U \\Sigma V^\\top) X^\\top \\right\\|_F^2", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "adaptive_compressor": { "id": "adaptive_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "adaptive_ranks": { "id": "adaptive_ranks", "category": "E", "primary_formula": "\\\\frac{\\\\sum_{i=1}^r \\\\sigma_i^2}{\\\\sum_{j=1}^d \\\\sigma_j^2} \\\\ge \\\\tau \\\\quad (\\\\tau = 0.92)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "adasvd_compressor": { "id": "adasvd_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "admm_compressor": { "id": "admm_compressor", "category": "E", "primary_formula": "\\tau = \\text{TopK}_{k} \\left( \\{ |W_{i,j}| \\}_{i=1, j=1}^{N, M} \\right)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "admm_pruner": { "id": "admm_pruner", "category": "G", "primary_formula": "\\min_{W, Z} \\frac{1}{2} \\|W - W_0\\|_F^2 + g(Z) \\quad \\text{subject to} \\quad W - Z = 0", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "armor_pruner": { "id": "armor_pruner", "category": "E", "primary_formula": "W = W_{\\text{sparse}} + R", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "ast_pruner": { "id": "ast_pruner", "category": "G", "primary_formula": "\\Sigma_X = Q \\Lambda Q^\\top, \\quad Q^\\top Q = I_{d_{\\text{in}}}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "astro_quantizer": { "id": "astro_quantizer", "category": "H", "primary_formula": "W = S \\odot (Q_W - Z), \\quad Q_W \\in \\{-8, \\dots, 7\\}, \\quad S = \\text{scale}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "bonsai_pruner": { "id": "bonsai_pruner", "category": "G", "primary_formula": "s_t = s_{\\text{target}} \\cdot \\left( 1 - \\left( 1 - \\frac{t}{T} \\right)^3 \\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "cachenotes_compressor": { "id": "cachenotes_compressor", "category": "E", "primary_formula": "W_{\\text{cpt}} = \\operatorname{diag}(I_{\\text{task}}) W \\in \\mathbb{R}^{d_{\\text{out}} \\times d_{\\text{in}}}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "come": { "id": "come", "category": "A", "primary_formula": "M_{\\text{compressed}} = X W_{\\text{bridge}}, \\quad W_{\\text{bridge}} = U_r \\Sigma_r V_r^\\top", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "compot_compressor": { "id": "compot_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "coopq_quantizer": { "id": "coopq_quantizer", "category": "H", "primary_formula": "b_l \\\\ge b_{\\\\min} = 4, \\\\quad \\\\forall l \\\\in \\\\{1, \\\\dots, L\\\\}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "cr_compressor": { "id": "cr_compressor", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "cur_compressor": { "id": "cur_compressor", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "cur_linear": { "id": "cur_linear", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "cur_moe": { "id": "cur_moe", "category": "F", "primary_formula": "y = \\sum_{i=1}^K G(x)_i \\cdot E_i(x), \\quad G(x) = \\text{Softmax}(\\text{TopK}(x W_g, K=2))", "steps": [ { "id": 1, "name": "Router Gating", "formula": "G(x) = \text{Softmax}(\text{TopK}(x W_g, k))", "flops": 12288 }, { "id": 2, "name": "Expert Execution", "formula": "Y = \\sum_{e \\in \text{TopK}} G(x)_e E_e(x)", "flops": 2359296 } ], "total_flops_per_token": 2371584 }, "cur_moe_compressor": { "id": "cur_moe_compressor", "category": "F", "primary_formula": "W \\approx C \\cdot U \\cdot R, \\quad C = W_{[:, J]}, \\; R = W_{[I, :]}, \\; U = C^\\dagger W R^\\dagger", "steps": [ { "id": 1, "name": "Router Gating", "formula": "G(x) = \text{Softmax}(\text{TopK}(x W_g, k))", "flops": 12288 }, { "id": 2, "name": "Expert Execution", "formula": "Y = \\sum_{e \\in \text{TopK}} G(x)_e E_e(x)", "flops": 2359296 } ], "total_flops_per_token": 2371584 }, "cure": { "id": "cure", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "delta_kv": { "id": "delta_kv", "category": "A", "primary_formula": "R_K \\in \\mathbb{R}^{B \\times H \\times 1 \\times M \\times D}, \\quad R_V \\in \\mathbb{R}^{B \\times H \\times 1 \\times M \\times D}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "direct_svd": { "id": "direct_svd", "category": "E", "primary_formula": "\\min_{\\operatorname{rank}(\\hat{W}) \\le r} \\|W - \\hat{W}\\|_F = \\sqrt{\\sum_{i=r+1}^{\\min(d_{\\text{in}}, d_{\\text{out}})} \\sigma_i^2}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "dobi": { "id": "dobi", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "dobi_svd": { "id": "dobi_svd", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "dynamic_entropy": { "id": "dynamic_entropy", "category": "E", "primary_formula": "\\widetilde{W} = W C_X^{1/2} = U \\Sigma V^\\top", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "dynamic_rank": { "id": "dynamic_rank", "category": "E", "primary_formula": "H(\\Sigma_l) = -\\sum_{i=1}^d p_i \\ln(p_i), \\quad p_i = \\frac{\\sigma_i}{\\sum_{j=1}^d \\sigma_j}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "enlighten_compressor": { "id": "enlighten_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "entropy_budget_allocation": { "id": "entropy_budget_allocation", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "erc_svd": { "id": "erc_svd", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "faq_quantizer": { "id": "faq_quantizer", "category": "H", "primary_formula": "W = S \\odot (Q_W - Z), \\quad Q_W \\in \\{-8, \\dots, 7\\}, \\quad S = \\text{scale}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "flap": { "id": "flap", "category": "G", "primary_formula": "I(i) = \\frac{1}{N} \\sum_{t=1}^N |X_{t, i} - \\bar{X}_i| \\cdot \\|W_{:, i}\\|_2", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "flat_llm": { "id": "flat_llm", "category": "G", "primary_formula": "\\min_{W_r} \\text{Tr}((W - W_r) R_{XX} (W - W_r)^\\top)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "frandi": { "id": "frandi", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "frandi_compressor": { "id": "frandi_compressor", "category": "B", "primary_formula": "\\Omega \\sim \\mathcal{N}(0, I), \\; Y = (W W^\\top)^q W \\Omega, \\; Q = \\text{qr}(Y), \\; W \\approx Q (Q^\\top W)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "gfws_d_compressor": { "id": "gfws_d_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "grecodec": { "id": "grecodec", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "grecodec_compressor": { "id": "grecodec_compressor", "category": "E", "primary_formula": "\\min_{u, v} \\|W - u v^\\top\\|_F^2 \\implies u = \\frac{W v}{\\|v\\|^2}, \\; v = \\frac{W^\\top u}{\\|u\\|^2}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "hashed_linear": { "id": "hashed_linear", "category": "E", "primary_formula": "h(i, j) = \\left( i \\cdot P_1 + j \\cdot P_2 \\right) \\pmod B", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "hashed_net_compressor": { "id": "hashed_net_compressor", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "huffman_coder": { "id": "huffman_coder", "category": "E", "primary_formula": "f(a_k) = \\sum_{i=1}^N \\mathbb{I}(w_i = a_k), \\quad p(a_k) = \\frac{f(a_k)}{N}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "hybrid_svd_layer": { "id": "hybrid_svd_layer", "category": "E", "primary_formula": "W = U_r \\Sigma_r V_r^\\top + M_{\\text{sparse}}, \\quad \\|M_{\\text{sparse}}\\|_0 \\le k", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "hybridkv": { "id": "hybridkv", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "hybridkv_compressor": { "id": "hybridkv_compressor", "category": "I", "primary_formula": "\\sigma_i^2 = \\text{Var}_j(W_{i, j}) = \\frac{1}{D_{\\text{in}}} \\sum_{j=1}^{D_{\\text{in}}} \\left( W_{i, j} - \\mu_i \\right)^2 \\quad \\text{for } i \\in \\{1, \\dots, D_{\\text{out}}\\}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "kmeans_quantize": { "id": "kmeans_quantize", "category": "H", "primary_formula": "W = S \\odot (Q_W - Z), \\quad Q_W \\in \\{-8, \\dots, 7\\}, \\quad S = \\text{scale}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "kronecker_compressor": { "id": "kronecker_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "kronecker_linear": { "id": "kronecker_linear", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "kronecker_tt_lora": { "id": "kronecker_tt_lora", "category": "E", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "kvtc": { "id": "kvtc", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "kvtc_compressor": { "id": "kvtc_compressor", "category": "H", "primary_formula": "K_{\\text{comp}} = K V_K^\\top, \\; V_{\\text{comp}} = V V_V^\\top, \\quad V_K, V_V \\in \\mathbb{R}^{r \\times d_k}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "liftr": { "id": "liftr", "category": "A", "primary_formula": "Y = W \\Omega, \\quad Q = \\text{qr}(Y)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "lillama_compressor": { "id": "lillama_compressor", "category": "E", "primary_formula": "C_{YX} = \\frac{1}{N} Y^\\top X = \\frac{1}{N} \\left( X W^\\top + \\mathbf{1}_N b^\\top \\right)^\\top X = W \\left( \\frac{1}{N} X^\\top X \\right) + b \\left( \\frac{1}{N} \\mathbf{1}_N^\\top X \\right)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "lsc_compressor": { "id": "lsc_compressor", "category": "E", "primary_formula": "\\tilde{W}_{u,v} = \\sum_{n=0}^{N-1} \\sum_{m=0}^{M-1} W_{nm} \\cos\\left(\\frac{\\pi u (2n+1)}{2N}\\right) \\cos\\left(\\frac{\\pi v (2m+1)}{2M}\\right)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "mamba_shedder": { "id": "mamba_shedder", "category": "D", "primary_formula": "s_j = \\sqrt{ \\frac{1}{N_{\\text{tokens}}} \\sum_{i=1}^{N_{\\text{tokens}}} X_{i, j}^2 } \\quad \\text{for } j \\in \\{1, \\dots, D_{\\text{in}}\\}", "steps": [ { "id": 1, "name": "Forget Gate", "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", "flops": 1179648 }, { "id": 2, "name": "Input Gate", "formula": "i_t = \\sigma(W_i x_t)", "flops": 1179648 }, { "id": 3, "name": "Value Projection", "formula": "v_t = W_v x_t", "flops": 1179648 }, { "id": 4, "name": "Recurrent State Update", "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", "flops": 3072 } ], "total_flops_per_token": 3542016 }, "mgie_svd": { "id": "mgie_svd", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "mgie_svd_compressor": { "id": "mgie_svd_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "mlorq_compressor": { "id": "mlorq_compressor", "category": "E", "primary_formula": "W \\approx \\text{Quant}(A) \\cdot B^\\top + S \\cdot \\text{scale}, \\quad A \\in \\mathbb{R}^{d_{out} \\times r}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "modegpt": { "id": "modegpt", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "modegpt_compressor": { "id": "modegpt_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "nli_softmax": { "id": "nli_softmax", "category": "H", "primary_formula": "W' = Q_2^\\top W Q_1, \\quad W_{sliced} = W'[:d'_1, :d'_2], \\quad Q^\\top Q = I", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "nmf_compressor": { "id": "nmf_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "oats": { "id": "oats", "category": "B", "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "oats_compressor": { "id": "oats_compressor", "category": "E", "primary_formula": "\\min_{\\text{rank}(\\hat{W}) \\le r} \\|(W - \\hat{W}) L\\|_F^2", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "optimal_subspace_projection": { "id": "optimal_subspace_projection", "category": "E", "primary_formula": "C = \\\\frac{1}{N} X^\\\\top X + \\\\epsilon I", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "padre_attention": { "id": "padre_attention", "category": "B", "primary_formula": "c^{KV}_t = W^{DKV} x_t, \\quad k^C_t = W^{UK} c^{KV}_t, \\quad v^C_t = W^{UV} c^{KV}_t", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "proxsparse": { "id": "proxsparse", "category": "E", "primary_formula": "W = U_r \\Sigma_r V_r^\\top + S", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "proxsparse_pruner": { "id": "proxsparse_pruner", "category": "G", "primary_formula": "W^{(t+1)} = \\text{prox}_{\\lambda \\|\\cdot\\|_1}\\left(W^{(t)} - \\eta \\nabla f(W^{(t)})\\right) = \\text{sign}(W) \\max(0, |W| - \\lambda \\eta)", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "pruning_utils": { "id": "pruning_utils", "category": "G", "primary_formula": "S_{i, j} = |W_{i, j}| \\cdot \\|X_{:, j}\\|_2", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "replaceme_compressor": { "id": "replaceme_compressor", "category": "E", "primary_formula": "\\text{CKA}(X_l, X_{l+1}) = \\frac{\\|X_l^\\top X_{l+1}\\|_F^2}{\\|X_l^\\top X_l\\|_F \\|X_{l+1}^\\top X_{l+1}\\|_F}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "roast_hashed_linear": { "id": "roast_hashed_linear", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "rotational_slicegpt_int4": { "id": "rotational_slicegpt_int4", "category": "H", "primary_formula": "\\\\widetilde{W} = W Q, \\\\quad \\\\widetilde{X} = Q^\\\\top X, \\\\quad Q^\\\\top Q = I", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "rpca": { "id": "rpca", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "rpca_compressor": { "id": "rpca_compressor", "category": "E", "primary_formula": "\\min_{L, S} \\|L\\|_* + \\lambda \\|S\\|_1 \\quad \\text{s.t.} \\quad W = L + S, \\quad \\text{rank}(L) \\le r", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "saes_svd": { "id": "saes_svd", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "saes_svd_compressor": { "id": "saes_svd_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "saes_svd_v2": { "id": "saes_svd_v2", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top + \\text{SAE}(W - U_r \\Sigma_r V_r^\\top)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "salr": { "id": "salr", "category": "E", "primary_formula": "\\min_{\\text{rank}(\\widehat{W}) \\le r} \\|W - \\widehat{W}\\|_F^2 = \\sum_{i=r+1}^{D} \\sigma_i^2", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "sdrm": { "id": "sdrm", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "sdrm_compressor": { "id": "sdrm_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "slicegpt": { "id": "slicegpt", "category": "A", "primary_formula": "y = W x + b = W (I_{d_{\\text{in}}}) x + b = W (Q Q^\\top) x + b = (W Q) (Q^\\top x) + b = \\widetilde{W} \\widetilde{x} + b", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "slicegpt_compressor": { "id": "slicegpt_compressor", "category": "A", "primary_formula": "W_{\\text{pruned}} = (Q^\\top W Q)_{[:, \\mathcal{I}]}, \\quad Q = \\text{eigh}(X X^\\top), \\; |\\mathcal{I}| = (1 - 0.45) d_{in}", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "slicegpt_pca": { "id": "slicegpt_pca", "category": "E", "primary_formula": "\\Sigma = \\frac{1}{N - 1} X_c^\\top X_c + \\epsilon I_{d_{\\text{in}}}", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "slim_compressor": { "id": "slim_compressor", "category": "E", "primary_formula": "W_q = \\\\operatorname{Quant}_b(W_0)", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "soka": { "id": "soka", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "soka_compressor": { "id": "soka_compressor", "category": "E", "primary_formula": "W = U \\\\Sigma V^\\\\top \\\\approx U_r \\\\Sigma_r V_r^\\\\top = A B^\\\\top", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "sparsegpt": { "id": "sparsegpt", "category": "E", "primary_formula": "\\Delta W = -\\frac{w_q}{[H^{-1}]_{qq}} H_{:, q}^{-1}, \\quad H = 2 X X^\\top", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "spectraformer_attention": { "id": "spectraformer_attention", "category": "B", "primary_formula": "c^{KV}_t = W^{DKV} x_t, \\quad k^C_t = W^{UK} c^{KV}_t, \\quad v^C_t = W^{UV} c^{KV}_t", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "subspace_projection": { "id": "subspace_projection", "category": "E", "primary_formula": "\\min_{P, \\{C_k\\}} \\sum_{k=1}^K \\|W_k - P C_k\\|_F^2 \\quad \\text{s.t.} \\quad P^\\top P = I", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "svd_linear": { "id": "svd_linear", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = (U_r \\sqrt{\\Sigma_r}) (\\sqrt{\\Sigma_r} V_r^\\top) = A B^\\top", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "swift_svd": { "id": "swift_svd", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "swift_svd_act": { "id": "swift_svd_act", "category": "E", "primary_formula": "\\min_{A, B} \\left\\|(W - A B) \\Sigma_{XX}^{1/2}\\right\\|_F^2, \\quad \\Sigma_{XX} = \\frac{1}{N} X X^\\top", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "swift_svd_compressor": { "id": "swift_svd_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "tlinformer_attention": { "id": "tlinformer_attention", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "tomoe": { "id": "tomoe", "category": "F", "primary_formula": "y = \\sum_{i=1}^K G(x)_i \\cdot E_i(x), \\quad G(x) = \\text{Softmax}(\\text{TopK}(x W_g, K=2))", "steps": [ { "id": 1, "name": "Router Gating", "formula": "G(x) = \text{Softmax}(\text{TopK}(x W_g, k))", "flops": 12288 }, { "id": 2, "name": "Expert Execution", "formula": "Y = \\sum_{e \\in \text{TopK}} G(x)_e E_e(x)", "flops": 2359296 } ], "total_flops_per_token": 2371584 }, "tomoe_compressor": { "id": "tomoe_compressor", "category": "F", "primary_formula": "y = \\sum_{i=1}^K G(x)_i \\cdot E_i(x), \\quad G(x) = \\text{Softmax}(\\text{TopK}(x W_g, K=2))", "steps": [ { "id": 1, "name": "Router Gating", "formula": "G(x) = \text{Softmax}(\text{TopK}(x W_g, k))", "flops": 12288 }, { "id": 2, "name": "Expert Execution", "formula": "Y = \\sum_{e \\in \text{TopK}} G(x)_e E_e(x)", "flops": 2359296 } ], "total_flops_per_token": 2371584 }, "tt_compressor": { "id": "tt_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "tt_linear": { "id": "tt_linear", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "uniql_compressor": { "id": "uniql_compressor", "category": "E", "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", "steps": [ { "id": 1, "name": "Low-Rank V-Factor", "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", "flops": 98304 }, { "id": 2, "name": "Low-Rank U-Factor", "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", "flops": 98304 } ], "total_flops_per_token": 196608 }, "zerokv_neo": { "id": "zerokv_neo", "category": "A", "primary_formula": "c^{KV}_t = W^{DKV} x_t, \\quad k^C_t = W^{UK} c^{KV}_t, \\quad v^C_t = W^{UV} c^{KV}_t", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 }, "zerokv_neo_compressor": { "id": "zerokv_neo_compressor", "category": "A", "primary_formula": "c^{KV}_t = W^{DKV} x_t, \\quad k^C_t = W^{UK} c^{KV}_t, \\quad v^C_t = W^{UV} c^{KV}_t", "steps": [ { "id": 1, "name": "QKV Projections", "formula": "Q, K, V = X W_Q, X W_K, X W_V", "flops": 3538944 }, { "id": 2, "name": "Attention Materialization", "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", "flops": 786432 }, { "id": 3, "name": "Value Aggregation", "formula": "O = A V", "flops": 786432 } ], "total_flops_per_token": 5111808 } }
{
"SOAFusedAttention": {
"id": "SOAFusedAttention",
"category": "E",
"primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})",
"steps": [
{
"id": 1,
"name": "Low-Rank V-Factor",
"formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}",
"flops": 98304
},
{
"id": 2,
"name": "Low-Rank U-Factor",
"formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}",
"flops": 98304
}
],
"total_flops_per_token": 196608
},
"adaptive_span_attention": {
"id": "adaptive_span_attention",
"category": "C",
"primary_formula": "S_{ij} = \\frac{Q_i K_j^\\top}{\\sqrt{d}}",
"steps": [
{
"id": 1,
"name": "QKV Projections",
"formula": "Q, K, V = X W_Q, X W_K, X W_V",
"flops": 3538944
},
{
"id": 2,
"name": "Attention Materialization",
"formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})",
"flops": 786432
},
{
"id": 3,
"name": "Value Aggregation",
"formula": "O = A V",
"flops": 786432
}
],
"total_flops_per_token": 5111808
},
"adaptive_span_inject": {
"id": "adaptive_span_inject",
"category": "C",
"primary_formula": "S_{ij} = \\frac{Q_i K_j^\\top}{\\sqrt{d}}",
"steps": [
{
"id": 1,
"name": "QKV Projections",
"formula": "Q, K, V = X W_Q, X W_K, X W_V",
"flops": 3538944
},
{
"id": 2,
"name": "Attention Materialization",
"formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})",
"flops": 786432
},
{
"id": 3,
"name": "Value Aggregation",
"formula": "O = A V",
"flops": 786432
}
],
"total_flops_per_token": 5111808
},
"adaptive_sparse_attention": {
"id": "adaptive_sparse_attention",
"category": "E",
"primary_formula": "\\gamma_h = \\sigma(\\text{logit}_h) \\in (0, 1)",
"steps": [
{
"id": 1,
"name": "Low-Rank V-Factor",
"formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}",
"flops": 98304
},
{
"id": 2,
"name": "Low-Rank U-Factor",
"formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}",
"flops": 98304
}
],
"total_flops_per_token": 196608
},
"adaptive_sparse_inject": {
"id": "adaptive_sparse_inject",
"category": "B",
"primary_formula": "\\gamma_h = \\sigma(\\text{logit}_h) \\in (0, 1)",
"steps": [
{
"id": 1,

Xet Storage Details

Size:
169 kB
·
Xet hash:
fce7b5ae6c62a64ba2e0f70001e74f5f1158cfa0fd514c01ee25a5e072c32544

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.