Remove cached API responses from tracking

data/cache/059867531490189c3a0afb5010753188.json

@@ -1 +0,0 @@
-[{"url": "https://arxiv.org/html/2508.06297v1", "snippet": "KVcachecompressionis a key technology for optimizing the inference efficiency ofLLMs, primarily by compressing the key and value tensors in the self-attention mechanism to reduce memory usage and improve computational efficiency.", "title": "KV Cache Compression for Inference Efficiency in LLMs: A Review - arXiv.org", "inferred_year": null, "hybrid_score": 0.0, "source": "duckduckgo"}, {"url": "https://arxiv.org/html/2502.01941v1", "snippet": "Abstract Thispaperinvestigates an under-explored challenge in large language models (LLMs): the impact ofKVcachecompressionmethodsonLLMs'fundamental capabilities. While existingmethodsachieve impressivecompressionratios on long-context benchmarks, their effects on core model capabilities remain understudied.", "title": "Can LLMs Maintain Fundamental Abilities under KV Cache Compression?", "inferred_year": null, "hybrid_score": 0.0, "source": "duckduckgo"}, {"url": "https://arxiv.org/pdf/2508.06297", "snippet": "KVcachecompressionis a key technology for optimizing the inference eficiency ofLLMs, primarily by compressing the key and value tensors in the self-attention mechanism to reduce memory usage and improve computational eficiency.", "title": "KV Cache Compression for Inference Efficiency in LLMs: A Review - arXiv.org", "inferred_year": null, "hybrid_score": 0.0, "source": "duckduckgo"}]

data/cache/43446953e7780e5d910ebed8e6e73ddf.json

@@ -1 +0,0 @@
-[{"title": "WindowKV: Task-Adaptive Group-Wise KV Cache Window Selection for Efficient LLM Inference", "abstract": "With the advancements in long-context inference capabilities of large language models (LLMs), the KV cache has become one of the foundational components. However, its substantial GPU memory consumption makes KV cache compression a key technique for enabling efficient LLM inference in industrial scenarios. While recent studies have focused on optimizing the memory occupied by the KV cache, they overlook two critical factors: preserving semantic coherence and considering task-specific characteristic during compression. To address these limitations, we propose a novel task-adaptive KV cache window selection method, WindowKV. WindowKV dynamically selects local semantic windows consisting of consecutive tokens, according to task-specific characteristics, ensuring the retained KV cache captures continuous, essential context. Additionally, we introduce an intra-group layer KV cache indices sharing strategy to reduce computational overhead, achieving a balance between performance and efficiency. We rigorously evaluate WindowKV on the LongBench benchmark, and the results demonstrate that it maintains a performance comparable to full KV cache retention while using only 12% of the original KV cache, significantly reducing memory requirements. Furthermore, our method also achieves state-of-the-art results in the Needle-in-a-Haystack evaluation, highlighting its effectiveness and robustness.", "year": 2025, "citation_count": 1, "paper_id": "982a6b235ccfe7fe7fe9a9b3b97dbdae58305172", "authors": ["Youhui Zuo", "Sibo Wei", "Chen Zhang", "Zhuorui Liu", "Wenpeng Lu", "Dawei Song"], "references": ["ef094815dc0118c8d1cc06d4bb3aa78bae0a3877", "a7ebfd44de6eb3d9df4a2954c348ff1e17349a66", "a7f2987eb07d92033a7b1cadbfc94e6c1489f9bc", "70b49c248edee60d0ad5c53b5010f480f0a3828b", "6c67e17fdd763f212e582f8de56b56dd5ed58832", "812356c723c082f88fb722531beaf45e344ffa1e", "2f96229c404a5cbd0c0d06492f5ea3d6bfcf50d2", "8b8a7f1ac390a2394802234d3c539da86c56de66", "1d4c48335d841014d0145256c3c4e7f6c426b8fb", "53a803388e83ae89261624099d7be4287ace67cb", "edd705ebe3546272b7fe952e2ed6088200adad76", "4c69d79c0ee7ac964284a75135b317d1ce7fb2d6", "c0b454e0a6aa51ff3ba56778787d0c43932ef6ba", "f288e2238ac8725baa7ca9874bbc3fed1e89a632", "ef1b02dc1b82f9955fc4760fcefd92c0fff9f227", "4ea5ca620122e6a9a2b000444d36491cebf49c7c", "6c323c535365e1c7cbfd9703cbec3b5650a3346b", "fdc53c2c10742464087c0525f77e32604827a21d", "b31a5884a8ebe96b6300839b28608b97f8f8ef76", "d6eeb2898bd9bd34744194ef543062dda6c4531a", "7bd4ca8706a79983d31ab74e6c79bfdfd949602e", "5ae6fb6b5a3c7df515ff4a82ac9673bae6a8e200", "bafe023fb072045dc0cd50316382a61c8dcb9fae", "163b4d6a79a5b19af88b8585456363340d9efd04", "57e849d0de13ed5f91d086936296721d4ff75a75", "30c0cdc414f68211d5d0514df027cec22e005174", "33be243ac9dd8723e6267dea45fd6a6172d4f6a5", "9dc624d7258d1a56117ca720aea953ce46b66b21", "dc52b09089704ebd6f471177474bc29741c50023", "204e3073870fae3d05bcbc2f6a8e263d9b72e776", "06e0f8f409ce76dbc88e1f811dda5058b3fa7e29", "87c8a7be8d5e2e2209e766c3e28a3e8ee5babb64"], "hybrid_score": 0.5059, "source": "semantic_scholar"}, {"title": "EliteKV: Scalable KV Cache Compression via RoPE Frequency Selection and Joint Low-Rank Projection", "abstract": "Rotary Position Embedding (RoPE) enables each attention head to capture multi-frequency information along the sequence dimension and is widely applied in foundation models. However, the nonlinearity introduced by RoPE complicates optimization of the key state in the Key-Value (KV) cache for RoPE-based attention. Existing KV cache compression methods typically store key state before rotation and apply the transformation during decoding, introducing additional computational overhead. This paper introduces EliteKV, a flexible modification framework for RoPE-based models supporting variable KV cache compression ratios. EliteKV first identifies the intrinsic frequency preference of each head using RoPElite, selectively restoring linearity to certain dimensions of key within attention computation. Building on this, joint low-rank compression of key and value enables partial cache sharing. Experimental results show that with minimal uptraining on only $0.6\\%$ of the original training data, RoPE-based models achieve a $75\\%$ reduction in KV cache size while preserving performance within a negligible margin. Furthermore, EliteKV consistently performs well across models of different scales within the same family.", "year": 2025, "citation_count": 1, "paper_id": "4a7716817d20b0355618f268937fd2fe9574f620", "authors": ["Yuhao Zhou", "Sirui Song", "Boyang Liu", "Zhiheng Xi", "Senjie Jin", "Xiaoran Fan", "Zhihao Zhang", "Wei Li", "Xuanjing Huang"], "references": [], "hybrid_score": 0.4931, "source": "semantic_scholar"}, {"title": "LiveVLM: Efficient Online Video Understanding via Streaming-Oriented KV Cache and Retrieval", "abstract": "Recent developments in Video Large Language Models (Video LLMs) have enabled models to process long video sequences and demonstrate remarkable performance. Nonetheless, studies predominantly focus on offline video question answering, neglecting memory usage and response speed that are essential in various real-world applications, such as Deepseek services, autonomous driving, and robotics. To mitigate these challenges, we propose $\\textbf{LiveVLM}$, a training-free framework specifically designed for streaming, online video understanding and real-time interaction. Unlike existing works that process videos only after one question is posed, LiveVLM constructs an innovative streaming-oriented KV cache to process video streams in real-time, retain long-term video details and eliminate redundant KVs, ensuring prompt responses to user queries. For continuous video streams, LiveVLM generates and compresses video key-value tensors (video KVs) to reserve visual information while improving memory efficiency. Furthermore, when a new question is proposed, LiveVLM incorporates an online question-answering process that efficiently fetches both short-term and long-term visual information, while minimizing interference from redundant context. Extensive experiments demonstrate that LiveVLM enables the foundation LLaVA-OneVision model to process 44$\\times$ number of frames on the same device, and achieves up to 5$\\times$ speedup in response speed compared with SoTA online methods at an input of 256 frames, while maintaining the same or better model performance.", "year": 2025, "citation_count": 21, "paper_id": "8aef30425e454ddc3440bc11925adba4e972d451", "authors": ["Zhenyu Ning", "Guangda Liu", "Qihao Jin", "Wenchao Ding", "Minyi Guo", "Jieru Zhao"], "references": ["2024e5247104cf2261314e813e0bf8d48467ec43", "5c308e16788bb80d9a6292c05448d319928f0be5", "8a925f0d95356be426f17c63e89c2eb18f32f37f", "933e1246c24fed43eeb44e354796aa76c21746ca", "f61cc9b5583c6295d5cd756ec0f34e4c003aab29", "9ab991106044733043922fee457a1e3311060c2a", "f8d105e957b04e94f23c1d5ee45afac5b3c99c7f", "5f49ec9560ca9e03eff32a607f6caabc08f98926", "958e80006e3b62a0b41e50877d9e554c339615a7", "7943ec4a67151a559b25cd34369e661c9a7924c8", "ad6f68db45aaebc0e61b342d03da4c2702ce5697", "a22b23b65cdbc0335b872ed7c7a7206c710f8997", "4d5b247c1274ca953aaf91e2c2094b69d8ce183d", "e9938183e43ddd343543da1c2faa67e48115c4b2", "1a71f7b216b710b936da666027014adb83af8e7a", "40e8af970329135ec95057d73e239dab805ad128", "31a1bbafebae80a1024302b34618fc98ecd80e98", "9d21874dc08a0b3cef93a0273f0752080a372f4e", "9803d83bbb28d02fb01f00e0e05aa3c192a87255", "d081584960c42f7793502bb496e46f682e3e43b3", "d019f8a137f62cd2c08ef42be54bbb980a8362f2", "f9cfedb64717235ede7e5921c76f94acb3899ef9", "c6dd31c40836d616b50feab17aff569591c4262e", "22552dd0e7789f175a302055f06444a12e22b652", "6df24cfb2ded90dfa443139c2e3fb6317179a358", "58aa6e143156e59e6d7d902c56a3275ef7a2287d", "887306ee7ebd9eebd7faea24106ec8e8f1a50987", "9d29da83aba362c728c36f4dea9dde678ae3e2b2", "1784c987e681d60c634765fe64c8d9c26f73d5ff", "49873ee415619efd9e1e4c16f73ee066ff008c1f", "f32b5b59fc4a988c49b112e4d4a06d684d4f117a", "c7453d3e481cd419ec103af45aed2a76ac4d55e3", "3421d12d98956c36c5d39c0ef023b378b6922068", "c811bedbe8f4c21d0cba9f9175f7c2eb203284a7", "8296eef3797afd1515021ff568a694412c38101b", "2141ed804636a1cf339d606cd03fd3b3e9582133", "486c2df78cbb770a90a55f7fa3fe19102fba2c24", "107fb6eec2febbae12db29bf3e311aaf5680027c", "aad3d2e690f6c73f04a14622ceff51464bbc560e", "124d4d374fbef2016fa9880489871a58a7450644", "fdc53c2c10742464087c0525f77e32604827a21d", "656a6b3c0348d69cf9f98f95cbf68046941a4f29", "bf7025a2e5dbb3c09deae02a1aa98a256ca559e2", "00cb69a9f280317d1c59ac5827551ee9b10642b8", "35aba190f28b5c39df333c06ca21f46bd4845eba", "3f5b31c4f7350dc88002c121aecbdc82f86eb5bb", "78281482c1fdad8e167bab39cc9955c73d58ae8f", "87c5b281fa43e6f27191b20a8dd694eda1126336", "202967f77c4384bce80eaf2fa5737259008267d3", "6f870f7f02a8c59c3e23f407f3ef00dd1dcf8fc4", "057b80e235b10799d03876ad25465208a4c64caf", "b2f521c02c6ed3080c5fe123e938cdf4555e6fd2", "5b6fdb2aea424d02c141da81d62b04d739d62b96", "c735320ee0e5b2e580043b0a1504e592c0840e24", "33c3dbf86dd6e338e2a49bba9c2e2193d1eca08a"], "hybrid_score": 0.471, "source": "semantic_scholar"}, {"title": "Yan: Foundational Interactive Video Generation", "abstract": "We present Yan, a foundational framework for interactive video generation, covering the entire pipeline from simulation and generation to editing. Specifically, Yan comprises three core modules. AAA-level Simulation: We design a highly-compressed, low-latency 3D-VAE coupled with a KV-cache-based shift-window denoising inference process, achieving real-time 1080P/60FPS interactive simulation. Multi-Modal Generation: We introduce a hierarchical autoregressive caption method that injects game-specific knowledge into open-domain multi-modal video diffusion models (VDMs), then transforming the VDM into a frame-wise, action-controllable, real-time infinite interactive video generator. Notably, when the textual and visual prompts are sourced from different domains, the model demonstrates strong generalization, allowing it to blend and compose the style and mechanics across domains flexibly according to user prompts. Multi-Granularity Editing: We propose a hybrid model that explicitly disentangles interactive mechanics simulation from visual rendering, enabling multi-granularity video content editing during interaction through text. Collectively, Yan offers an integration of these modules, pushing interactive video generation beyond isolated capabilities toward a comprehensive AI-driven interactive creation paradigm, paving the way for the next generation of creative tools, media, and entertainment. The project page is: https://greatx3.github.io/Yan/.", "year": 2025, "citation_count": 6, "paper_id": "68aead5ee7904eac31b8c00627c671ac27d6afca", "authors": ["Deheng Ye", "Fangyun Zhou", "Jiacheng Lv", "Jian Ma", "Jun Zhang", "Jun Lv", "Junyou Li", "Minwen Deng", "Mingyu Yang", "Qiang Fu", "Wei Yang", "Wenkai Lv", "Yangbin Yu", "Yewen Wang", "Yonghang Guan", "Zhihao Hu", "Zhongbin Fang", "Zhongqian Sun"], "references": ["a8e2e3ff1770fd83228659e9e4d16114ddb9404b", "d0aef077f96cc06279b7a8e0a09c37623f7b4939", "492910d028c0094280f9243c4c9f270a031d22e6", "8257f34c0cb0c69ccf7df4d2a9423c42a8f4b052", "8877e9a5bf473404c35b758faa76905c202f0f36", "81a3c48930c06d8cdb6edcdeea2144412af00912", "f61cc9b5583c6295d5cd756ec0f34e4c003aab29", "8cc8db4b3846b7d9f33140af1a7353e4a0cb2863", "bcfecb076763b8cff41bc8719e7486fe759d6e62", "1994f886b74aff3ef31802769f2c502dbe0b25aa", "b95baf93d1cfb406b50aa370ce5ba18788b5891d", "047f0dcc0c73edc222be828136e50c335137545f", "9ca72cef4487494cd835e7abca65794407db62ee", "c0900c4149df25531488fae3cd6a5aac0139d0d4", "1fa298e3f745099d39ca5bd088fcb62f87e8cc2f", "5ad057577eac072aa70776c8faf83ef9787ab60c", "fe1cdf9d438af4b31e7d7cb3f53d2d3684f4f704", "75d7591078a74aa6bfcdaf5bde7dbe5146a45ecd", "21e3937cceb3320bdcdd4727b9e2b83b6529de8e", "40d63dc2b465c9081e4efc5a19514da151e97fe7", "cc6fc3c546b354abf6a0aa3b553f28a6b812489f", "02f4516502bc0d05fb8971687f37c5f319ca2704", "41a66997ce0a366bba3becf7c3f37c9aebb13fbd", "8eae862d9669e7001eeee17b49fba793df9672c4", "c064de2c71ebc5cf05493f49dc312b033c36b3b9", "6d8efe141ead5e397ca9a49814d049290292e974", "42d8143a2c803a2cf328f0b568a57dcd5b1dedf5", "516a4c05883d4dc130bb7a638ad8e427b6756aac", "c3d14e7a319ab764297a60112ce74af201762a73", "84f0a99d0f0015a6145c94468870d43ab1d166fd", "369b449415d50387fba048bbd4d26ee890df84b5", "c1caa303549764d220ff17dc1785985dd1ba6047", "ba184d335a9a08c52c5d25eabd7f4a8ea987918b", "36709000a9272e941338244ef80ff5ab1dd2bba1", "32a3c2fbd3e733bd0eea938517fec2ff8dc7c701", "163b4d6a79a5b19af88b8585456363340d9efd04", "6283502d6900a0b403e2454b1cb1cf16ddefd5a7", "efbe97d20c4ffe356e8826c01dc550bacc405add", "94b690162ead76af6a487d6e10998ea585c035d1", "c10075b3746a9f3dd5811970e93c8ca3ad39b39d", "6f870f7f02a8c59c3e23f407f3ef00dd1dcf8fc4", "014576b866078524286802b1d0e18628520aa886", "5c126ae3421f05768d8edd97ecd44b1364e2c99a", "bb94773450659ccda567a1c4dea389363b9d4c68", "c468bbde6a22d961829e1970e6ad5795e05418d1", "dce6f9d4017b1785979e7520fd0834ef8cf02f4b", "03a5b2aac53443e6078f0f63b35d4f95d6d54c5d", "a473f545318325ba23b7a6b477485d29777ba873", "8ba7d6525f7f759d439e1b066789215201b010f0", "ef4f5a50837a7c1b3e87b9300ffc7ba00d461a0f", "387ab57cf27e7b8b10e734cd88da5b1c11edc6c7"], "hybrid_score": 0.4371, "source": "semantic_scholar"}]

data/cache/cc860319625f8f1b0658f48c26ada610.json

@@ -1 +0,0 @@
-[{"url": "https://arxiv.org/html/2508.06297v1", "snippet": "August 8, 2025 -We focus on identifying the limitations and challenges of existing methods, such ascompatibility issues with different models and tasks. Additionally, this review highlights future research directions, including hybrid optimization techniques, adaptive dynamic strategies, and software-hardware ...", "title": "KV Cache Compression for Inference Efficiency in LLMs: A Review", "inferred_year": 2025, "hybrid_score": 0.0, "source": "duckduckgo"}, {"url": "https://arxiv.org/html/2502.01941v2", "snippet": "May 21, 2025 -This paper investigates an underexplored challenge in large language models (LLMs): the impact of KV cache compression methods on LLMs\u2019 fundamental capabilities. Although existing methods achieve impressive compression ratios on long-context benchmarks, their effects on core model capabilities remain understudied.", "title": "Can LLMs Maintain Fundamental Abilities under KV Cache Compression?", "inferred_year": 2025, "hybrid_score": 0.0, "source": "duckduckgo"}, {"url": "https://arxiv.org/abs/2502.01941", "snippet": "May 21, 2025 -This paper investigates an underexplored challenge in large language models (LLMs):the impact of KV cache compression methods on LLMs' fundamental capabilities. Although existing methods achieve impressive compression ratios on long-context ...", "title": "[2502.01941] Can LLMs Maintain Fundamental Abilities under KV Cache Compression?", "inferred_year": 2025, "hybrid_score": 0.0, "source": "duckduckgo"}]

data/cache/e5c4fcb986d222d9f17c613ce1b46edb.json

@@ -1 +0,0 @@
-[{"title": "KIVI: A Tuning-Free Asymmetric 2bit Quantization for KV Cache", "abstract": "Efficiently serving large language models (LLMs) requires batching of many requests to reduce the cost per request. Yet, with larger batch sizes and longer context lengths, the key-value (KV) cache, which stores attention keys and values to avoid re-computations, significantly increases memory demands and becomes the new bottleneck in speed and memory usage. Additionally, the loading of the KV cache causes the computational core to be idle, which limits the inference speed. A straightforward and effective solution to reduce KV cache size is quantization, which decreases the total bytes taken by KV cache. However, there is a lack of in-depth studies that explore the element distribution of KV cache to understand the hardness and limitation of KV cache quantization. To fill the gap, we conducted a comprehensive study on the element distribution in KV cache of popular LLMs. Our findings indicate that the key cache should be quantized per-channel, i.e., group elements along the channel dimension and quantize them together. In contrast, the value cache should be quantized per-token. From this analysis, we developed a tuning-free 2bit KV cache quantization algorithm named KIVI. With hardware-friendly implementation, KIVI can enable Llama, Falcon, and Mistral models to maintain almost the same quality while using $\\mathbf{2.6\\times}$ less peak memory (including model weight). This reduction in memory usage enables up to $\\mathbf{4\\times}$ larger batch size, bringing $\\mathbf{2.35\\times \\sim 3.47\\times}$ throughput on real LLM inference workload. The source code is available at https://github.com/jy-yuan/KIVI.", "year": 2024, "citation_count": 392, "paper_id": "a3e000e0d7f64c1d094c2a8bf6f43992cbabe91b", "authors": ["Zirui Liu", "Jiayi Yuan", "Hongye Jin", "Shaochen Zhong", "Zhaozhuo Xu", "Vladimir Braverman", "Beidi Chen", "Xia Hu"], "references": ["fbfe920579cc1c13358521d403cfce31f2afbead", "db52731a663fae1582eecdd16e88db213f8b2a74", "c811bedbe8f4c21d0cba9f9175f7c2eb203284a7", "b085968c4362fb286ad6c5ef71a5db9630da0498", "9529e50807f36acf3d2e4af994b5803c47e4746a", "db633c6b1c286c0386f0078d8a2e6224e03a6227", "fdc53c2c10742464087c0525f77e32604827a21d", "83b90f4a0ae4cc214eb3cc140ccfef9cd99fac05", "b31a5884a8ebe96b6300839b28608b97f8f8ef76", "104b0bb1da562d53cbda87aec79ef6a2827d191a", "3b7ef6f9f27e33e6a4e3bfac90dcb01ab09718bc", "0423fc7bc1880b850d07aec8ebd9217a70626572", "7a1e71cb1310c4a873e7a4e54d1a6dab0553adce", "d6eeb2898bd9bd34744194ef543062dda6c4531a", "50eb97f832ffcd2114f79957c977215176384e3d", "60b35c6d68acced19b0c66edcfc0ee0a2c11efed", "5ae6fb6b5a3c7df515ff4a82ac9673bae6a8e200", "5f187af087ebbaf1ce4bca686a4b1c2afee92b6d", "15dd43ded15e6dbf750278430bd822ee2d1b977f", "42a14d824caa3348046eb34c37e2ab7985faa7a3", "57e849d0de13ed5f91d086936296721d4ff75a75", "2c994fadbb84fb960d8306ee138dbeef41a5b323", "7d645a3fd276918374fd9483fd675c28e46506d1", "7da0f2501034522e3d50af7e9b8fa7ec9d7b65b6", "4be7d1524edb0137599a5cc95f72844b85a52fe1", "90abbc2cf38462b954ae1b772fac9532e2ccd8b0", "dc52b09089704ebd6f471177474bc29741c50023", "2b7c9fd2a94deaee3e7e56dc57bab0bd39d3683c", "b378e54c88d241aa917131beb65c96be3730f40c"], "hybrid_score": 0.6509, "source": "semantic_scholar"}, {"title": "KVQuant: Towards 10 Million Context Length LLM Inference with KV Cache Quantization", "abstract": "LLMs are seeing growing use for applications which require large context windows, and with these large context windows KV cache activations surface as the dominant contributor to memory consumption during inference. Quantization is a promising approach for compressing KV cache activations; however, existing solutions fail to represent activations accurately in sub-4-bit precision. Our work, KVQuant, facilitates low precision KV cache quantization by incorporating several novel methods: (i) Per-Channel Key Quantization, where we adjust the dimension along which we quantize the Key activations to better match the distribution; (ii) Pre-RoPE Key Quantization, where we quantize Key activations before the rotary positional embedding to mitigate its impact on quantization; (iii) Non-Uniform KV Cache Quantization, where we derive per-layer sensitivity-weighted non-uniform datatypes that better represent the distributions; and (iv) Per-Vector Dense-and-Sparse Quantization, where we isolate outliers separately for each vector to minimize skews in quantization ranges. By applying our method to the LLaMA, Llama-2, Llama-3, and Mistral models, we achieve<0.1 perplexity degradation with 3-bit quantization on both Wikitext-2 and C4, outperforming existing approaches. Our method enables serving LLaMA-7B with a context length of up to 1 million on a single A100-80GB GPU and up to 10 million on an 8-GPU system. We develop custom CUDA kernels for KVQuant, showing that we can achieve up to ~1.7x speedups, compared to baseline fp16 matrix-vector multiplications, for the LLaMA-7B model.", "year": 2024, "citation_count": 434, "paper_id": "b085968c4362fb286ad6c5ef71a5db9630da0498", "authors": ["Coleman Hooper", "Sehoon Kim", "Hiva Mohammadzadeh", "Michael W. Mahoney", "Y. Shao", "Kurt Keutzer", "A. Gholami"], "references": ["ac5824e9ff924a937d9eef379d0b581de2417678", "4c14b1c41cb0aaa68f5d3f4a432f55e7199657ea", "713806165610c237f551a7b68e6b09b3ded75502", "9529e50807f36acf3d2e4af994b5803c47e4746a", "db633c6b1c286c0386f0078d8a2e6224e03a6227", "6c323c535365e1c7cbfd9703cbec3b5650a3346b", "fdc53c2c10742464087c0525f77e32604827a21d", "b6346f9fa093b8e85df712485a2b851b9f680dac", "73290ecbec2f38d1d647ddef1ada69cee41725b3", "464cf829eaaeb2b3bafc84cc9203790e95102049", "b31a5884a8ebe96b6300839b28608b97f8f8ef76", "eb2c2330177f765038a2b17e2ee3498965865797", "104b0bb1da562d53cbda87aec79ef6a2827d191a", "f5afaccfe90268485a9961c5771ec5e71e9b806c", "3b7ef6f9f27e33e6a4e3bfac90dcb01ab09718bc", "51db4c39dc0bdf5c95c8bbe89bf4211b48d0b4df", "6bd3ee1ca608bc66a490f63f2fb107d79b44f3e2", "d6eeb2898bd9bd34744194ef543062dda6c4531a", "60b35c6d68acced19b0c66edcfc0ee0a2c11efed", "32ac52069e562d4f900afee70bdca63f53461481", "42a14d824caa3348046eb34c37e2ab7985faa7a3", "0a6906bd6f026d3da3031c641ed03081bd0b574e", "57e849d0de13ed5f91d086936296721d4ff75a75", "2c994fadbb84fb960d8306ee138dbeef41a5b323", "3f6243097a58e386aea1215fed4f372dee07a100", "4be7d1524edb0137599a5cc95f72844b85a52fe1", "73bcf4577284fa116ee73487b7cbb85c8266eaa0", "66c10bf1f11bc1b2d92204d8f8391d087f6de1c4", "814a4f680b9ba6baba23b93499f4b48af1a27678", "6c4b76232bb72897685d19b3d264c6ee3005bc2b", "1a858b96d2fdfeadf8c0f7126cbd55825223fb9d", "851e437dc09b4abd99bee73ade84b85437d61c4b", "e837dfa120e8ce3cd587bde7b0787ef43fa7832d", "e41c96b0d36c89f6dd1f21ccb80c7cc19af6a1cf", "efbd381493bb9636f489b965a2034d529cd56bcd", "e3aa232577bb427b1f3a34acbdef84bd85734042", "2b7c9fd2a94deaee3e7e56dc57bab0bd39d3683c", "81051b830a4f5606106765902a51ba281c9230f9", "9f0fe125af3cfbad99f1f2a6ada0daf61eef92b1"], "hybrid_score": 0.6081, "source": "semantic_scholar"}, {"title": "Fast-dLLM: Training-free Acceleration of Diffusion LLM by Enabling KV Cache and Parallel Decoding", "abstract": "Diffusion-based large language models (Diffusion LLMs) have shown promise for non-autoregressive text generation with parallel decoding capabilities. However, the practical inference speed of open-sourced Diffusion LLMs often lags behind autoregressive models due to the lack of Key-Value (KV) Cache and quality degradation when decoding multiple tokens simultaneously. To bridge this gap, we introduce a novel block-wise approximate KV Cache mechanism tailored for bidirectional diffusion models, enabling cache reuse with negligible performance drop. Additionally, we identify the root cause of generation quality degradation in parallel decoding as the disruption of token dependencies under the conditional independence assumption. To address this, we propose a confidence-aware parallel decoding strategy that selectively decodes tokens exceeding a confidence threshold, mitigating dependency violations and maintaining generation quality. Experimental results on LLaDA and Dream models across multiple LLM benchmarks demonstrate up to \\textbf{27.6$\\times$ throughput} improvement with minimal accuracy loss, closing the performance gap with autoregressive models and paving the way for practical deployment of Diffusion LLMs.", "year": 2025, "citation_count": 208, "paper_id": "5e9ffdd179df49be1129e32ae75fc89a2b68e676", "authors": ["Chengyue Wu", "Hao Zhang", "Shuchen Xue", "Zhijian Liu", "Shizhe Diao", "Ligeng Zhu", "Ping Luo", "Song Han", "Enze Xie"], "references": ["4a230ff814e6492da8c7dbd1b3d44842df1be989", "c6f896aa698b2d65160372bce057ea5f081904de", "83e3bb785cc56093b9cdac84e9bcacdb0f092d44", "7ffe883cf9ccfc8f212231d4f4b137d5096013e9", "1746d7b2c5c9a0351d51acbc636ac9c4b0eefd9a", "0d11a9674b68216b92e08cf7617a93fbd3fb91f4", "c3f18504c770549771b29c222db41b343dd1f36d", "b5c4c2f88fac2ca41854bbe8ae9740b9b03a0d3a", "385f0c77e72f95e7df74e98b4ff6b9ab19d52f93", "6a33c42eb058a04f3ae4ea08277d364522d0b0b8", "e85213146c7f50474c72116f33d36196c1c2e857", "a9ac6c4b165d8c4bf1a949d0931e7ca7a21fc55b", "f8d357d38bbcdd93889fe71762eb57842b2ab063", "7a73cc4bc0ce80661b5acbb51bbde30ea5167280", "1bed8c7541381b1f79027c240b64c9276573fc3c", "e8430dcca698b9f012615157c2a81b4983a10038", "9cbbb250a565228ba328038ee7944b89cff53e84", "a0a79dad89857a96f8f71b14238e5237cbfc4787", "3a22aad6c18a9559be3bbb197494b434b872a05a", "6d1433f3342fbee85ad1e2809e62734aec5c3853", "33433e9103b00aa0c42597cbfe13a429fbf5abdf", "a979742220a88b1d32e1fbe72c41e8ba3007053c", "3d8a2753649f3c493e2c237b0f4049858e958ae6", "e9b9a47cd81c66603c827f0f2bc4fba0d9ae77c4", "91b32fc0a23f0af53229fceaae9cce43a0406d2e", "2cd605106b88c85d7d8b865b1ef0f8c8293debf1", "add5f3f820b393e7ce5ed467814253824ecc484b", "395de0bd3837fdf4b4b5e5f04835bcc69c279481", "204e3073870fae3d05bcbc2f6a8e263d9b72e776", "2dcef55a07f8607a819c21fe84131ea269cc2e3c", "cfa3cb6b21410352a9aacb991cb0bd7e7d6a4ec0", "13bd881530d003403325cd2de57d8af42acd0d81"], "hybrid_score": 0.5691, "source": "semantic_scholar"}, {"title": "X-EcoMLA: Upcycling Pre-Trained Attention into MLA for Efficient and Extreme KV Compression", "abstract": "Multi-head latent attention (MLA) is designed to optimize KV cache memory through low-rank key-value joint compression. Rather than caching keys and values separately, MLA stores their compressed latent representations, reducing memory overhead while maintaining the performance. While MLA improves memory efficiency without compromising language model accuracy, its major limitation lies in its integration during the pre-training phase, requiring models to be trained from scratch. This raises a key question: can we use MLA's benefits fully or partially in models that have already been pre-trained with different attention mechanisms? In this paper, we propose X-EcoMLA to deploy post training distillation to enable the upcycling of Transformer-based attention into an efficient hybrid MLA variant through lightweight post-training adaptation, bypassing the need for extensive pre-training. We demonstrate that leveraging the dark knowledge of a well-trained model can enhance training accuracy and enable extreme KV cache compression in MLA without compromising model performance. The experimental results show that our proposed method can effectively compress the KV cache while preserving the performance on the benchmarks; specifically, for Llama3.2-1B-Instruct baseline, a 6.4x compression achieves the same average score by using only 3.6B training tokens and 70 GPU hours on AMD MI300, whereas a 10.6x compression have less than 0.1% average score drop with 7B training tokens and 140 GPU hours. The code for this work is available at https://github.com/AMD-AGI/AMD-Hybrid-Models.", "year": 2025, "citation_count": 2, "paper_id": "7aa4f9d94b6c81b8778cdfa7d1d6200f5edd4028", "authors": ["Guihong Li", "Mehdi Rezagholizadeh", "Mingyu Yang", "Vikram V. Appia", "E. Barsoum"], "references": ["4cbf0b9fd18a1850ce588244b073927c372a0d4f", "2eed1fad9bbf887d4395de40f20144c4fafefd7f", "d4fb143e6adbc86e0b200d1d131908db1ff24770", "230762f388c4b6f0e8af4554e2df5fd4248b522d", "bab5d963ba2d59fd74cc22f36bf14924025f1b5a", "40e8af970329135ec95057d73e239dab805ad128", "f06f347b28656a5453af8239a705b02a13c8e28a", "6c67e17fdd763f212e582f8de56b56dd5ed58832", "54f4ce7ff3390d9b8ffff90ff9be4f6e14046cd2", "022f386eb66fc5532dd6f439e7a356fd33ebb9a2", "dc3b69ed88f62d1a95c25f8cbd01d144b00312f2", "ca9f5b3bf0f54ad97513e6175b30497873670fed", "1d4c48335d841014d0145256c3c4e7f6c426b8fb", "53a803388e83ae89261624099d7be4287ace67cb", "05c1dc502ed51162580ccd320d5668d2fec94a7a", "7a54aad06171f59149aca5380863c62729c70b41", "1759d78e00b811b2b4b35b49e22f7ec11694f5ad", "f4a0c4154203808f362e4678f3741b3d317fdc82", "7bbc7595196a0606a07506c4fb1473e5e87f6082", "fdc53c2c10742464087c0525f77e32604827a21d", "2a38daf98d506477f8180806f503409d5036eaf4", "e586a4591ba0303b769f2c07cbddaf1899cb72e4", "5ae6fb6b5a3c7df515ff4a82ac9673bae6a8e200", "397e0e0d20f00d8fbfecd2fd36b14f13e2181d0e", "814a4f680b9ba6baba23b93499f4b48af1a27678", "90abbc2cf38462b954ae1b772fac9532e2ccd8b0", "925ad2897d1b5decbea320d07e99afa9110e09b2", "04f4e55e14150b7c48b0287ba77c7443df76ed45", "dc52b09089704ebd6f471177474bc29741c50023", "0c3c4c88c7b07596221ac640c7b7102686e3eae3", "8b0f27bb594b1eaaf493eaf1e2ee723a2b0a19ad", "1536e8958697c5364f68b2e2448905dbbeb3a0ca", "88bb0a28bb58d847183ec505dda89b63771bb495", "636a79420d838eabe4af7fb25d6437de45ab64e8", "5b6fdb2aea424d02c141da81d62b04d739d62b96", "92e121c6e114fe3cfb89370df03847c66a9b4e28"], "hybrid_score": 0.527, "source": "semantic_scholar"}]

data/cache/ee737556fd55e47471ca6ec835d5b80f.json

@@ -1 +0,0 @@
-[{"url": "https://arxiv.org/pdf/2510.00636", "snippet": "methods, significant research effort has focused on developing post-training compression techniques that \u00b7 can be applied to existing models without modification. Early approaches (Li et al., 2025; Oren et al., 2024) directly utilize attention scores to rank KV pairs by importance. However, these methods require \u00b7 access to the full attention matrix, making them incompatible with Flash Attention (Dao et al., 2022)", "title": "Expected Attention: KV Cache Compression by Estimating Attention", "inferred_year": 2025, "hybrid_score": 0.0, "source": "duckduckgo"}, {"url": "https://arxiv.org/pdf/2310.01801", "snippet": "Deepspeed (DS)(Aminabadi et al., 2022) by adding the KV cache sparsity operation.", "title": "Published as a conference paper at ICLR 2024 MODEL TELLS YOU WHAT TO DISCARD:", "inferred_year": 2022, "hybrid_score": 0.0, "source": "duckduckgo"}, {"url": "https://arxiv.org/pdf/2503.24000", "snippet": "Wang, G. Razorattention: Efficient kv cache compression \u00b7 through retrieval heads. arXiv preprint arXiv:2407.15891, ... S. Quest: Query-aware sparsity for efficient long-context \u00b7 llm inference. arXiv preprint arXiv:2406.10774, 2024b.", "title": "RETHINKING KEY-VALUE CACHE COMPRESSION TECHNIQUES FOR LARGE", "inferred_year": null, "hybrid_score": 0.0, "source": "duckduckgo"}]