diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..0ece512aabdd797aea05be34232585e6215d02e6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,35 +1,5 @@ -*.7z filter=lfs diff=lfs merge=lfs -text -*.arrow filter=lfs diff=lfs merge=lfs -text -*.bin filter=lfs diff=lfs merge=lfs -text -*.bz2 filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.ftz filter=lfs diff=lfs merge=lfs -text -*.gz filter=lfs diff=lfs merge=lfs -text -*.h5 filter=lfs diff=lfs merge=lfs -text -*.joblib filter=lfs diff=lfs merge=lfs -text -*.lfs.* filter=lfs diff=lfs merge=lfs -text -*.mlmodel filter=lfs diff=lfs merge=lfs -text -*.model filter=lfs diff=lfs merge=lfs -text -*.msgpack filter=lfs diff=lfs merge=lfs -text -*.npy filter=lfs diff=lfs merge=lfs -text -*.npz filter=lfs diff=lfs merge=lfs -text -*.onnx filter=lfs diff=lfs merge=lfs -text -*.ot filter=lfs diff=lfs merge=lfs -text -*.parquet filter=lfs diff=lfs merge=lfs -text -*.pb filter=lfs diff=lfs merge=lfs -text -*.pickle filter=lfs diff=lfs merge=lfs -text -*.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text -*.pth filter=lfs diff=lfs merge=lfs -text -*.rar filter=lfs diff=lfs merge=lfs -text *.safetensors filter=lfs diff=lfs merge=lfs -text -saved_model/**/* filter=lfs diff=lfs merge=lfs -text -*.tar.* filter=lfs diff=lfs merge=lfs -text -*.tar filter=lfs diff=lfs merge=lfs -text -*.tflite filter=lfs diff=lfs merge=lfs -text -*.tgz filter=lfs diff=lfs merge=lfs -text -*.wasm filter=lfs diff=lfs merge=lfs -text -*.xz filter=lfs diff=lfs merge=lfs -text -*.zip filter=lfs diff=lfs merge=lfs -text -*.zst filter=lfs diff=lfs merge=lfs -text -*tfevents* filter=lfs diff=lfs merge=lfs -text +assets/figures/ksa_v2_compression.png filter=lfs diff=lfs merge=lfs -text +assets/figures/ksa_v2_overview.png filter=lfs diff=lfs merge=lfs -text +assets/figures/ksa_v2_results.png filter=lfs diff=lfs merge=lfs -text +assets/figures/ksa_v2_sparse.png filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e35926ee40654b54b59b95165056414136b4a29d --- /dev/null +++ b/README.md @@ -0,0 +1,227 @@ +--- +license: apache-2.0 +language: +- en +- zh +library_name: transformers +pipeline_tag: text-generation +tags: +- ksa +- ksa-v2 +- custom_code +- long-context +--- + + + +
+ +## 📖 Introduction + +**Kwai Summary Attention (KSA)** is our attention architecture for efficient +long-context modeling. Learnable Summary tokens at fixed chunk boundaries +preserve a compressed representation of distant history. + +**KSA V2** gives these Summaries two complementary roles: + +- **Compressed KSA (C-KSA)** aggregates Summary KV into compact Meta KV for + more efficient long-range memory. +- **Sparse KSA (S-KSA)** uses Summaries to build block-level routing keys. + A lightweight Indexer selects relevant blocks, whose original KV is read + directly by the main attention. + +We retain Hybrid KSA's 3:1 layer layout, replacing KSA layers with C-KSA and +Full layers with S-KSA. + +
From Hybrid KSA to KSA V2: shared Summaries, layer-local KV compression and sparse original-token access.
+ +This base model builds on our **1.9B Hybrid KSA backbone trained from scratch**. It is not an instruction-tuned model. + +## 🔥 News + +- **2026-10-13** — [KSA V2 technical report](https://arxiv.org/abs/2610.18654) released. +- **2026-05-08** — [KSA-4B-base](https://huggingface.co/OpenOneRec/KSA-4B-base) weights released. +- **2026-04-28** — [KSA technical report](https://arxiv.org/abs/2604.24432) and training code released. + +## ✨ Method + +**C-KSA: Summary → Meta KV.** Learnable dimension-wise pooling compresses groups +of Summary KV within each attention layer. Summary tokens continue through +the backbone at their original chunk granularity; stronger KV compression does +not require a new large-chunk backbone. Each query attends to remote Meta, +boundary Summary and local text KV. Meta entries remain local to the layer +that constructs them; they are not propagated as new tokens. + +
C-KSA hierarchical compression, illustrated with chunk size 4 and two Summaries per Meta entry.
+ +**S-KSA: Summary → block index.** A Summary query aggregates each text block +into a representation for low-dimensional routing. Main attention reads the +selected blocks' original KV together with sinks and local context. A +blockwise-max dense-attention teacher supervises the Indexer. + +
S-KSA block routing, illustrated with chunk size 4 and block size 2. Released model configurations are listed below.
+ +**Three adaptation stages.** Compression-module warmup → full-parameter C-KSA +CPT → S-KSA aggregation and Indexer warmup. The final stage freezes inherited +parameters; its checkpoint is exported directly for sparse inference. + +## 🤖 Model configuration + +This directory contains **KSA-v2-1.9B**. + +| Configuration | KSA-v2-1.9B | KSA-v2-4B | +| :--- | :--- | :--- | +| Number of layers | 24 | 36 | +| Hidden size | 2048 | 2560 | +| Intermediate size | 6144 | 9728 | +| Attention heads (Q / KV) | 16 / 16 (MHA) | 32 / 8 (GQA) | +| Head dimension | 128 | 128 | +| Hybrid ratio (C-KSA : S-KSA) | 3 : 1 | 3 : 1 | +| Maximum context (text tokens) | 131072 | 131072 | +| Summary chunk size | 8 | 8 | +| C-KSA sliding chunk number | 128 | 128 | +| C-KSA group size | 128 | 32 | +| S-KSA block size | 4 | 4 | +| S-KSA remote token budget | 4096 | 4096 | +| Sink tokens | 8 | 8 | +| S-KSA local window (text tokens) | 1024 | 1024 | +| Indexer heads × head dimension | 8 × 128 | 8 × 128 | +| Indexer query rank | 512 | 512 | +| Indexer RoPE dimension | 64 | 64 | +| Indexer Q/K RMSNorm | False | True | +| Teacher pooling | max | max | +| Tied embeddings | False | True | + +The remote budget includes eight sink tokens; the local window is additional. + +Weights are stored in FP32 and loaded in BF16. + +## 📈 Results + +
1.9B results: long-context and general capabilities (left), theoretical per-token inference FLOPs and KV cache size (right).
+ +We evaluate KSA V2 on two Hybrid KSA backbones: a 1.9B model trained from +scratch and a 4B model continually pretrained from Qwen3-4B. Both undergo +three-stage adaptation; original baseline results are reused from KSA V1. + +### 1.9B: Long-context retrieval + +C-KSA uses group128 and S-KSA uses block4. Hybrid KSA is the parent model before adaptation. + +| Model | 4K | 8K | 16K | 32K | 64K | 128K | Mean | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Full | 76.08 | 72.85 | 73.24 | 69.06 | 65.32 | 48.75 | 67.55 | +| Hybrid SWA | 74.54 | 71.69 | 69.54 | 67.86 | 63.03 | 56.64 | 67.22 | +| Hybrid SCA | 77.72 | 75.22 | 72.55 | 67.74 | 63.54 | 58.01 | 69.13 | +| Hybrid GDN | 79.83 | 76.01 | 74.04 | 70.41 | 69.39 | 59.87 | 71.59 | +| Pure KSA | 70.44 | 65.91 | 66.74 | 62.54 | 57.13 | 39.29 | 60.34 | +| Hybrid KSA | 80.65 | 73.35 | 74.07 | 72.30 | 69.95 | 65.35 | 72.61 | +| C-KSA + Full | 81.52 | 72.24 | 74.44 | 71.93 | 69.61 | 66.31 | 72.68 | +| **C-KSA + S-KSA** | 81.48 | 72.42 | 75.67 | 70.78 | 68.99 | 65.01 | 72.39 | + +### 4B: Scaling to a GQA backbone + +C-KSA uses group32 and S-KSA uses block4, preserving the backbone's GQA structure. + +| Model | 4K | 8K | 16K | 32K | 64K | 128K | Mean | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Full | 92.88 | 91.38 | 89.12 | 84.74 | 78.16 | 65.86 | 83.69 | +| Hybrid SWA | 91.30 | 88.03 | 82.87 | 78.94 | 73.88 | 66.27 | 80.22 | +| Hybrid SCA | 86.02 | 84.28 | 80.67 | 76.89 | 68.88 | 60.94 | 76.28 | +| Hybrid Linear | 86.39 | 83.86 | 78.06 | 76.48 | 73.50 | 67.98 | 77.71 | +| Pure KSA | 91.55 | 86.78 | 84.78 | 80.30 | 76.09 | 66.81 | 81.05 | +| Hybrid KSA | 92.97 | 90.53 | 88.86 | 86.65 | 76.04 | 71.67 | 84.45 | +| Hybrid KSA + CPT | 92.22 | 89.85 | 87.20 | 83.09 | 74.74 | 72.03 | 83.19 | +| C-KSA + Full | 92.77 | 88.12 | 86.27 | 82.68 | 78.58 | 73.92 | 83.72 | +| **C-KSA + S-KSA** | 93.38 | 87.78 | 85.45 | 83.60 | 80.28 | 74.41 | 84.15 | + +Hybrid KSA + CPT is a continuation control with no architectural changes. + +See the technical report for further ablations and inference efficiency. + +## Quick start + +Linux, Python 3.12 and H100/H800. Download the **entire model directory**, +including `kernel/`, and run from it: + +```bash +python -m pip install torch==2.6.0 --index-url https://download.pytorch.org/whl/cu126 +python -m pip install -r requirements.txt + +CUDA_VISIBLE_DEVICES=0 python generate.py \ + --prompt "The main advantage of long-context language models is" \ + --max-new-tokens 128 +``` + +Omit `--prompt` for interactive completion. The optional `--apply-chat-template` +only applies the tokenizer template; it does not add instruction tuning. + +Transformers 5.0.0 and the bundled `ksa_attn` / FlashAttention CuTe sources are +used. No training checkout, kernel build or FlashAttention 2 CUDA extension +is required. + +### Transformers usage + +Set the bundled kernel paths before importing Transformers: + +```bash +export PYTHONPATH="$PWD/kernel:$PWD/kernel/deps${PYTHONPATH:+:$PYTHONPATH}" +``` + +```python +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained(".", local_files_only=True) +model = AutoModelForCausalLM.from_pretrained( + ".", + trust_remote_code=True, + local_files_only=True, + dtype=torch.bfloat16, + device_map={"": "cuda:0"}, + attn_implementation="sdpa", +).eval() + +inputs = tokenizer("The main advantage of long-context models is", return_tensors="pt") +inputs = inputs.to(model.device) +with torch.inference_mode(): + output = model.generate( + **inputs, + max_new_tokens=128, + do_sample=False, + pad_token_id=tokenizer.eos_token_id, + ) +print(tokenizer.decode(output[0, inputs.input_ids.shape[1]:], skip_special_tokens=True)) +``` + +## License and citation + +Weights and our runtime: [Apache-2.0](LICENSE). Qwen-derived files retain their +copyright notices. Bundled FlashAttention CuTe retains its +[BSD-3-Clause license](kernel/deps/flash_attn/LICENSE) and +[AUTHORS](kernel/deps/flash_attn/AUTHORS); its initializer exposes CuTe without +the separate FlashAttention 2 extension. + +```bibtex +@techreport{kwai2026ksav2, + title = {Kwai Summary Attention V2 Technical Report}, + author = {OneRec Team}, + year = {2026}, + institution = {Kuaishou Technology}, + url = {https://arxiv.org/abs/2610.18654} +} +``` diff --git a/README_zh.md b/README_zh.md new file mode 100644 index 0000000000000000000000000000000000000000..a6b4d9722a76a6f2f0053c60be153e8645fd58bd --- /dev/null +++ b/README_zh.md @@ -0,0 +1,212 @@ +--- +license: apache-2.0 +language: +- en +- zh +library_name: transformers +pipeline_tag: text-generation +tags: +- ksa +- ksa-v2 +- custom_code +- long-context +--- + + + + + +## 📖 简介 + +**Kwai Summary Attention(KSA)** 是我们面向长上下文建模的高效注意力机制。 +KSA 在固定 chunk 边界插入可学习的 Summary Token,以压缩后的状态保留对远端历史的访问。 + +**KSA V2** 进一步让 Summary 承担两个角色: + +- **Compressed KSA(C-KSA)**:将多份 Summary KV 聚合为 Meta KV,进一步降低长程记忆的存储开销。 +- **Sparse KSA(S-KSA)**:以 Summary 聚合原文块的选路表示,用轻量 Indexer 选择相关块,直接读取这些块的原始 KV。 + +我们保留 Hybrid KSA 的 3:1 层间布局,将 KSA 层替换为 C-KSA、Full 层替换为 S-KSA, +分别优化长程记忆和原文读取。 + +
从 Hybrid KSA 到 KSA V2:复用 Summary,在注意力层内压缩 KV,并稀疏读取原文。
+ +本模型基于**从零训练的 1.9B Hybrid KSA 主干**进行适配,是未经过指令微调的基础模型。 + +## 🔥 新闻 + +- **2026-10-13** —— [KSA V2 技术报告](https://arxiv.org/abs/2610.18654)发布。 +- **2026-05-08** —— [KSA-4B-base](https://huggingface.co/OpenOneRec/KSA-4B-base) 权重发布。 +- **2026-04-28** —— [KSA 技术报告](https://arxiv.org/abs/2604.24432)与训练代码发布。 + +## ✨ 方法 + +**C-KSA:Summary → Meta KV。** 在注意力层内进行可学习的逐维聚合。 +每组 Summary 生成一份更紧凑的 Meta KV;Summary Token 本身仍按原 chunk 粒度传递, +不必为增加压缩率重新训练更大 chunk 的主干。查询同时访问远程 Meta、边界 Summary +和局部原文 KV;Meta 仅在当前层内使用,不作为新 token 传递。 + +
C-KSA 两级压缩示意,图中 chunk 大小为 4,每 2 个 Summary 聚合为一个 Meta。
+ +**S-KSA:Summary → 块级索引。** 以 Summary 为 query 聚合原文块,再映射到低维索引空间。 +主注意力读取选中块的原始 KV,并保留 sink 与局部窗口。Indexer 用按块取 max 的 +dense attention teacher 监督训练。 + +
S-KSA 块级选路示意,图中 chunk 大小为 4、block 大小为 2;发布模型的实际配置见下表。
+ +**三个适配阶段。** 压缩模块 warmup → C-KSA 全参数 CPT → S-KSA 聚合模块与 Indexer warmup。 +最后一阶段冻结继承参数,完成后直接导出用于稀疏推理。 + +## 🤖 模型配置 + +当前目录为 **KSA-v2-1.9B**。 + +| 配置项 | KSA-v2-1.9B | KSA-v2-4B | +| :--- | :--- | :--- | +| 层数 | 24 | 36 | +| Hidden size | 2048 | 2560 | +| Intermediate size | 6144 | 9728 | +| Attention heads(Q / KV) | 16 / 16(MHA) | 32 / 8(GQA) | +| Head dimension | 128 | 128 | +| Hybrid 比例(C-KSA : S-KSA) | 3 : 1 | 3 : 1 | +| 最大上下文(原文 token) | 131072 | 131072 | +| Summary chunk size | 8 | 8 | +| C-KSA sliding chunk number | 128 | 128 | +| C-KSA group size | 128 | 32 | +| S-KSA block size | 4 | 4 | +| S-KSA 远程 token 预算 | 4096 | 4096 | +| Sink tokens | 8 | 8 | +| S-KSA 局部窗口(原文 token) | 1024 | 1024 | +| Indexer heads × head dimension | 8 × 128 | 8 × 128 | +| Indexer query rank | 512 | 512 | +| Indexer RoPE dimension | 64 | 64 | +| Indexer Q/K RMSNorm | False | True | +| Teacher 聚合 | max | max | +| Tied embeddings | False | True | + +远程预算包含 8 个 sink token,局部窗口另计。 + +权重以 FP32 保存,推理时以 BF16 加载。 + +## 📈 实验结果 + +
1.9B 主结果:长上下文与通用能力(左),单 token 推理 FLOPs 与 KV cache 大小的理论值(右)。
+ +我们在两种 Hybrid KSA 主干上验证 KSA V2:从零训练的 1.9B,以及基于 Qwen3-4B +继续预训练的 4B。两者均经过三阶段适配,原始 baseline 结果沿用 KSA V1。 + +### 1.9B:长上下文检索 + +C-KSA 使用 group128,S-KSA 使用 block4;Hybrid KSA 是适配前的父模型。 + +| 模型 | 4K | 8K | 16K | 32K | 64K | 128K | Mean | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Full | 76.08 | 72.85 | 73.24 | 69.06 | 65.32 | 48.75 | 67.55 | +| Hybrid SWA | 74.54 | 71.69 | 69.54 | 67.86 | 63.03 | 56.64 | 67.22 | +| Hybrid SCA | 77.72 | 75.22 | 72.55 | 67.74 | 63.54 | 58.01 | 69.13 | +| Hybrid GDN | 79.83 | 76.01 | 74.04 | 70.41 | 69.39 | 59.87 | 71.59 | +| Pure KSA | 70.44 | 65.91 | 66.74 | 62.54 | 57.13 | 39.29 | 60.34 | +| Hybrid KSA | 80.65 | 73.35 | 74.07 | 72.30 | 69.95 | 65.35 | 72.61 | +| C-KSA + Full | 81.52 | 72.24 | 74.44 | 71.93 | 69.61 | 66.31 | 72.68 | +| **C-KSA + S-KSA** | 81.48 | 72.42 | 75.67 | 70.78 | 68.99 | 65.01 | 72.39 | + +### 4B:扩展到 GQA 主干 + +C-KSA 使用 group32,S-KSA 使用 block4,保留主干的 GQA 结构。 + +| 模型 | 4K | 8K | 16K | 32K | 64K | 128K | Mean | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Full | 92.88 | 91.38 | 89.12 | 84.74 | 78.16 | 65.86 | 83.69 | +| Hybrid SWA | 91.30 | 88.03 | 82.87 | 78.94 | 73.88 | 66.27 | 80.22 | +| Hybrid SCA | 86.02 | 84.28 | 80.67 | 76.89 | 68.88 | 60.94 | 76.28 | +| Hybrid Linear | 86.39 | 83.86 | 78.06 | 76.48 | 73.50 | 67.98 | 77.71 | +| Pure KSA | 91.55 | 86.78 | 84.78 | 80.30 | 76.09 | 66.81 | 81.05 | +| Hybrid KSA | 92.97 | 90.53 | 88.86 | 86.65 | 76.04 | 71.67 | 84.45 | +| Hybrid KSA + CPT | 92.22 | 89.85 | 87.20 | 83.09 | 74.74 | 72.03 | 83.19 | +| C-KSA + Full | 92.77 | 88.12 | 86.27 | 82.68 | 78.58 | 73.92 | 83.72 | +| **C-KSA + S-KSA** | 93.38 | 87.78 | 85.45 | 83.60 | 80.28 | 74.41 | 84.15 | + +Hybrid KSA + CPT 是不改模型结构的续训对照。 + +更多消融实验和推理效率分析见技术报告。 + +## 快速开始 + +Linux、Python 3.12、H100/H800。下载**完整模型目录**(包含 `kernel/`),并在该目录下执行: + +```bash +python -m pip install torch==2.6.0 --index-url https://download.pytorch.org/whl/cu126 +python -m pip install -r requirements.txt + +CUDA_VISIBLE_DEVICES=0 python generate.py \ + --prompt "The main advantage of long-context language models is" \ + --max-new-tokens 128 +``` + +省略 `--prompt` 可进行交互式文本续写。`--apply-chat-template` 只应用 tokenizer 中的对话模板,不代表模型经过指令微调。 + +使用 Transformers 5.0.0 和随包提供的 `ksa_attn`/FlashAttention CuTe 源码,无需训练仓库、单独编译内核或安装 FlashAttention 2 CUDA 扩展。 + +### Transformers 用法 + +导入 Transformers 前设置随包内核路径: + +```bash +export PYTHONPATH="$PWD/kernel:$PWD/kernel/deps${PYTHONPATH:+:$PYTHONPATH}" +``` + +```python +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained(".", local_files_only=True) +model = AutoModelForCausalLM.from_pretrained( + ".", + trust_remote_code=True, + local_files_only=True, + dtype=torch.bfloat16, + device_map={"": "cuda:0"}, + attn_implementation="sdpa", +).eval() + +inputs = tokenizer("The main advantage of long-context models is", return_tensors="pt") +inputs = inputs.to(model.device) +with torch.inference_mode(): + output = model.generate( + **inputs, + max_new_tokens=128, + do_sample=False, + pad_token_id=tokenizer.eos_token_id, + ) +print(tokenizer.decode(output[0, inputs.input_ids.shape[1]:], skip_special_tokens=True)) +``` + +## 许可证与引用 + +模型权重及我们的推理代码采用 [Apache-2.0](LICENSE)。基于 Qwen 的文件保留原版权声明。 +随包的 FlashAttention CuTe 保留其 [BSD-3-Clause 许可证](kernel/deps/flash_attn/LICENSE) +与 [AUTHORS](kernel/deps/flash_attn/AUTHORS);入口仅加载 CuTe,不依赖单独的 FlashAttention 2 扩展。 + +```bibtex +@techreport{kwai2026ksav2, + title = {Kwai Summary Attention V2 Technical Report}, + author = {OneRec Team}, + year = {2026}, + institution = {Kuaishou Technology}, + url = {https://arxiv.org/abs/2610.18654} +} +``` diff --git a/assets/figures/ksa_v2_compression.png b/assets/figures/ksa_v2_compression.png new file mode 100644 index 0000000000000000000000000000000000000000..2b672846fb2fb33ed8363826717bf8da5b6a3b6e --- /dev/null +++ b/assets/figures/ksa_v2_compression.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff04e1cb08f2cab60f3095980042806c5fbcd9b6a77b3934fa0d379595469085 +size 104606 diff --git a/assets/figures/ksa_v2_overview.png b/assets/figures/ksa_v2_overview.png new file mode 100644 index 0000000000000000000000000000000000000000..3383af800c765ced92d851b81b1a18f23104134d --- /dev/null +++ b/assets/figures/ksa_v2_overview.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7508b6a895f43d88fd8bc45d55e81bcb05ad7e02e19a9b69bdc50b746cf16eef +size 132820 diff --git a/assets/figures/ksa_v2_results.png b/assets/figures/ksa_v2_results.png new file mode 100644 index 0000000000000000000000000000000000000000..e83e1a8714417b8ce2e5f67c264e6f68ba4767e8 --- /dev/null +++ b/assets/figures/ksa_v2_results.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d9602c8998e8c391dc81d5b9ab60ec20d6424267e3ebdfa76921bbc1372468e +size 151904 diff --git a/assets/figures/ksa_v2_sparse.png b/assets/figures/ksa_v2_sparse.png new file mode 100644 index 0000000000000000000000000000000000000000..877212506f1aed13f613b0a09640cbc18d2a798f --- /dev/null +++ b/assets/figures/ksa_v2_sparse.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c0059cf52966b500b865700c0866174cfebbb7024559aa8b129ec6e00a0724d +size 127212 diff --git a/cache_ring_buffer.py b/cache_ring_buffer.py new file mode 100644 index 0000000000000000000000000000000000000000..07ad42ff70e73e1f72a5d3e37126cbd3d479bd15 --- /dev/null +++ b/cache_ring_buffer.py @@ -0,0 +1,892 @@ +# coding=utf-8 +# Copyright 2025 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Any, Callable, Optional, Union +import torch +from transformers.models.qwen3.configuration_qwen3 import Qwen3Config +from .runtime_config import _get_shared_batch_mask, _parse_config_pattern +from .layer_norm_rope import rotate_half + + +class Qwen3RingBufferCache: + """ + Ring buffer KV cache with summary support. + + Two strategies based on per-layer sliding_chunk_num: + - Large window layers (is_large_window=True): append-only buffer storing only text KV. + Summary KV is NOT stored since text tokens attend to all text KV directly. + - Small window layers (is_large_window=False): + Three buffers: + 1. key_cache: [ring(ws) | old_summaries(growing) | chunk_mirror(≤C)] + → attention input, steady state is a single contiguous slice + 2. new_summary_buf: ring buffer of size scn, stores summaries whose text + is still in the window (not needed for attention) + 3. chunk_buf: size C, holds current chunk's text KV + + RoPE position information is baked into KV, so physical order doesn't matter. + """ + + is_compileable = False + _SUMMARY_INIT_CAP = 512 + _APPEND_HEADROOM = 1024 + + def __init__(self, config: Qwen3Config, sliding_chunk_nums: list[int]): + super().__init__() + self.summary_chunk_size = getattr(config, "summary_chunk_size", 0) + self.summary_token_num = getattr(config, "summary_token_num", 0) + self.num_hidden_layers = config.num_hidden_layers + + self.sliding_chunk_nums = sliding_chunk_nums + large_window_threshold = min(sliding_chunk_nums) * self.summary_chunk_size + self.is_large_window = [sv * self.summary_chunk_size > large_window_threshold for sv in sliding_chunk_nums] + self.window_sizes = [sv * self.summary_chunk_size for sv in sliding_chunk_nums] + pool_mode = getattr(config, "summary_meta_pool_mode", "none") or "none" + pool_groups = _parse_config_pattern( + getattr(config, "summary_meta_pool_group_size", 0) or 0 + ) + if not isinstance(pool_groups, list): + pool_groups = [int(pool_groups)] * config.num_hidden_layers + self.meta_pool_group_sizes = [int(v) for v in pool_groups] + self.meta_pool_enabled = [ + pool_mode in ("mean", "scalar", "dimension_wise") + and group_size > 1 + for group_size in self.meta_pool_group_sizes + ] + self.meta_pool_mode = pool_mode + + self.key_cache = [None for _ in range(config.num_hidden_layers)] + self.value_cache = [None for _ in range(config.num_hidden_layers)] + # Full-layer Lightning Indexer keys are not main-attention K/V. The + # summary-indexer variant stores one key per raw chunk; token-indexer + # mode stores one key per raw text token. + self._sksa_index_keys = [None for _ in range(config.num_hidden_layers)] + self._sparse_gather_storage = {} + # Independent chunk-Q/K mode needs the auxiliary K states for the + # currently accumulating raw chunk. They are separate from the + # backbone attention cache and are discarded once its summary token is + # emitted. + self._sksa_chunk_k = [None for _ in range(config.num_hidden_layers)] + self._sksa_chunk_v = [None for _ in range(config.num_hidden_layers)] + + # Large window: append-only + self._text_len = [0] * config.num_hidden_layers + self._capacity = [0] * config.num_hidden_layers + + # Small window: ring buffer + summary + self._window_write_ptr = [0] * config.num_hidden_layers + self._n_valid_window = [0] * config.num_hidden_layers + self._old_summary_len = [0] * config.num_hidden_layers # old summaries in key_cache + self._old_summary_cap = [0] * config.num_hidden_layers + self._old_summary_total = [0] * config.num_hidden_layers + + # Hierarchical old-summary state. Complete groups are stored as post-RoPE + # meta KV; fewer than group_size evicted summaries remain residual KV. + self._meta_key_cache = [None for _ in range(config.num_hidden_layers)] + self._meta_value_cache = [None for _ in range(config.num_hidden_layers)] + self._meta_len = [0] * config.num_hidden_layers + self._meta_cap = [0] * config.num_hidden_layers + self._residual_pre_key_buf = [None for _ in range(config.num_hidden_layers)] + self._residual_score_buf = [None for _ in range(config.num_hidden_layers)] + self._residual_cos_buf = [None for _ in range(config.num_hidden_layers)] + self._residual_sin_buf = [None for _ in range(config.num_hidden_layers)] + self._residual_position_buf = [None for _ in range(config.num_hidden_layers)] + + # New summary ring buffer (small window only): summaries whose text is still in window + self._new_sum_key_buf = [None for _ in range(config.num_hidden_layers)] + self._new_sum_value_buf = [None for _ in range(config.num_hidden_layers)] + self._new_sum_pre_key_buf = [None for _ in range(config.num_hidden_layers)] + self._new_sum_score_buf = [None for _ in range(config.num_hidden_layers)] + self._new_sum_cos_buf = [None for _ in range(config.num_hidden_layers)] + self._new_sum_sin_buf = [None for _ in range(config.num_hidden_layers)] + self._new_sum_position_buf = [None for _ in range(config.num_hidden_layers)] + self._new_sum_len = [0] * config.num_hidden_layers # how many filled (≤ scn) + self._new_sum_write_ptr = [0] * config.num_hidden_layers # ring write pointer + + # Mirror region length (small window only): partial chunk text KV count in key_cache + self._chunk_buf_len = [0] * config.num_hidden_layers + + # Common + self.cur_chunk_sizes = [0] * config.num_hidden_layers + self.true_tokens = [0] * config.num_hidden_layers + self._total_chunks = [0] * config.num_hidden_layers # completed chunks count + self._reorganized = [False] * config.num_hidden_layers + self._prefill_text_positions = None + self._prefill_summary_positions = None + + + + + def __len__(self): + return self.num_hidden_layers + + def get_seq_length(self, layer_idx: Optional[int] = 0) -> int: + """Returns nonzero only when this layer's cache is in post-reorganize state.""" + if layer_idx >= self.num_hidden_layers: + return 0 + if not self._reorganized[layer_idx]: + return 0 + if self.is_large_window[layer_idx]: + return self._text_len[layer_idx] + else: + return (self._n_valid_window[layer_idx] + self._chunk_buf_len[layer_idx] + + self._old_summary_len[layer_idx] + self._new_sum_len[layer_idx]) + + def get_cur_chunk_size(self, layer_idx: Optional[int] = None) -> int: + if layer_idx is None: + layer_idx = self.num_hidden_layers - 1 + return self.cur_chunk_sizes[layer_idx] + + def get_true_token_num(self, layer_idx: Optional[int] = None) -> int: + if layer_idx is None: + layer_idx = self.num_hidden_layers - 1 + return self.true_tokens[layer_idx] + + def set_sksa_index_keys( + self, layer_idx: int, index_keys: torch.Tensor + ) -> None: + if index_keys.ndim != 3: + raise ValueError("S-KSA index keys must use [B,M,D] layout") + self._sksa_index_keys[layer_idx] = index_keys.contiguous() + + def append_sksa_index_keys( + self, layer_idx: int, index_keys: torch.Tensor + ) -> None: + if index_keys.ndim != 3: + raise ValueError("S-KSA index keys must use [B,M,D] layout") + current = self._sksa_index_keys[layer_idx] + if current is None: + self._sksa_index_keys[layer_idx] = index_keys.contiguous() + else: + if current.shape[0] != index_keys.shape[0] or current.shape[2] != index_keys.shape[2]: + raise ValueError("S-KSA index key cache geometry changed during decode") + self._sksa_index_keys[layer_idx] = torch.cat( + (current, index_keys), dim=1 + ) + + def get_sksa_index_keys(self, layer_idx: int) -> torch.Tensor: + index_keys = self._sksa_index_keys[layer_idx] + if index_keys is None: + raise RuntimeError(f"S-KSA index key cache for layer {layer_idx} is uninitialized") + return index_keys + + def set_sksa_chunk_k_from_prefill( + self, + layer_idx: int, + chunk_key_states: Optional[torch.Tensor], + summary_mask: torch.Tensor, + ) -> None: + """Retain auxiliary K for the incomplete raw chunk after prefill.""" + if chunk_key_states is None: + self._sksa_chunk_k[layer_idx] = None + return + if chunk_key_states.ndim != 4: + raise ValueError("S-KSA chunk K must use [B,H,S,D] layout") + shared = _get_shared_batch_mask(summary_mask, "summary_mask") + positions = self._prefill_text_positions + text_key = (chunk_key_states.index_select(2, positions) if positions is not None + else chunk_key_states[:, :, ~shared, :]) + partial = text_key.shape[2] % self.summary_chunk_size + if partial: + self._sksa_chunk_k[layer_idx] = text_key[:, :, -partial:, :].contiguous() + else: + self._sksa_chunk_k[layer_idx] = None + + def append_sksa_chunk_k( + self, layer_idx: int, chunk_key_states: Optional[torch.Tensor] + ) -> None: + """Append one decoded raw token's independent auxiliary K state.""" + if chunk_key_states is None: + return + if chunk_key_states.ndim != 4 or chunk_key_states.shape[2] != 1: + raise ValueError("decoded S-KSA chunk K must use [B,H,1,D] layout") + current = self._sksa_chunk_k[layer_idx] + self._sksa_chunk_k[layer_idx] = ( + chunk_key_states.contiguous() + if current is None + else torch.cat((current, chunk_key_states), dim=2).contiguous() + ) + + def get_current_sksa_chunk_k(self, layer_idx: int) -> Optional[torch.Tensor]: + return self._sksa_chunk_k[layer_idx] + + def clear_sksa_chunk_k(self, layer_idx: int) -> None: + self._sksa_chunk_k[layer_idx] = None + + def set_sksa_chunk_v_from_prefill( + self, + layer_idx: int, + chunk_value_states: Optional[torch.Tensor], + summary_mask: torch.Tensor, + ) -> None: + """Retain auxiliary V for the incomplete raw chunk after prefill.""" + if chunk_value_states is None: + self._sksa_chunk_v[layer_idx] = None + return + if chunk_value_states.ndim != 4: + raise ValueError("S-KSA chunk V must use [B,H,S,D] layout") + shared = _get_shared_batch_mask(summary_mask, "summary_mask") + positions = self._prefill_text_positions + text_value = (chunk_value_states.index_select(2, positions) if positions is not None + else chunk_value_states[:, :, ~shared, :]) + partial = text_value.shape[2] % self.summary_chunk_size + if partial: + self._sksa_chunk_v[layer_idx] = text_value[:, :, -partial:, :].contiguous() + else: + self._sksa_chunk_v[layer_idx] = None + + def append_sksa_chunk_v( + self, layer_idx: int, chunk_value_states: Optional[torch.Tensor] + ) -> None: + """Append one decoded raw token's independent auxiliary V state.""" + if chunk_value_states is None: + return + if chunk_value_states.ndim != 4 or chunk_value_states.shape[2] != 1: + raise ValueError("decoded S-KSA chunk V must use [B,H,1,D] layout") + current = self._sksa_chunk_v[layer_idx] + self._sksa_chunk_v[layer_idx] = ( + chunk_value_states.contiguous() + if current is None + else torch.cat((current, chunk_value_states), dim=2).contiguous() + ) + + def get_current_sksa_chunk_v(self, layer_idx: int) -> Optional[torch.Tensor]: + return self._sksa_chunk_v[layer_idx] + + def clear_sksa_chunk_v(self, layer_idx: int) -> None: + self._sksa_chunk_v[layer_idx] = None + + # ── Prefill: per-layer one-shot init into ring buffer layout ── + + def update( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + cache_kwargs: Optional[dict[str, Any]] = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Deprecated for prefill — use init_layer_from_prefill instead. + + Kept only to surface accidental calls; the prefill path now skips this + intermediate full-length buffer entirely. + """ + raise NotImplementedError( + "Qwen3RingBufferCache.update() is no longer used; prefill goes through " + "init_layer_from_prefill, decode goes through update_text/update_summary." + ) + + def prepare_prefill(self, text_positions, summary_positions): + self._prefill_text_positions = text_positions + self._prefill_summary_positions = summary_positions + + def init_layer_from_prefill( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + summary_mask: torch.Tensor, + layer_idx: int, + *, + pre_rope_key_states: Optional[torch.Tensor] = None, + summary_scores: Optional[torch.Tensor] = None, + summary_cos: Optional[torch.Tensor] = None, + summary_sin: Optional[torch.Tensor] = None, + summary_position_ids: Optional[torch.Tensor] = None, + meta_key_states: Optional[torch.Tensor] = None, + meta_value_states: Optional[torch.Tensor] = None, + sksa_chunk_key_states: Optional[torch.Tensor] = None, + ): + """Allocate this layer's cache directly into the post-reorganize ring buffer + layout from prefill K/V (already projected + RoPE applied), with NO intermediate + full-length buffer. Replaces the old update() + reorganize_after_prefill() pair + for the prefill path. + + key_states / value_states: [bsz, kv_heads, prefill_len_expanded, head_dim] + summary_mask: [bsz, prefill_len_expanded] (bool, batch-uniform) + """ + assert not self._reorganized[layer_idx], f"layer {layer_idx} already reorganized" + + prefill_k = key_states + prefill_v = value_states + bsz, heads, prefill_len, head_dim = prefill_k.shape + device, dtype = prefill_k.device, prefill_k.dtype + + sm = _get_shared_batch_mask(summary_mask, "summary_mask") + text_mask = ~sm + text_positions = self._prefill_text_positions + summary_positions = self._prefill_summary_positions + if text_positions is None: + text_positions = torch.nonzero(text_mask, as_tuple=False).flatten() + summary_positions = torch.nonzero(sm, as_tuple=False).flatten() + n_text = text_positions.numel() + + block = self.summary_chunk_size + self.summary_token_num + # Match what update() + reorganize_after_prefill() would have set: + # update(): cur_chunk_sizes += prefill_len; true_tokens += n_text + # reorganize: cur_chunk_sizes %= block + self.cur_chunk_sizes[layer_idx] = prefill_len % block + self.true_tokens[layer_idx] = n_text + + if self.is_large_window[layer_idx]: + text_k = prefill_k.index_select(2, text_positions) + text_v = prefill_v.index_select(2, text_positions) + cap = n_text + self._APPEND_HEADROOM + new_k = torch.empty(bsz, heads, cap, head_dim, dtype=dtype, device=device) + new_v = torch.empty(bsz, heads, cap, head_dim, dtype=dtype, device=device) + if n_text > 0: + new_k[:, :, :n_text, :].copy_(text_k) + new_v[:, :, :n_text, :].copy_(text_v) + self.key_cache[layer_idx] = new_k + self.value_cache[layer_idx] = new_v + self._text_len[layer_idx] = n_text + self._capacity[layer_idx] = cap + self._total_chunks[layer_idx] = n_text // self.summary_chunk_size + else: + all_summary_k = prefill_k.index_select(2, summary_positions) + all_summary_v = prefill_v.index_select(2, summary_positions) + n_summary = all_summary_k.shape[2] + + C = self.summary_chunk_size + ws = self.window_sizes[layer_idx] + scn = self.sliding_chunk_nums[layer_idx] + + n_complete_chunks = n_text // C + n_partial = n_text % C + n_complete_text = n_complete_chunks * C + + n_window_chunks = min(scn, n_complete_chunks) + n_window_text = n_window_chunks * C + window_start = n_complete_text - n_window_text + # Only retain the live window and partial chunk, avoiding a + # full-context text KV temporary on compressed layers. + live_positions = text_positions[window_start:] + text_k = prefill_k.index_select(2, live_positions) + text_v = prefill_v.index_select(2, live_positions) + + n_old_total = max(0, n_summary - n_window_chunks) + n_new = n_summary - n_old_total + hierarchical = self.meta_pool_enabled[layer_idx] + group_size = self.meta_pool_group_sizes[layer_idx] + n_meta = n_old_total // group_size if hierarchical else 0 + n_old = n_old_total % group_size if hierarchical else n_old_total + old_begin = n_meta * group_size if hierarchical else 0 + + old_s_cap = max(self._SUMMARY_INIT_CAP, (n_old + 1) * 2) + total_cap = ws + old_s_cap + C + new_k = torch.empty(bsz, heads, total_cap, head_dim, dtype=dtype, device=device) + new_v = torch.empty(bsz, heads, total_cap, head_dim, dtype=dtype, device=device) + + if n_window_text > 0: + new_k[:, :, :n_window_text, :].copy_(text_k[:, :, :n_window_text, :]) + new_v[:, :, :n_window_text, :].copy_(text_v[:, :, :n_window_text, :]) + self._n_valid_window[layer_idx] = n_window_text + self._window_write_ptr[layer_idx] = n_window_text % ws + + if n_old > 0: + new_k[:, :, ws:ws + n_old, :].copy_( + all_summary_k[:, :, old_begin:n_old_total, :] + ) + new_v[:, :, ws:ws + n_old, :].copy_( + all_summary_v[:, :, old_begin:n_old_total, :] + ) + self._old_summary_len[layer_idx] = n_old + self._old_summary_total[layer_idx] = n_old_total + self._old_summary_cap[layer_idx] = old_s_cap + + if hierarchical: + required = ( + pre_rope_key_states, + summary_scores, + summary_cos, + summary_sin, + summary_position_ids, + meta_key_states, + meta_value_states, + ) + if any(item is None for item in required): + raise ValueError("hierarchical prefill cache requires pre-RoPE and meta state") + all_pre_k = pre_rope_key_states.index_select(2, summary_positions) + meta_cap = max(self._SUMMARY_INIT_CAP, (n_meta + 1) * 2) + meta_k_buf = torch.empty( + bsz, heads, meta_cap, head_dim, dtype=dtype, device=device + ) + meta_v_buf = torch.empty_like(meta_k_buf) + if n_meta: + meta_k_buf[:, :, :n_meta, :].copy_(meta_key_states[:, :, :n_meta, :]) + meta_v_buf[:, :, :n_meta, :].copy_(meta_value_states[:, :, :n_meta, :]) + self._meta_key_cache[layer_idx] = meta_k_buf + self._meta_value_cache[layer_idx] = meta_v_buf + self._meta_len[layer_idx] = n_meta + self._meta_cap[layer_idx] = meta_cap + + residual_pre = torch.empty( + bsz, heads, group_size, head_dim, dtype=dtype, device=device + ) + score_width = ( + heads * head_dim + if self.meta_pool_mode == "dimension_wise" + else 1 + ) + residual_score = torch.empty( + bsz, + group_size, + score_width, + dtype=summary_scores.dtype, + device=device, + ) + residual_cos = torch.empty( + bsz, group_size, head_dim, dtype=summary_cos.dtype, device=device + ) + residual_sin = torch.empty_like(residual_cos) + residual_position = torch.empty( + bsz, group_size, dtype=torch.long, device=device + ) + if n_old: + residual_pre[:, :, :n_old, :].copy_( + all_pre_k[:, :, old_begin:n_old_total, :] + ) + residual_score[:, :n_old, :].copy_( + summary_scores[:, old_begin:n_old_total] + ) + residual_cos[:, :n_old, :].copy_( + summary_cos[:, old_begin:n_old_total, :] + ) + residual_sin[:, :n_old, :].copy_( + summary_sin[:, old_begin:n_old_total, :] + ) + residual_position[:, :n_old].copy_( + summary_position_ids[:, old_begin:n_old_total] + ) + self._residual_pre_key_buf[layer_idx] = residual_pre + self._residual_score_buf[layer_idx] = residual_score + self._residual_cos_buf[layer_idx] = residual_cos + self._residual_sin_buf[layer_idx] = residual_sin + self._residual_position_buf[layer_idx] = residual_position + + if n_partial > 0: + mirror_start = ws + n_old + new_k[:, :, mirror_start:mirror_start + n_partial, :].copy_( + text_k[:, :, n_window_text:, :]) + new_v[:, :, mirror_start:mirror_start + n_partial, :].copy_( + text_v[:, :, n_window_text:, :]) + + self.key_cache[layer_idx] = new_k + self.value_cache[layer_idx] = new_v + self._capacity[layer_idx] = total_cap + self._text_len[layer_idx] = 0 + + ns_buf_k = torch.empty(bsz, heads, scn, head_dim, dtype=dtype, device=device) + ns_buf_v = torch.empty(bsz, heads, scn, head_dim, dtype=dtype, device=device) + if n_new > 0: + ns_buf_k[:, :, :n_new, :].copy_(all_summary_k[:, :, n_old_total:, :]) + ns_buf_v[:, :, :n_new, :].copy_(all_summary_v[:, :, n_old_total:, :]) + self._new_sum_key_buf[layer_idx] = ns_buf_k + self._new_sum_value_buf[layer_idx] = ns_buf_v + if hierarchical: + ns_pre = torch.empty(bsz, heads, scn, head_dim, dtype=dtype, device=device) + score_width = ( + heads * head_dim + if self.meta_pool_mode == "dimension_wise" + else 1 + ) + ns_score = torch.empty( + bsz, + scn, + score_width, + dtype=summary_scores.dtype, + device=device, + ) + ns_cos = torch.empty(bsz, scn, head_dim, dtype=summary_cos.dtype, device=device) + ns_sin = torch.empty_like(ns_cos) + ns_position = torch.empty( + bsz, scn, dtype=torch.long, device=device + ) + if n_new: + ns_pre[:, :, :n_new, :].copy_(all_pre_k[:, :, n_old_total:, :]) + ns_score[:, :n_new, :].copy_(summary_scores[:, n_old_total:]) + ns_cos[:, :n_new, :].copy_(summary_cos[:, n_old_total:, :]) + ns_sin[:, :n_new, :].copy_(summary_sin[:, n_old_total:, :]) + ns_position[:, :n_new].copy_( + summary_position_ids[:, n_old_total:] + ) + self._new_sum_pre_key_buf[layer_idx] = ns_pre + self._new_sum_score_buf[layer_idx] = ns_score + self._new_sum_cos_buf[layer_idx] = ns_cos + self._new_sum_sin_buf[layer_idx] = ns_sin + self._new_sum_position_buf[layer_idx] = ns_position + self._new_sum_len[layer_idx] = n_new + self._new_sum_write_ptr[layer_idx] = n_new % scn + + self._chunk_buf_len[layer_idx] = n_partial + self._total_chunks[layer_idx] = n_old_total + n_new + + self._reorganized[layer_idx] = True + self.set_sksa_chunk_k_from_prefill( + layer_idx, sksa_chunk_key_states, summary_mask + ) + + # ── Reorganize after prefill ── + + def reorganize_after_prefill(self, summary_mask: torch.Tensor): + """No-op sanity check. + + Prefill path now writes each layer directly into the ring buffer layout via + init_layer_from_prefill, so by the time Qwen3Model.forward calls this, + all layers must already be reorganized. + """ + for layer_idx in range(self.num_hidden_layers): + assert self._reorganized[layer_idx], ( + f"layer {layer_idx} was not initialized via init_layer_from_prefill" + ) + + self._prefill_text_positions = None + self._prefill_summary_positions = None + + # ── Decode: text token update ── + + def update_text( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + *, + sksa_chunk_key_states: Optional[torch.Tensor] = None, + ): + """Write a single text token KV during decode.""" + if self.is_large_window[layer_idx]: + cur = self._text_len[layer_idx] + new_len = cur + 1 + if new_len > self._capacity[layer_idx]: + cap = max(new_len + self._APPEND_HEADROOM, self._capacity[layer_idx] * 2) + old_k, old_v = self.key_cache[layer_idx], self.value_cache[layer_idx] + bsz, heads, _, head_dim = old_k.shape + new_k = torch.empty(bsz, heads, cap, head_dim, dtype=old_k.dtype, device=old_k.device) + new_v = torch.empty(bsz, heads, cap, head_dim, dtype=old_v.dtype, device=old_v.device) + new_k[:, :, :cur, :].copy_(old_k[:, :, :cur, :]) + new_v[:, :, :cur, :].copy_(old_v[:, :, :cur, :]) + self.key_cache[layer_idx] = new_k + self.value_cache[layer_idx] = new_v + self._capacity[layer_idx] = cap + self.key_cache[layer_idx][:, :, cur:new_len, :].copy_(key_states) + self.value_cache[layer_idx][:, :, cur:new_len, :].copy_(value_states) + self._text_len[layer_idx] = new_len + else: + # Write only to key_cache mirror region (chunk_buf eliminated) + ws = self.window_sizes[layer_idx] + n_old = self._old_summary_len[layer_idx] + pos = self._chunk_buf_len[layer_idx] + mirror_pos = ws + n_old + pos + self.key_cache[layer_idx][:, :, mirror_pos:mirror_pos+1, :].copy_(key_states) + self.value_cache[layer_idx][:, :, mirror_pos:mirror_pos+1, :].copy_(value_states) + self._chunk_buf_len[layer_idx] = pos + 1 + + self.cur_chunk_sizes[layer_idx] += 1 + self.true_tokens[layer_idx] += 1 + self.append_sksa_chunk_k(layer_idx, sksa_chunk_key_states) + + # ── Decode: summary token update ── + + def update_summary( + self, + key_states: torch.Tensor, + value_states: torch.Tensor, + layer_idx: int, + *, + pre_rope_key_states: Optional[torch.Tensor] = None, + summary_scores: Optional[torch.Tensor] = None, + summary_cos: Optional[torch.Tensor] = None, + summary_sin: Optional[torch.Tensor] = None, + summary_position_ids: Optional[torch.Tensor] = None, + position_bias: Optional[torch.Tensor] = None, + ): + """Write summary token KV during decode (chunk boundary). + + Large window: skip. + Small window (order matters — flush mirror before evict to avoid clobbering): + 1. Flush mirror region → ring + 2. Evict oldest new_summary → old_summary in key_cache (if full) + 3. Write new summary → new_summary_buf + """ + n_summary = key_states.shape[2] + + if self.is_large_window[layer_idx]: + self.cur_chunk_sizes[layer_idx] += n_summary + self._total_chunks[layer_idx] += n_summary + return + + C = self.summary_chunk_size + ws = self.window_sizes[layer_idx] + scn = self.sliding_chunk_nums[layer_idx] + hierarchical = self.meta_pool_enabled[layer_idx] + group_size = self.meta_pool_group_sizes[layer_idx] + cbl = self._chunk_buf_len[layer_idx] + ptr = self._window_write_ptr[layer_idx] + n_old = self._old_summary_len[layer_idx] + + # Step 1: Flush mirror region → ring (must happen before evict touches mirror[0]) + mirror_start = ws + n_old + if ptr + cbl <= ws: + self.key_cache[layer_idx][:, :, ptr:ptr + cbl, :].copy_( + self.key_cache[layer_idx][:, :, mirror_start:mirror_start + cbl, :]) + self.value_cache[layer_idx][:, :, ptr:ptr + cbl, :].copy_( + self.value_cache[layer_idx][:, :, mirror_start:mirror_start + cbl, :]) + else: + first = ws - ptr + self.key_cache[layer_idx][:, :, ptr:ws, :].copy_( + self.key_cache[layer_idx][:, :, mirror_start:mirror_start + first, :]) + self.value_cache[layer_idx][:, :, ptr:ws, :].copy_( + self.value_cache[layer_idx][:, :, mirror_start:mirror_start + first, :]) + rest = cbl - first + self.key_cache[layer_idx][:, :, :rest, :].copy_( + self.key_cache[layer_idx][:, :, mirror_start + first:mirror_start + cbl, :]) + self.value_cache[layer_idx][:, :, :rest, :].copy_( + self.value_cache[layer_idx][:, :, mirror_start + first:mirror_start + cbl, :]) + + self._window_write_ptr[layer_idx] = (ptr + cbl) % ws + if self._n_valid_window[layer_idx] < ws: + self._n_valid_window[layer_idx] = min(ws, self._n_valid_window[layer_idx] + cbl) + self._chunk_buf_len[layer_idx] = 0 + + # Step 2: Evict oldest new_summary → old_summary (now safe — mirror already flushed) + if self._new_sum_len[layer_idx] >= scn: + read_ptr = self._new_sum_write_ptr[layer_idx] + old_dst = ws + n_old # == mirror_start, but mirror data is already in ring + + # Check capacity for old_summary growth + needed = old_dst + 1 + C # +1 for new old_sum, +C for future chunk mirror + if needed > self._capacity[layer_idx]: + new_s_cap = max(self._old_summary_cap[layer_idx] * 2, n_old + self._SUMMARY_INIT_CAP) + new_total = ws + new_s_cap + C + old_k, old_v = self.key_cache[layer_idx], self.value_cache[layer_idx] + bsz, heads, _, head_dim = old_k.shape + nk = torch.empty(bsz, heads, new_total, head_dim, dtype=old_k.dtype, device=old_k.device) + nv = torch.empty(bsz, heads, new_total, head_dim, dtype=old_v.dtype, device=old_v.device) + copy_len = ws + n_old + nk[:, :, :copy_len, :].copy_(old_k[:, :, :copy_len, :]) + nv[:, :, :copy_len, :].copy_(old_v[:, :, :copy_len, :]) + self.key_cache[layer_idx] = nk + self.value_cache[layer_idx] = nv + self._old_summary_cap[layer_idx] = new_s_cap + self._capacity[layer_idx] = new_total + + self.key_cache[layer_idx][:, :, old_dst:old_dst+1, :].copy_( + self._new_sum_key_buf[layer_idx][:, :, read_ptr:read_ptr+1, :]) + self.value_cache[layer_idx][:, :, old_dst:old_dst+1, :].copy_( + self._new_sum_value_buf[layer_idx][:, :, read_ptr:read_ptr+1, :]) + n_old += 1 + self._old_summary_total[layer_idx] += 1 + + if hierarchical: + self._residual_pre_key_buf[layer_idx][:, :, n_old-1:n_old, :].copy_( + self._new_sum_pre_key_buf[layer_idx][:, :, read_ptr:read_ptr+1, :] + ) + self._residual_score_buf[layer_idx][:, n_old-1:n_old, :].copy_( + self._new_sum_score_buf[layer_idx][:, read_ptr:read_ptr+1, :] + ) + self._residual_cos_buf[layer_idx][:, n_old-1:n_old, :].copy_( + self._new_sum_cos_buf[layer_idx][:, read_ptr:read_ptr+1, :] + ) + self._residual_sin_buf[layer_idx][:, n_old-1:n_old, :].copy_( + self._new_sum_sin_buf[layer_idx][:, read_ptr:read_ptr+1, :] + ) + self._residual_position_buf[layer_idx][:, n_old-1:n_old].copy_( + self._new_sum_position_buf[layer_idx][ + :, read_ptr:read_ptr+1 + ] + ) + + if n_old == group_size: + scores = self._residual_score_buf[layer_idx][ + :, :group_size, : + ] + if self.meta_pool_mode == "dimension_wise": + if position_bias is None: + weights = torch.full_like(scores, 1.0 / group_size) + else: + weights = torch.softmax( + ( + scores + + position_bias.view( + 1, group_size, scores.shape[-1] + ) + ).float(), + dim=1, + ).to(key_states.dtype) + bsz, heads, _, head_dim = self._residual_pre_key_buf[ + layer_idx + ].shape + weights = weights.view( + bsz, group_size, heads, head_dim + ).permute(0, 2, 1, 3) + else: + scalar_scores = scores.squeeze(-1) + if position_bias is None: + scalar_weights = torch.full_like( + scalar_scores, 1.0 / group_size + ) + else: + scalar_weights = torch.softmax( + ( + scalar_scores + + position_bias.view(1, group_size) + ).float(), + dim=1, + ).to(key_states.dtype) + weights = scalar_weights[:, None, :, None] + meta_pre_k = ( + self._residual_pre_key_buf[layer_idx][:, :, :group_size, :] + * weights + ).sum(dim=2) + meta_v = ( + self.value_cache[layer_idx][:, :, ws:ws + group_size, :] + * weights + ).sum(dim=2) + last_cos = self._residual_cos_buf[layer_idx][:, group_size - 1, :] + last_sin = self._residual_sin_buf[layer_idx][:, group_size - 1, :] + meta_k = ( + meta_pre_k * last_cos[:, None, :] + + rotate_half(meta_pre_k) * last_sin[:, None, :] + ) + + meta_dst = self._meta_len[layer_idx] + if meta_dst >= self._meta_cap[layer_idx]: + old_meta_k = self._meta_key_cache[layer_idx] + old_meta_v = self._meta_value_cache[layer_idx] + new_cap = max(meta_dst + self._SUMMARY_INIT_CAP, self._meta_cap[layer_idx] * 2) + bsz, heads, _, head_dim = old_meta_k.shape + new_meta_k = torch.empty( + bsz, heads, new_cap, head_dim, + dtype=old_meta_k.dtype, device=old_meta_k.device, + ) + new_meta_v = torch.empty_like(new_meta_k) + new_meta_k[:, :, :meta_dst, :].copy_(old_meta_k[:, :, :meta_dst, :]) + new_meta_v[:, :, :meta_dst, :].copy_(old_meta_v[:, :, :meta_dst, :]) + self._meta_key_cache[layer_idx] = new_meta_k + self._meta_value_cache[layer_idx] = new_meta_v + self._meta_cap[layer_idx] = new_cap + self._meta_key_cache[layer_idx][:, :, meta_dst:meta_dst+1, :].copy_( + meta_k.unsqueeze(2) + ) + self._meta_value_cache[layer_idx][:, :, meta_dst:meta_dst+1, :].copy_( + meta_v.unsqueeze(2) + ) + self._meta_len[layer_idx] += 1 + n_old = 0 + + self._old_summary_len[layer_idx] = n_old + + # Step 3: Write new summary to new_summary_buf (overwrite oldest slot) + w_ptr = self._new_sum_write_ptr[layer_idx] + self._new_sum_key_buf[layer_idx][:, :, w_ptr:w_ptr+1, :].copy_(key_states) + self._new_sum_value_buf[layer_idx][:, :, w_ptr:w_ptr+1, :].copy_(value_states) + if hierarchical: + if any( + item is None + for item in ( + pre_rope_key_states, + summary_scores, + summary_cos, + summary_sin, + summary_position_ids, + ) + ): + raise ValueError("hierarchical decode requires pre-RoPE summary state") + self._new_sum_pre_key_buf[layer_idx][:, :, w_ptr:w_ptr+1, :].copy_( + pre_rope_key_states + ) + self._new_sum_score_buf[layer_idx][:, w_ptr:w_ptr+1, :].copy_( + summary_scores + ) + self._new_sum_cos_buf[layer_idx][:, w_ptr:w_ptr+1, :].copy_(summary_cos) + self._new_sum_sin_buf[layer_idx][:, w_ptr:w_ptr+1, :].copy_(summary_sin) + self._new_sum_position_buf[layer_idx][ + :, w_ptr:w_ptr+1 + ].copy_(summary_position_ids) + self._new_sum_write_ptr[layer_idx] = (w_ptr + 1) % scn + if self._new_sum_len[layer_idx] < scn: + self._new_sum_len[layer_idx] += 1 + + self.cur_chunk_sizes[layer_idx] += n_summary + self._total_chunks[layer_idx] += n_summary + + # ── Decode: get KV for attention ── + + def get_attention_kv(self, layer_idx: int) -> tuple[torch.Tensor, torch.Tensor, int]: + """Get full KV for text token attention. + + Large window: buffer[:text_len] + Small window (steady state): key_cache[:ws + n_old + cbl] — single slice, zero cat + """ + if self.is_large_window[layer_idx]: + tl = self._text_len[layer_idx] + return (self.key_cache[layer_idx][:, :, :tl, :], + self.value_cache[layer_idx][:, :, :tl, :], 0) + + ws = self.window_sizes[layer_idx] + nv = self._n_valid_window[layer_idx] + cbl = self._chunk_buf_len[layer_idx] + n_old = self._old_summary_len[layer_idx] + + # Steady state: ring full → [ring(ws) | old_sums(n_old) | chunk_mirror(cbl)] contiguous + if nv == ws: + end = ws + n_old + cbl + base_k = self.key_cache[layer_idx][:, :, :end, :] + base_v = self.value_cache[layer_idx][:, :, :end, :] + else: + # Warmup: ring not full, [nv:ws] is gap → cat + parts_k, parts_v = [], [] + if nv > 0: + parts_k.append(self.key_cache[layer_idx][:, :, :nv, :]) + parts_v.append(self.value_cache[layer_idx][:, :, :nv, :]) + if cbl > 0: + mirror_start = ws + n_old + parts_k.append(self.key_cache[layer_idx][:, :, mirror_start:mirror_start + cbl, :]) + parts_v.append(self.value_cache[layer_idx][:, :, mirror_start:mirror_start + cbl, :]) + if n_old > 0: + parts_k.append(self.key_cache[layer_idx][:, :, ws:ws + n_old, :]) + parts_v.append(self.value_cache[layer_idx][:, :, ws:ws + n_old, :]) + base_k = parts_k[0] if len(parts_k) == 1 else torch.cat(parts_k, dim=2) + base_v = parts_v[0] if len(parts_v) == 1 else torch.cat(parts_v, dim=2) + + meta_len = self._meta_len[layer_idx] + if meta_len: + base_k = torch.cat( + (base_k, self._meta_key_cache[layer_idx][:, :, :meta_len, :]), dim=2 + ) + base_v = torch.cat( + (base_v, self._meta_value_cache[layer_idx][:, :, :meta_len, :]), dim=2 + ) + return base_k, base_v, meta_len + + def get_current_chunk_kv(self, layer_idx: int) -> tuple[torch.Tensor, torch.Tensor]: + """Get KV of the current chunk's C text tokens for summary token attention.""" + C = self.summary_chunk_size + if self.is_large_window[layer_idx]: + tl = self._text_len[layer_idx] + return (self.key_cache[layer_idx][:, :, tl - C:tl, :], + self.value_cache[layer_idx][:, :, tl - C:tl, :]) + else: + ws = self.window_sizes[layer_idx] + n_old = self._old_summary_len[layer_idx] + cbl = self._chunk_buf_len[layer_idx] + mirror_start = ws + n_old + return (self.key_cache[layer_idx][:, :, mirror_start:mirror_start + cbl, :], + self.value_cache[layer_idx][:, :, mirror_start:mirror_start + cbl, :]) + + def reset_chunk_counter(self): + """Reset chunk counters after a chunk boundary step completes.""" + block = self.summary_chunk_size + self.summary_token_num + for layer_idx in range(self.num_hidden_layers): + if self.cur_chunk_sizes[layer_idx] >= block: + self.cur_chunk_sizes[layer_idx] %= block diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1127914c405aa23a6541fe6b497286df39f2bbba --- /dev/null +++ b/config.json @@ -0,0 +1,79 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "auto_map": { + "AutoModel": "modeling_qwen3.Qwen3Model", + "AutoModelForCausalLM": "modeling_qwen3.Qwen3ForCausalLM" + }, + "bos_token_id": 151643, + "eos_token_id": 151643, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "max_position_embeddings": 131072, + "max_window_layers": 24, + "mix_coeff": 0.0, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 24, + "num_key_value_heads": 16, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sksa_apply_sparse_attention": true, + "sksa_enabled": true, + "sksa_indexer_block_size": 4, + "sksa_indexer_chunk_attention_include_summary": false, + "sksa_indexer_chunk_attention_qk_source": "independent", + "sksa_indexer_chunk_attention_v_source": "independent", + "sksa_indexer_force_sink": true, + "sksa_indexer_head_dim": 128, + "sksa_indexer_heads": 8, + "sksa_indexer_qk_norm": false, + "sksa_indexer_query_block_size": 512, + "sksa_indexer_query_rank": 512, + "sksa_indexer_rope_dim": 64, + "sksa_indexer_scorer_backend": "fused", + "sksa_indexer_summary_key_source": "chunk_attention", + "sksa_indexer_teacher_pooling": "max", + "sksa_indexer_token_budget": 4096, + "sksa_local_window_tokens": 1024, + "sksa_routing_mode": "summary_indexer", + "sksa_sparse_full_layers": [ + 3, + 7, + 11, + 15, + 19, + 23 + ], + "sliding_window": null, + "summary_chunk_position_ids_type": "origin", + "summary_chunk_size": 8, + "summary_independent_attention_layernorm": false, + "summary_independent_parameters": true, + "summary_layer_freq": "([1]*4)*6", + "summary_meta_pool_block_size": 512, + "summary_meta_pool_group_size": "([128]*1+[128]*1+[128]*1+[0]*1)*6", + "summary_meta_pool_mass_correction": "log_group_size", + "summary_meta_pool_mode": "dimension_wise", + "summary_meta_pool_position": "group_last", + "summary_partial_chunk_policy": "defer", + "summary_sliding_chunk_num": "([128]*3+[16384]*1)*6", + "summary_token_begin": 151936, + "summary_token_num": 1, + "summary_token_position_ids_type": "last_chunk_slice_right", + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "5.0.0", + "truncate_predict_nums": 151936, + "use_cache": true, + "use_sliding_window": false, + "use_summary_attention": true, + "vocab_size": 151937 +} diff --git a/elementwise_fusion.py b/elementwise_fusion.py new file mode 100644 index 0000000000000000000000000000000000000000..a4c52a8b432ff019fe470b16c0b3997890895284 --- /dev/null +++ b/elementwise_fusion.py @@ -0,0 +1,75 @@ +"""BF16 inference fusion; FMA disabled, original cast points explicit. + +Torch pow/mean is retained verbatim: no alternate RMS reduction is substituted. +The Q/K path materializes normalized pre-RoPE keys because meta pooling uses them. +""" +import torch +import triton as tr +import triton.language as tl +import triton.language.extra.cuda.libdevice as lib + + +@tr.jit +def norm_tail(X,V,W,Y,N:tl.constexpr,D:tl.constexpr,EPS:tl.constexpr,BLOCK:tl.constexpr): + i=tl.program_id(0)*BLOCK+tl.arange(0,BLOCK) + x=tl.load(X+i,i&": 5789,
+ "CON": 5790,
+ "Ġrepl": 5791,
+ "Ġregular": 5792,
+ "Storage": 5793,
+ "ramework": 5794,
+ "Ġgoal": 5795,
+ "Ġtouch": 5796,
+ ".widget": 5797,
+ "Ġbuilt": 5798,
+ "des": 5799,
+ "Part": 5800,
+ "(re": 5801,
+ "Ġworth": 5802,
+ "hib": 5803,
+ "game": 5804,
+ "Ġв": 5805,
+ "acion": 5806,
+ "ĠWhite": 5807,
+ "(type": 5808,
+ "(`": 5809,
+ "Ġnatural": 5810,
+ "Ġinj": 5811,
+ "Ġcalcul": 5812,
+ "ĠApril": 5813,
+ ".List": 5814,
+ "Ġassociated": 5815,
+ "ĉSystem": 5816,
+ "~~": 5817,
+ "=[": 5818,
+ "Ġstorage": 5819,
+ "Ġbytes": 5820,
+ "Ġtravel": 5821,
+ "Ġsou": 5822,
+ "Ġpassed": 5823,
+ "!=": 5824,
+ "ascript": 5825,
+ ".open": 5826,
+ "Ġgrid": 5827,
+ "Ġbus": 5828,
+ "Ġrecogn": 5829,
+ "Ab": 5830,
+ "Ġhon": 5831,
+ "ĠCenter": 5832,
+ "Ġprec": 5833,
+ "build": 5834,
+ "HTML": 5835,
+ "ĠSan": 5836,
+ "Ġcountries": 5837,
+ "aled": 5838,
+ "token": 5839,
+ "kt": 5840,
+ "Ġqual": 5841,
+ "Last": 5842,
+ "adow": 5843,
+ "Ġmanufact": 5844,
+ "idad": 5845,
+ "jango": 5846,
+ "Next": 5847,
+ "xf": 5848,
+ ".a": 5849,
+ "Ġporno": 5850,
+ "ĠPM": 5851,
+ "erve": 5852,
+ "iting": 5853,
+ "_th": 5854,
+ "ci": 5855,
+ "=None": 5856,
+ "gs": 5857,
+ "Ġlogin": 5858,
+ "atives": 5859,
+ "']);Ċ": 5860,
+ "Äħ": 5861,
+ "Ġill": 5862,
+ "IA": 5863,
+ "children": 5864,
+ "DO": 5865,
+ "Ġlevels": 5866,
+ "Ġ{{": 5867,
+ "Ġlooks": 5868,
+ "Ġ\"#": 5869,
+ "ToString": 5870,
+ "Ġnecessary": 5871,
+ "ĠĠĠĊ": 5872,
+ "cell": 5873,
+ "Entry": 5874,
+ "Ġ'#": 5875,
+ "Ġextrem": 5876,
+ "Selector": 5877,
+ "Ġplaceholder": 5878,
+ "Load": 5879,
+ "Ġreleased": 5880,
+ "ORE": 5881,
+ "Enumer": 5882,
+ "ĠTV": 5883,
+ "SET": 5884,
+ "inq": 5885,
+ "Press": 5886,
+ "ĠDepartment": 5887,
+ "Ġproperties": 5888,
+ "Ġrespond": 5889,
+ "Search": 5890,
+ "ael": 5891,
+ "Ġrequ": 5892,
+ "ĠBook": 5893,
+ "/Ċ": 5894,
+ "(st": 5895,
+ "Ġfinancial": 5896,
+ "icket": 5897,
+ "_input": 5898,
+ "Ġthreat": 5899,
+ "(in": 5900,
+ "Strip": 5901,
+ "ìĿ": 5902,
+ "ção": 5903,
+ "Ġevidence": 5904,
+ "));": 5905,
+ "ĠBro": 5906,
+ "Ġ[];Ċ": 5907,
+ "Ġou": 5908,
+ "buf": 5909,
+ "Script": 5910,
+ "dat": 5911,
+ "Ġrule": 5912,
+ "#import": 5913,
+ "=\"/": 5914,
+ "Serial": 5915,
+ "Ġstarting": 5916,
+ "[index": 5917,
+ "ae": 5918,
+ "Ġcontrib": 5919,
+ "session": 5920,
+ "_new": 5921,
+ "utable": 5922,
+ "ober": 5923,
+ "Ġ\"./": 5924,
+ "Ġlogger": 5925,
+ "Ġrecently": 5926,
+ "Ġreturned": 5927,
+ "ččĊ": 5928,
+ ")))Ċ": 5929,
+ "itions": 5930,
+ "Ġseek": 5931,
+ "Ġcommunic": 5932,
+ "Ġ\".": 5933,
+ "Ġusername": 5934,
+ "ECT": 5935,
+ "DS": 5936,
+ "Ġotherwise": 5937,
+ "ĠGerman": 5938,
+ ".aw": 5939,
+ "Adapter": 5940,
+ "ixel": 5941,
+ "Ġsystems": 5942,
+ "Ġdrop": 5943,
+ "Ġstructure": 5944,
+ "Ġ$(\"#": 5945,
+ "encies": 5946,
+ "anning": 5947,
+ "ĠLink": 5948,
+ "ĠResponse": 5949,
+ "Ġstri": 5950,
+ "ż": 5951,
+ "ĠDB": 5952,
+ "æĹ": 5953,
+ "android": 5954,
+ "submit": 5955,
+ "otion": 5956,
+ "(@": 5957,
+ ".test": 5958,
+ "ĊĊĊĊĊĊĊĊ": 5959,
+ "];čĊ": 5960,
+ "Ġdirectly": 5961,
+ "Ġ\"%": 5962,
+ "ris": 5963,
+ "elta": 5964,
+ "AIL": 5965,
+ "){čĊ": 5966,
+ "mine": 5967,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 5968,
+ "(k": 5969,
+ "bon": 5970,
+ "asic": 5971,
+ "pite": 5972,
+ "___": 5973,
+ "Max": 5974,
+ "Ġerrors": 5975,
+ "ĠWhile": 5976,
+ "Ġarguments": 5977,
+ "Ġensure": 5978,
+ "Right": 5979,
+ "-based": 5980,
+ "Web": 5981,
+ "Ġ-=": 5982,
+ "Ġintrodu": 5983,
+ "ĠInst": 5984,
+ "ĠWash": 5985,
+ "ordin": 5986,
+ "join": 5987,
+ "Database": 5988,
+ "Ġgrad": 5989,
+ "Ġusually": 5990,
+ "ITE": 5991,
+ "Props": 5992,
+ "?>Ċ": 5993,
+ "ĠGo": 5994,
+ "@Override": 5995,
+ "REF": 5996,
+ "Ġip": 5997,
+ "ĠAustral": 5998,
+ "Ġist": 5999,
+ "ViewById": 6000,
+ "Ġserious": 6001,
+ "Ġcustomer": 6002,
+ ".prototype": 6003,
+ "odo": 6004,
+ "cor": 6005,
+ "Ġdoor": 6006,
+ "ĠWITHOUT": 6007,
+ "Ġplant": 6008,
+ "Ġbegan": 6009,
+ "Ġdistance": 6010,
+ "()).": 6011,
+ "Ġchance": 6012,
+ "Ġord": 6013,
+ "came": 6014,
+ "pragma": 6015,
+ "Ġprotect": 6016,
+ "ragment": 6017,
+ "ĠNode": 6018,
+ "ening": 6019,
+ "Ñĩ": 6020,
+ "Ġroute": 6021,
+ "ĠSchool": 6022,
+ "hi": 6023,
+ "Ġneighb": 6024,
+ "After": 6025,
+ "licit": 6026,
+ "Ġcontr": 6027,
+ "Ġprimary": 6028,
+ "AA": 6029,
+ ".WriteLine": 6030,
+ "utils": 6031,
+ "Ġbi": 6032,
+ "Red": 6033,
+ ".Linq": 6034,
+ ".object": 6035,
+ "Ġleaders": 6036,
+ "unities": 6037,
+ "Ġgun": 6038,
+ "onth": 6039,
+ "ĠDev": 6040,
+ "FILE": 6041,
+ "Ġcomments": 6042,
+ "_len": 6043,
+ "arrow": 6044,
+ "amount": 6045,
+ "Range": 6046,
+ "sert": 6047,
+ "GridView": 6048,
+ "Ġupdated": 6049,
+ "ĠMo": 6050,
+ "Ġinform": 6051,
+ "ociety": 6052,
+ "ala": 6053,
+ "Access": 6054,
+ "Ġhab": 6055,
+ "Ġcreat": 6056,
+ "_arg": 6057,
+ "ĠJanuary": 6058,
+ "ĠDay": 6059,
+ "\")čĊ": 6060,
+ "uple": 6061,
+ "document": 6062,
+ "gorith": 6063,
+ "menu": 6064,
+ "ĠOver": 6065,
+ "bb": 6066,
+ ".title": 6067,
+ "_out": 6068,
+ "Ġled": 6069,
+ "uri": 6070,
+ "Ġ?>": 6071,
+ "gl": 6072,
+ "Ġbank": 6073,
+ "ayment": 6074,
+ "ĉprintf": 6075,
+ "MD": 6076,
+ "Ġsample": 6077,
+ "Ġhands": 6078,
+ "ĠVersion": 6079,
+ "uario": 6080,
+ "Ġoffers": 6081,
+ "ityEngine": 6082,
+ "Ġshape": 6083,
+ "Ġsleep": 6084,
+ "_point": 6085,
+ "Settings": 6086,
+ "Ġachie": 6087,
+ "Ġsold": 6088,
+ "ota": 6089,
+ ".bind": 6090,
+ "Am": 6091,
+ "Ġsafe": 6092,
+ "Store": 6093,
+ "Ġshared": 6094,
+ "Ġpriv": 6095,
+ "_VAL": 6096,
+ "Ġsens": 6097,
+ "){": 6098,
+ "Ġremember": 6099,
+ "shared": 6100,
+ "element": 6101,
+ "Ġshoot": 6102,
+ "Vert": 6103,
+ "cout": 6104,
+ "Ġenv": 6105,
+ "_label": 6106,
+ "Ġ>Ċ": 6107,
+ "run": 6108,
+ "Ġscene": 6109,
+ "(array": 6110,
+ "device": 6111,
+ "_title": 6112,
+ "agon": 6113,
+ "]čĊ": 6114,
+ "aby": 6115,
+ "Ġbecame": 6116,
+ "boolean": 6117,
+ "Ġpark": 6118,
+ "ĠCode": 6119,
+ "upload": 6120,
+ "riday": 6121,
+ "ĠSeptember": 6122,
+ "Fe": 6123,
+ "Ġsen": 6124,
+ "cing": 6125,
+ "FL": 6126,
+ "Col": 6127,
+ "uts": 6128,
+ "_page": 6129,
+ "inn": 6130,
+ "Ġimplied": 6131,
+ "aling": 6132,
+ "Ġyourself": 6133,
+ ".Count": 6134,
+ "conf": 6135,
+ "Ġaud": 6136,
+ "_init": 6137,
+ ".)": 6138,
+ "Ġwrote": 6139,
+ "NG": 6140,
+ ".Error": 6141,
+ "ä»": 6142,
+ ".for": 6143,
+ "Ġequal": 6144,
+ "ĠRequest": 6145,
+ "Ġserial": 6146,
+ "Ġallows": 6147,
+ "XX": 6148,
+ "Ġmiddle": 6149,
+ "chor": 6150,
+ "ø": 6151,
+ "erval": 6152,
+ ".Column": 6153,
+ "reading": 6154,
+ "Ġescort": 6155,
+ "ĠAugust": 6156,
+ "Ġquickly": 6157,
+ "Ġweap": 6158,
+ "ĠCG": 6159,
+ "ropri": 6160,
+ "ho": 6161,
+ "Ġcop": 6162,
+ "(struct": 6163,
+ "ĠBig": 6164,
+ "Ġvs": 6165,
+ "Ġfrequ": 6166,
+ ".Value": 6167,
+ "Ġactions": 6168,
+ "Ġproper": 6169,
+ "Ġinn": 6170,
+ "Ġobjects": 6171,
+ "Ġmatrix": 6172,
+ "avascript": 6173,
+ "Ġones": 6174,
+ ".group": 6175,
+ "Ġgreen": 6176,
+ "Ġpaint": 6177,
+ "ools": 6178,
+ "ycl": 6179,
+ "encode": 6180,
+ "olt": 6181,
+ "comment": 6182,
+ ".api": 6183,
+ "Dir": 6184,
+ "Ġune": 6185,
+ "izont": 6186,
+ ".position": 6187,
+ "Ġdesigned": 6188,
+ "_val": 6189,
+ "avi": 6190,
+ "iring": 6191,
+ "tab": 6192,
+ "Ġlayer": 6193,
+ "Ġviews": 6194,
+ "Ġreve": 6195,
+ "rael": 6196,
+ "ĠON": 6197,
+ "rics": 6198,
+ "np": 6199,
+ "Ġcore": 6200,
+ "());čĊ": 6201,
+ "Main": 6202,
+ "Ġexpert": 6203,
+ "ĉĉčĊ": 6204,
+ "_en": 6205,
+ "Ġ/>": 6206,
+ "utter": 6207,
+ "IAL": 6208,
+ "ails": 6209,
+ "ĠKing": 6210,
+ "*/ĊĊ": 6211,
+ "ĠMet": 6212,
+ "_end": 6213,
+ "addr": 6214,
+ "ora": 6215,
+ "Ġir": 6216,
+ "Min": 6217,
+ "Ġsurpr": 6218,
+ "Ġrepe": 6219,
+ "Ġdirectory": 6220,
+ "PUT": 6221,
+ "-S": 6222,
+ "Ġelection": 6223,
+ "haps": 6224,
+ ".pre": 6225,
+ "cm": 6226,
+ "Values": 6227,
+ "Ġ\"Ċ": 6228,
+ "column": 6229,
+ "ivil": 6230,
+ "Login": 6231,
+ "inue": 6232,
+ "Ġbeautiful": 6233,
+ "Ġsecret": 6234,
+ "(event": 6235,
+ "Ġchat": 6236,
+ "ums": 6237,
+ "Ġorigin": 6238,
+ "Ġeffects": 6239,
+ "Ġmanagement": 6240,
+ "illa": 6241,
+ "tk": 6242,
+ "Ġsetting": 6243,
+ "ĠCour": 6244,
+ "Ġmassage": 6245,
+ "ĉend": 6246,
+ "Ġhappy": 6247,
+ "Ġfinish": 6248,
+ "Ġcamera": 6249,
+ "ĠVer": 6250,
+ "ĠDemocr": 6251,
+ "ĠHer": 6252,
+ "(Q": 6253,
+ "cons": 6254,
+ "ita": 6255,
+ "Ġ'.": 6256,
+ "{}": 6257,
+ "ĉC": 6258,
+ "Ġstuff": 6259,
+ "Ġ:Ċ": 6260,
+ "ĠAR": 6261,
+ "Task": 6262,
+ "hidden": 6263,
+ "eros": 6264,
+ "IGN": 6265,
+ "atio": 6266,
+ "ĠHealth": 6267,
+ "olute": 6268,
+ "Enter": 6269,
+ "'>": 6270,
+ "ĠTwitter": 6271,
+ "ĠCounty": 6272,
+ "scribe": 6273,
+ "Ġ=>Ċ": 6274,
+ "Ġhy": 6275,
+ "fit": 6276,
+ "Ġmilitary": 6277,
+ "Ġsale": 6278,
+ "required": 6279,
+ "non": 6280,
+ "bootstrap": 6281,
+ "hold": 6282,
+ "rim": 6283,
+ "-old": 6284,
+ "ĠDown": 6285,
+ "Ġmention": 6286,
+ "contact": 6287,
+ "_group": 6288,
+ "oday": 6289,
+ "Ġtown": 6290,
+ "Ġsolution": 6291,
+ "uate": 6292,
+ "elling": 6293,
+ "]->": 6294,
+ "otes": 6295,
+ "ental": 6296,
+ "omen": 6297,
+ "ospital": 6298,
+ "ĠSup": 6299,
+ "_EN": 6300,
+ "Ġslow": 6301,
+ "SESSION": 6302,
+ "Ġblue": 6303,
+ "ago": 6304,
+ "Ġlives": 6305,
+ "Ġ^": 6306,
+ ".un": 6307,
+ "inst": 6308,
+ "enge": 6309,
+ "Ġcustomers": 6310,
+ "Ġcast": 6311,
+ "udget": 6312,
+ "ï¼ģ": 6313,
+ "icens": 6314,
+ "Ġdetermin": 6315,
+ "Selected": 6316,
+ "_pl": 6317,
+ "ueue": 6318,
+ "Ġdark": 6319,
+ "//ĊĊ": 6320,
+ "si": 6321,
+ "thern": 6322,
+ "ĠJapan": 6323,
+ "/w": 6324,
+ "PU": 6325,
+ "ĠEast": 6326,
+ "ovie": 6327,
+ "Ġpackage": 6328,
+ "Ġnor": 6329,
+ "Ġapi": 6330,
+ "bot": 6331,
+ "\"];Ċ": 6332,
+ "_post": 6333,
+ "ulate": 6334,
+ "Ġclub": 6335,
+ "'));Ċ": 6336,
+ "Ġloop": 6337,
+ "PIO": 6338,
+ "ione": 6339,
+ "shot": 6340,
+ "Initial": 6341,
+ "Ġplayed": 6342,
+ "register": 6343,
+ "rought": 6344,
+ "_max": 6345,
+ "acement": 6346,
+ "match": 6347,
+ "raphics": 6348,
+ "AST": 6349,
+ "Ġexisting": 6350,
+ "Ġcomplex": 6351,
+ "DA": 6352,
+ ".Ch": 6353,
+ ".common": 6354,
+ "mo": 6355,
+ "Ġ'../../": 6356,
+ "ito": 6357,
+ "Ġanalysis": 6358,
+ "Ġdeliver": 6359,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ": 6360,
+ "idx": 6361,
+ "Ãł": 6362,
+ "ongo": 6363,
+ "ĠEnglish": 6364,
+ "Ċ": 9992,
+ "_default": 9993,
+ "ĠDatabase": 9994,
+ "rep": 9995,
+ "ESS": 9996,
+ "nergy": 9997,
+ ".Find": 9998,
+ "_mask": 9999,
+ "Ġrise": 10000,
+ "Ġkernel": 10001,
+ "::$": 10002,
+ ".Q": 10003,
+ "Ġoffering": 10004,
+ "decl": 10005,
+ "ĠCS": 10006,
+ "Ġlisted": 10007,
+ "Ġmostly": 10008,
+ "enger": 10009,
+ "Ġblocks": 10010,
+ "olo": 10011,
+ "Ġgoverning": 10012,
+ "\\F": 10013,
+ "Ġconcent": 10014,
+ ".getText": 10015,
+ "Ġmb": 10016,
+ "Ġoccurred": 10017,
+ "Ġchanging": 10018,
+ "Scene": 10019,
+ "_CODE": 10020,
+ "Beh": 10021,
+ "\"The": 10022,
+ "Ġtile": 10023,
+ "ĠAssociation": 10024,
+ "ĉP": 10025,
+ "alty": 10026,
+ "_ad": 10027,
+ "odies": 10028,
+ "iated": 10029,
+ "Ġprepared": 10030,
+ "possible": 10031,
+ "Ġmort": 10032,
+ "TEST": 10033,
+ "Ġignore": 10034,
+ "Ġcalc": 10035,
+ "Ġrs": 10036,
+ "ĠassertEquals": 10037,
+ "Ġsz": 10038,
+ "ĠTHIS": 10039,
+ ".\"Ċ": 10040,
+ "Ġcanvas": 10041,
+ "java": 10042,
+ "Ġdut": 10043,
+ "VALID": 10044,
+ ".sql": 10045,
+ ".input": 10046,
+ "Ġaux": 10047,
+ "Sup": 10048,
+ "Ġartist": 10049,
+ "Vec": 10050,
+ "_TIME": 10051,
+ ".stringify": 10052,
+ "etween": 10053,
+ "ĠCategory": 10054,
+ "Ġ[-": 10055,
+ "ĠDevExpress": 10056,
+ "ĠJul": 10057,
+ "Ġring": 10058,
+ ".ed": 10059,
+ "YY": 10060,
+ "Let": 10061,
+ "TextField": 10062,
+ "Ġflat": 10063,
+ "_print": 10064,
+ "ĠOTHER": 10065,
+ "adian": 10066,
+ "Ġchecked": 10067,
+ "ele": 10068,
+ "Align": 10069,
+ "standing": 10070,
+ "Ġ[],": 10071,
+ "Ġlab": 10072,
+ "ucky": 10073,
+ "ĠChristmas": 10074,
+ "(image": 10075,
+ ".module": 10076,
+ "Ġlots": 10077,
+ "Ġslightly": 10078,
+ "(final": 10079,
+ "erge": 10080,
+ "è¿": 10081,
+ "ĠPolice": 10082,
+ "ĠRight": 10083,
+ "Ġaward": 10084,
+ "ĠOS": 10085,
+ "Ġ{}ĊĊ": 10086,
+ "Ġptr": 10087,
+ "oves": 10088,
+ "icated": 10089,
+ "ем": 10090,
+ "Ġmanage": 10091,
+ "oliday": 10092,
+ "Amount": 10093,
+ "oolStrip": 10094,
+ "tbody": 10095,
+ "Nav": 10096,
+ "wrap": 10097,
+ "BB": 10098,
+ "Ġwatching": 10099,
+ "arios": 10100,
+ "Ġoptional": 10101,
+ "_K": 10102,
+ "ĠLicensed": 10103,
+ ".Map": 10104,
+ "Timer": 10105,
+ "ĠAP": 10106,
+ "ĠRev": 10107,
+ "(o": 10108,
+ ",c": 10109,
+ "umin": 10110,
+ "etailed": 10111,
+ "ĠHy": 10112,
+ "Ġblank": 10113,
+ "agger": 10114,
+ "ĠSelf": 10115,
+ "()[": 10116,
+ ".make": 10117,
+ "earn": 10118,
+ "channel": 10119,
+ ";Ċ": 10133,
+ "World": 10134,
+ "Ġpython": 10135,
+ "Ġlif": 10136,
+ "Ġtrav": 10137,
+ "Ġconven": 10138,
+ "company": 10139,
+ "ĠClub": 10140,
+ "Ver": 10141,
+ "Btn": 10142,
+ "Ġzone": 10143,
+ "products": 10144,
+ "ĠEduc": 10145,
+ "Ġverify": 10146,
+ "ĠMil": 10147,
+ "ono": 10148,
+ "]);ĊĊ": 10149,
+ "ENCE": 10150,
+ "Ġpacket": 10151,
+ "Ġcer": 10152,
+ "Ġenumer": 10153,
+ "Ġpars": 10154,
+ "formed": 10155,
+ "Ġoccup": 10156,
+ "tre": 10157,
+ "Ġexercise": 10158,
+ "Day": 10159,
+ "_sum": 10160,
+ "Ġasking": 10161,
+ "aption": 10162,
+ "Ġorders": 10163,
+ "Ġspending": 10164,
+ "ĠERR": 10165,
+ ".Dis": 10166,
+ "ĠUtil": 10167,
+ "âĢľI": 10168,
+ "\\'": 10169,
+ "?)": 10170,
+ "/>Ċ": 10171,
+ "Ġemot": 10172,
+ "Ġinfluence": 10173,
+ "ĠAfrica": 10174,
+ "atters": 10175,
+ "Ùħ": 10176,
+ ".session": 10177,
+ "Ġchief": 10178,
+ "ĉĉĉĉĉĉĉĉĉĉĉ": 10179,
+ "Ġtom": 10180,
+ "cluded": 10181,
+ "serial": 10182,
+ "_handler": 10183,
+ ".Type": 10184,
+ "aped": 10185,
+ "Ġpolicies": 10186,
+ "-ex": 10187,
+ "-tr": 10188,
+ "blank": 10189,
+ "merce": 10190,
+ "Ġcoverage": 10191,
+ "Ġrc": 10192,
+ "_matrix": 10193,
+ "_box": 10194,
+ "Ġcharges": 10195,
+ "ĠBoston": 10196,
+ "Pe": 10197,
+ "Ġcircum": 10198,
+ "Ġfilled": 10199,
+ "Ġnorth": 10200,
+ "ictureBox": 10201,
+ "ĉres": 10202,
+ "è®": 10203,
+ "Ġtermin": 10204,
+ "Ġ[â̦": 10205,
+ "IRECT": 10206,
+ "Ġber": 10207,
+ "Ġ\"../../": 10208,
+ "retch": 10209,
+ ".code": 10210,
+ "_col": 10211,
+ "ĠGovernment": 10212,
+ "Ġargv": 10213,
+ "ĠLord": 10214,
+ "asi": 10215,
+ "Exec": 10216,
+ "ĉlet": 10217,
+ "vertis": 10218,
+ "Ġdiscussion": 10219,
+ "enance": 10220,
+ "outube": 10221,
+ "typeof": 10222,
+ "Ġserved": 10223,
+ "ĠPut": 10224,
+ "ĉx": 10225,
+ "Ġsweet": 10226,
+ "Before": 10227,
+ "ategy": 10228,
+ ".of": 10229,
+ "ĠMaterial": 10230,
+ "Sort": 10231,
+ "ONT": 10232,
+ "igital": 10233,
+ "Why": 10234,
+ "Ġsust": 10235,
+ "Ġç": 10236,
+ "abet": 10237,
+ "Ġsegment": 10238,
+ "Ġ[],Ċ": 10239,
+ "ĠMuslim": 10240,
+ "ĠfindViewById": 10241,
+ "cut": 10242,
+ "_TEXT": 10243,
+ "ĠMary": 10244,
+ "Ġloved": 10245,
+ "Ġlie": 10246,
+ "ĠJO": 10247,
+ "Ġisset": 10248,
+ "month": 10249,
+ "Ġprime": 10250,
+ "ti": 10251,
+ "ĠCarol": 10252,
+ "Use": 10253,
+ "ĠPop": 10254,
+ "ĠSave": 10255,
+ "Interval": 10256,
+ "execute": 10257,
+ "dy": 10258,
+ "ĠIran": 10259,
+ "_cont": 10260,
+ "ĉT": 10261,
+ "Ġphase": 10262,
+ "checkbox": 10263,
+ "week": 10264,
+ "Ġhide": 10265,
+ "Ġtil": 10266,
+ "Ġju": 10267,
+ "Custom": 10268,
+ "burg": 10269,
+ "/M": 10270,
+ "TON": 10271,
+ "Ġquant": 10272,
+ "Ġrub": 10273,
+ "ixels": 10274,
+ "Ġinstalled": 10275,
+ "Ġdump": 10276,
+ "Ġproperly": 10277,
+ "(List": 10278,
+ "Ġdecide": 10279,
+ "apply": 10280,
+ "Has": 10281,
+ "Ġkeeping": 10282,
+ "Ġcitizens": 10283,
+ "Ġjoint": 10284,
+ "pool": 10285,
+ "Socket": 10286,
+ "_op": 10287,
+ "Ġweapon": 10288,
+ "gnore": 10289,
+ "ĠExec": 10290,
+ "otten": 10291,
+ "ĠMS": 10292,
+ "Ġ(-": 10293,
+ "ĠReview": 10294,
+ "Ġexamples": 10295,
+ "Ġtight": 10296,
+ "!(": 10297,
+ "DP": 10298,
+ "ĠMessageBox": 10299,
+ "Ġphotograph": 10300,
+ "URI": 10301,
+ "ét": 10302,
+ "low": 10303,
+ "ĠGrand": 10304,
+ ".persistence": 10305,
+ "Ġmaintain": 10306,
+ "Ġnums": 10307,
+ "Ġzip": 10308,
+ "ials": 10309,
+ "ĠGets": 10310,
+ "peg": 10311,
+ "ĠBuffer": 10312,
+ "~~~~": 10313,
+ "rastructure": 10314,
+ "ĠPL": 10315,
+ "uen": 10316,
+ "obby": 10317,
+ "sizeof": 10318,
+ "Ġpic": 10319,
+ "Ġseed": 10320,
+ "Ġexperienced": 10321,
+ "Ġodd": 10322,
+ "Ġkick": 10323,
+ "Ġprocedure": 10324,
+ "avigator": 10325,
+ "-on": 10326,
+ ",j": 10327,
+ "ĠAlthough": 10328,
+ "ĠuserId": 10329,
+ "accept": 10330,
+ "Blue": 10331,
+ "IColor": 10332,
+ "layer": 10333,
+ "available": 10334,
+ "Ġends": 10335,
+ ".table": 10336,
+ "Ġdataset": 10337,
+ "bus": 10338,
+ "Ġexplain": 10339,
+ "(pro": 10340,
+ "ĠCommittee": 10341,
+ "Ġnoted": 10342,
+ "]:Ċ": 10343,
+ "Dim": 10344,
+ "stdio": 10345,
+ ".\",Ċ": 10346,
+ "_source": 10347,
+ "ĠWeek": 10348,
+ "ĠEdge": 10349,
+ "Ġoperating": 10350,
+ "Ġeste": 10351,
+ "ipl": 10352,
+ "agination": 10353,
+ "Ġproceed": 10354,
+ "Ġanimation": 10355,
+ ".Models": 10356,
+ "ĠWatch": 10357,
+ "iat": 10358,
+ "Ġoppon": 10359,
+ "/A": 10360,
+ "Report": 10361,
+ "Ġsounds": 10362,
+ "_buf": 10363,
+ "IELD": 10364,
+ "Ġbund": 10365,
+ "ĉget": 10366,
+ ".pr": 10367,
+ "(tmp": 10368,
+ "Ġkid": 10369,
+ ">ĊĊĊ": 10370,
+ "Ġyang": 10371,
+ "NotFound": 10372,
+ "ÑĨ": 10373,
+ "math": 10374,
+ "@gmail": 10375,
+ "ĠLIMIT": 10376,
+ "redients": 10377,
+ "Ġvent": 10378,
+ "avigate": 10379,
+ "Look": 10380,
+ "Ġreligious": 10381,
+ "Ġrand": 10382,
+ "rio": 10383,
+ "(GL": 10384,
+ "_ip": 10385,
+ "uan": 10386,
+ "iciency": 10387,
+ "ĠChange": 10388,
+ ">čĊčĊ": 10389,
+ "ĠEntity": 10390,
+ "Ġrencontre": 10391,
+ "ĠRet": 10392,
+ "plan": 10393,
+ "én": 10394,
+ "BOOL": 10395,
+ "uries": 10396,
+ "train": 10397,
+ "Definition": 10398,
+ "============": 10399,
+ "zz": 10400,
+ "Animation": 10401,
+ "ĠOK": 10402,
+ "_menu": 10403,
+ ".bl": 10404,
+ "_score": 10405,
+ "Ġacad": 10406,
+ "(System": 10407,
+ "Ġrefresh": 10408,
+ "'=>$": 10409,
+ ".Graphics": 10410,
+ "amento": 10411,
+ "pid": 10412,
+ "tc": 10413,
+ "Ġtips": 10414,
+ "Ġhomes": 10415,
+ "Ġfuel": 10416,
+ "âĸ": 10417,
+ "_helper": 10418,
+ "ĠĠčĊ": 10419,
+ "ĠRoom": 10420,
+ ".Close": 10421,
+ "_attr": 10422,
+ "ĠMount": 10423,
+ "ĠEv": 10424,
+ "arser": 10425,
+ "_top": 10426,
+ "eah": 10427,
+ "ĠDelete": 10428,
+ "ãĢį": 10429,
+ "uke": 10430,
+ "Ġusage": 10431,
+ "aria": 10432,
+ "_dev": 10433,
+ "Ġtexture": 10434,
+ "Ġconversation": 10435,
+ "eper": 10436,
+ "Bean": 10437,
+ "done": 10438,
+ "nonatomic": 10439,
+ "ĠSecond": 10440,
+ "Ġshooting": 10441,
+ "_pre": 10442,
+ "Components": 10443,
+ "Ġ]ĊĊ": 10444,
+ "__,": 10445,
+ "stitution": 10446,
+ ".Char": 10447,
+ ">();ĊĊ": 10448,
+ "Ġpresented": 10449,
+ "Ġwa": 10450,
+ "oker": 10451,
+ "-ĊĊ": 10452,
+ "iner": 10453,
+ "Ġbecoming": 10454,
+ "Ġincident": 10455,
+ "Att": 10456,
+ "Ġrevealed": 10457,
+ "forc": 10458,
+ "Ġboot": 10459,
+ ".page": 10460,
+ "Enumerator": 10461,
+ "_->": 10462,
+ "Photo": 10463,
+ "Ġspring": 10464,
+ ".\",": 10465,
+ "ĠDictionary": 10466,
+ "BJECT": 10467,
+ "Ġlocations": 10468,
+ "Ġsamples": 10469,
+ "InputStream": 10470,
+ "ĠBrown": 10471,
+ "Ġstats": 10472,
+ "quality": 10473,
+ "Ñħ": 10474,
+ "-dis": 10475,
+ "Ġhelping": 10476,
+ "Ġped": 10477,
+ "(se": 10478,
+ "ĠWho": 10479,
+ "alian": 10480,
+ "internal": 10481,
+ "Ġft": 10482,
+ ">().": 10483,
+ "->{": 10484,
+ "Ġmine": 10485,
+ "Ġsector": 10486,
+ "Ġgro": 10487,
+ "Ġopportunities": 10488,
+ "Ġü": 10489,
+ "Ġmp": 10490,
+ "Ġalleged": 10491,
+ "Ġdoubt": 10492,
+ "Mouse": 10493,
+ "About": 10494,
+ "_part": 10495,
+ "Ġchair": 10496,
+ "Ġstopped": 10497,
+ "loop": 10498,
+ "entities": 10499,
+ "Ġapps": 10500,
+ "ansion": 10501,
+ "Ġmental": 10502,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 10503,
+ "FR": 10504,
+ "Ġdefend": 10505,
+ "care": 10506,
+ "Ġideal": 10507,
+ "/api": 10508,
+ "urface": 10509,
+ "Ġele": 10510,
+ "ulator": 10511,
+ "ĠRights": 10512,
+ "anguages": 10513,
+ "Ġfunds": 10514,
+ "Ġadapt": 10515,
+ "Attributes": 10516,
+ "Ġdeploy": 10517,
+ "opts": 10518,
+ "Ġvalidation": 10519,
+ "Ġconcerns": 10520,
+ "uce": 10521,
+ ".num": 10522,
+ "ulture": 10523,
+ "ila": 10524,
+ "Ġcup": 10525,
+ "Ġpure": 10526,
+ ".Fore": 10527,
+ "ĠHashMap": 10528,
+ ".valueOf": 10529,
+ "asm": 10530,
+ "MO": 10531,
+ "Ġcs": 10532,
+ "Ġstores": 10533,
+ "Ġ************************************************************************": 10534,
+ "Ġcommunication": 10535,
+ "mem": 10536,
+ ".EventHandler": 10537,
+ ".Status": 10538,
+ "_right": 10539,
+ ".setOn": 10540,
+ "Sheet": 10541,
+ "Ġidentify": 10542,
+ "enerated": 10543,
+ "ordered": 10544,
+ "Ġ\"[": 10545,
+ "Ġswe": 10546,
+ "Condition": 10547,
+ "ĠAccording": 10548,
+ "Ġprepare": 10549,
+ "Ġrob": 10550,
+ "Pool": 10551,
+ "Ġsport": 10552,
+ "rv": 10553,
+ "ĠRouter": 10554,
+ "Ġalternative": 10555,
+ "([]": 10556,
+ "ĠChicago": 10557,
+ "ipher": 10558,
+ "ische": 10559,
+ "ĠDirector": 10560,
+ "kl": 10561,
+ "ĠWil": 10562,
+ "keys": 10563,
+ "Ġmysql": 10564,
+ "Ġwelcome": 10565,
+ "king": 10566,
+ "ĠManager": 10567,
+ "Ġcaught": 10568,
+ ")}Ċ": 10569,
+ "Score": 10570,
+ "_PR": 10571,
+ "Ġsurvey": 10572,
+ "hab": 10573,
+ "Headers": 10574,
+ "ADER": 10575,
+ "Ġdecor": 10576,
+ "Ġturns": 10577,
+ "Ġradius": 10578,
+ "errupt": 10579,
+ "Cor": 10580,
+ "Ġmel": 10581,
+ "Ġintr": 10582,
+ "(q": 10583,
+ "ĠAC": 10584,
+ "amos": 10585,
+ "MAX": 10586,
+ "ĠGrid": 10587,
+ "ĠJesus": 10588,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 10589,
+ ".DE": 10590,
+ "Ġts": 10591,
+ "Ġlinked": 10592,
+ "free": 10593,
+ "ĠQt": 10594,
+ "Ġ/**čĊ": 10595,
+ "Ġfaster": 10596,
+ "ctr": 10597,
+ "_J": 10598,
+ "DT": 10599,
+ ".Check": 10600,
+ "Ġcombination": 10601,
+ "Ġintended": 10602,
+ "-the": 10603,
+ "-type": 10604,
+ "ectors": 10605,
+ "ami": 10606,
+ "uting": 10607,
+ "Ġuma": 10608,
+ "XML": 10609,
+ "UCT": 10610,
+ "Ap": 10611,
+ "ĠRandom": 10612,
+ "Ġran": 10613,
+ ".sort": 10614,
+ "Ġsorted": 10615,
+ ".Un": 10616,
+ "_PER": 10617,
+ "itory": 10618,
+ "Ġpriority": 10619,
+ "ĠGal": 10620,
+ "ĠOld": 10621,
+ "hot": 10622,
+ "ĠDisplay": 10623,
+ "(sub": 10624,
+ "_TH": 10625,
+ "_Y": 10626,
+ "ĠCare": 10627,
+ "loading": 10628,
+ "Kind": 10629,
+ "_handle": 10630,
+ ",,": 10631,
+ "rase": 10632,
+ "_replace": 10633,
+ ".addEventListener": 10634,
+ "ĠRT": 10635,
+ "Ġentered": 10636,
+ "gers": 10637,
+ "Ġich": 10638,
+ "(start": 10639,
+ "/app": 10640,
+ "Ġbrother": 10641,
+ "Memory": 10642,
+ "Outlet": 10643,
+ "Ġutf": 10644,
+ "prec": 10645,
+ "Ġnavigation": 10646,
+ "ORK": 10647,
+ "Ġdst": 10648,
+ "Detail": 10649,
+ "Ġaudience": 10650,
+ "Ġdur": 10651,
+ "Ġcluster": 10652,
+ "unched": 10653,
+ "Ġ],": 10654,
+ "Ġcomfortable": 10655,
+ ".values": 10656,
+ "ĠTotal": 10657,
+ "Ġsnap": 10658,
+ "Ġstandards": 10659,
+ "Ġperformed": 10660,
+ "hand": 10661,
+ "(\"@": 10662,
+ "åŃ": 10663,
+ "Ġphil": 10664,
+ "ibr": 10665,
+ "trim": 10666,
+ "Ġforget": 10667,
+ "Ġdoctor": 10668,
+ ".TextBox": 10669,
+ "icons": 10670,
+ ",s": 10671,
+ "ĠOp": 10672,
+ "Sm": 10673,
+ "Stop": 10674,
+ "ĉList": 10675,
+ "ĉu": 10676,
+ "Comment": 10677,
+ "_VERSION": 10678,
+ ".Xtra": 10679,
+ "Person": 10680,
+ "rb": 10681,
+ "LOB": 10682,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ": 10683,
+ "ĠCentral": 10684,
+ "ICK": 10685,
+ "raq": 10686,
+ "Ġputting": 10687,
+ "Ġmd": 10688,
+ "ĠLove": 10689,
+ "Program": 10690,
+ "Border": 10691,
+ "oor": 10692,
+ "Ġallowing": 10693,
+ "after": 10694,
+ "Ġentries": 10695,
+ "ĠMaybe": 10696,
+ "]).": 10697,
+ "ĠShort": 10698,
+ ")\\": 10699,
+ ".now": 10700,
+ "friend": 10701,
+ "Ġprefer": 10702,
+ "ĠGPIO": 10703,
+ "osis": 10704,
+ "ĠGameObject": 10705,
+ "Ġskip": 10706,
+ "Ġcompetition": 10707,
+ "_match": 10708,
+ "lications": 10709,
+ "_CONT": 10710,
+ ".groupBox": 10711,
+ "Ġals": 10712,
+ "\"We": 10713,
+ "_eq": 10714,
+ "lan": 10715,
+ "_search": 10716,
+ "ĠMusic": 10717,
+ "asis": 10718,
+ "Ġbind": 10719,
+ "ĠIsland": 10720,
+ "rum": 10721,
+ "(E": 10722,
+ "Ġseat": 10723,
+ "Video": 10724,
+ "Ġack": 10725,
+ "reek": 10726,
+ "={()": 10727,
+ "Ġrating": 10728,
+ "Ġrestaurant": 10729,
+ "DEX": 10730,
+ "(buf": 10731,
+ "pping": 10732,
+ "uality": 10733,
+ "Ġleague": 10734,
+ "Ġfocused": 10735,
+ "apon": 10736,
+ "$data": 10737,
+ "CLUD": 10738,
+ "CLUDING": 10739,
+ "Ġabsolute": 10740,
+ "(query": 10741,
+ "Ġtells": 10742,
+ "Ang": 10743,
+ "Ġcommunities": 10744,
+ "Ġhonest": 10745,
+ "oking": 10746,
+ "Ġapart": 10747,
+ "arity": 10748,
+ "/$": 10749,
+ "_module": 10750,
+ "ĠEnc": 10751,
+ ".an": 10752,
+ ".Config": 10753,
+ "Cre": 10754,
+ "Ġshock": 10755,
+ "ĠArab": 10756,
+ "IENT": 10757,
+ "/re": 10758,
+ "Ġretrie": 10759,
+ "ycler": 10760,
+ "isa": 10761,
+ "ĠOrgan": 10762,
+ ".graph": 10763,
+ "Ġí": 10764,
+ "ĠBAS": 10765,
+ "Enum": 10766,
+ "Ġpossibly": 10767,
+ "ÑĢаÐ": 10768,
+ "ĠJapanese": 10769,
+ "Ġcraft": 10770,
+ "ĠPlace": 10771,
+ "Ġtalent": 10772,
+ "Ġfunding": 10773,
+ "Ġconfirmed": 10774,
+ "Ġcycle": 10775,
+ "/x": 10776,
+ "GE": 10777,
+ "Ġhearing": 10778,
+ "Ġplants": 10779,
+ "Ġmouth": 10780,
+ "pages": 10781,
+ "oria": 10782,
+ "ĠRemove": 10783,
+ "_total": 10784,
+ "Ġod": 10785,
+ "ollapse": 10786,
+ "door": 10787,
+ "Ġbought": 10788,
+ "Ġaddr": 10789,
+ "ARCH": 10790,
+ "_dim": 10791,
+ "dden": 10792,
+ "Ġdecades": 10793,
+ "REQUEST": 10794,
+ "Ġversions": 10795,
+ "fire": 10796,
+ "Ġmoves": 10797,
+ "fb": 10798,
+ "Ġcoffee": 10799,
+ ".connect": 10800,
+ "ĠRow": 10801,
+ "Ġschema": 10802,
+ "Scope": 10803,
+ "-Type": 10804,
+ "Ġfighting": 10805,
+ "Ġretail": 10806,
+ "Ġmodified": 10807,
+ "TF": 10808,
+ "Files": 10809,
+ "nie": 10810,
+ "_command": 10811,
+ "stone": 10812,
+ "ĠÑĤ": 10813,
+ "_thread": 10814,
+ "Ġbond": 10815,
+ "ĠDevelopment": 10816,
+ "Ġpt": 10817,
+ "FORM": 10818,
+ "plet": 10819,
+ "Ġidentified": 10820,
+ "cpp": 10821,
+ "Ġcoding": 10822,
+ "oked": 10823,
+ "ĠMaster": 10824,
+ "IDTH": 10825,
+ "Ġresidents": 10826,
+ "redit": 10827,
+ "ĠPhoto": 10828,
+ "=-": 10829,
+ "unte": 10830,
+ "ateur": 10831,
+ "_STATE": 10832,
+ "ĠSing": 10833,
+ "Ġsheet": 10834,
+ ".val": 10835,
+ "orse": 10836,
+ "Ġhers": 10837,
+ "Ġdetermined": 10838,
+ "Common": 10839,
+ "Ġwed": 10840,
+ "_queue": 10841,
+ "PH": 10842,
+ "ĠAtl": 10843,
+ "cred": 10844,
+ "/LICENSE": 10845,
+ "Ġmes": 10846,
+ "Ġadvanced": 10847,
+ ".java": 10848,
+ ".Sh": 10849,
+ "Go": 10850,
+ "kill": 10851,
+ "fp": 10852,
+ "_settings": 10853,
+ "Ġpal": 10854,
+ "Ġtruck": 10855,
+ "Ġcombined": 10856,
+ "Ġ\"${": 10857,
+ "ĠCorpor": 10858,
+ "Ġjoined": 10859,
+ "ĠJose": 10860,
+ "ĠCup": 10861,
+ "uns": 10862,
+ "estival": 10863,
+ "levision": 10864,
+ "Ġbroken": 10865,
+ "Ġmarriage": 10866,
+ "ĠWestern": 10867,
+ "Ġrepresents": 10868,
+ "ĠTitle": 10869,
+ "Ġss": 10870,
+ ".Ass": 10871,
+ "ongoose": 10872,
+ "iento": 10873,
+ "<>();Ċ": 10874,
+ "Ġabsolutely": 10875,
+ "Ġsmooth": 10876,
+ "TERN": 10877,
+ "ĠUnless": 10878,
+ "Word": 10879,
+ "Ġmerge": 10880,
+ "igan": 10881,
+ "ĠVol": 10882,
+ "Ġnn": 10883,
+ ".getId": 10884,
+ "Ġз": 10885,
+ "Ġsexy": 10886,
+ "Ġseeking": 10887,
+ "Single": 10888,
+ ".this": 10889,
+ "Ġkom": 10890,
+ "bound": 10891,
+ ";\"": 10892,
+ "ĠfontSize": 10893,
+ "_df": 10894,
+ "Ġinjury": 10895,
+ "(H": 10896,
+ "Ġissued": 10897,
+ "_END": 10898,
+ ":self": 10899,
+ "Ġpatch": 10900,
+ "Ġleaves": 10901,
+ "Ġadopt": 10902,
+ "FileName": 10903,
+ "ãĢIJ": 10904,
+ "Ġexecutive": 10905,
+ "ĠByte": 10906,
+ "]))Ċ": 10907,
+ "Ġnu": 10908,
+ "outing": 10909,
+ "cluding": 10910,
+ "-R": 10911,
+ ".options": 10912,
+ "Ġsubstant": 10913,
+ "avax": 10914,
+ "ĠBUT": 10915,
+ "Ġtechnical": 10916,
+ "Ġtwice": 10917,
+ "Ġmás": 10918,
+ "Ġunivers": 10919,
+ "yr": 10920,
+ "Ġdrag": 10921,
+ "ĠDC": 10922,
+ "Ġsed": 10923,
+ "Ġbot": 10924,
+ "ĠPal": 10925,
+ "ĠHall": 10926,
+ "forcement": 10927,
+ "Ġauch": 10928,
+ ".mod": 10929,
+ "notation": 10930,
+ "_files": 10931,
+ ".line": 10932,
+ "_flag": 10933,
+ "[name": 10934,
+ "Ġresolution": 10935,
+ "Ġbott": 10936,
+ "(\"[": 10937,
+ "ende": 10938,
+ "(arr": 10939,
+ "Free": 10940,
+ "(@\"": 10941,
+ "ĠDistrict": 10942,
+ "PEC": 10943,
+ ":-": 10944,
+ "Picker": 10945,
+ "ĠJo": 10946,
+ "ĠĠĠĠĠĊ": 10947,
+ "ĠRiver": 10948,
+ "_rows": 10949,
+ "Ġhelpful": 10950,
+ "Ġmassive": 10951,
+ "---Ċ": 10952,
+ "Ġmeasures": 10953,
+ "ĠRuntime": 10954,
+ "Ġworry": 10955,
+ "ĠSpec": 10956,
+ "ĉD": 10957,
+ "ãĢij": 10958,
+ "Ġ){Ċ": 10959,
+ "Ġworse": 10960,
+ "(filename": 10961,
+ "Ġlay": 10962,
+ "Ġmagic": 10963,
+ "ĠTheir": 10964,
+ "oul": 10965,
+ "stroy": 10966,
+ "ĠWhere": 10967,
+ "Ġsudden": 10968,
+ "Ġdefe": 10969,
+ "Ġbinding": 10970,
+ "Ġflight": 10971,
+ "ĠOnInit": 10972,
+ "ĠWomen": 10973,
+ "ĠPolicy": 10974,
+ "Ġdrugs": 10975,
+ "ishing": 10976,
+ "('../": 10977,
+ "ĠMel": 10978,
+ "peat": 10979,
+ "tor": 10980,
+ "Ġproposed": 10981,
+ "Ġstated": 10982,
+ "_RES": 10983,
+ "Ġeast": 10984,
+ "ĠCONDITION": 10985,
+ "_desc": 10986,
+ "Ġwinning": 10987,
+ "folio": 10988,
+ "Mapper": 10989,
+ "ĠPan": 10990,
+ "ĠAnge": 10991,
+ ".servlet": 10992,
+ "Ġcopies": 10993,
+ "LM": 10994,
+ "Ġvm": 10995,
+ "åį": 10996,
+ "Ġdictionary": 10997,
+ "Seg": 10998,
+ "elines": 10999,
+ "ĠSend": 11000,
+ "Ġiron": 11001,
+ "ĠFort": 11002,
+ ".domain": 11003,
+ "Ġdebate": 11004,
+ "NotNull": 11005,
+ "eq": 11006,
+ "acher": 11007,
+ "lf": 11008,
+ "ĉfmt": 11009,
+ "Ġlawy": 11010,
+ "ÄŁ": 11011,
+ "ĠMen": 11012,
+ "Ġtrim": 11013,
+ "(NULL": 11014,
+ "Ġ!!": 11015,
+ "Ġpad": 11016,
+ "Ġfollows": 11017,
+ "\"][\"": 11018,
+ "requ": 11019,
+ "ĠEp": 11020,
+ ".github": 11021,
+ "(img": 11022,
+ "eto": 11023,
+ "('\\": 11024,
+ "Services": 11025,
+ "umbnail": 11026,
+ "_main": 11027,
+ "pleted": 11028,
+ "fortunately": 11029,
+ "Ġwindows": 11030,
+ "Ġplane": 11031,
+ "ĠConnection": 11032,
+ ".local": 11033,
+ "uard": 11034,
+ "}\\": 11035,
+ "==\"": 11036,
+ "andon": 11037,
+ "ĠRoy": 11038,
+ "west": 11039,
+ "iginal": 11040,
+ "emies": 11041,
+ "itz": 11042,
+ "'):Ċ": 11043,
+ "ĠPeter": 11044,
+ "Ġtough": 11045,
+ "Ġreduced": 11046,
+ "Ġcalculate": 11047,
+ "Ġrapid": 11048,
+ "customer": 11049,
+ "Ġefficient": 11050,
+ "Ġmedium": 11051,
+ "Ġfell": 11052,
+ ".ref": 11053,
+ "ĠCas": 11054,
+ "Ġfeedback": 11055,
+ "Speed": 11056,
+ "(output": 11057,
+ "aje": 11058,
+ "Ġcategories": 11059,
+ "Ġfee": 11060,
+ "};": 11061,
+ "Ġdeleted": 11062,
+ "reh": 11063,
+ "Ġproof": 11064,
+ "Desc": 11065,
+ "Build": 11066,
+ "Ġsides": 11067,
+ ".ArrayList": 11068,
+ "-%": 11069,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 11070,
+ "ر": 11071,
+ ".match": 11072,
+ "ли": 11073,
+ "Ġfeels": 11074,
+ "Ġachieve": 11075,
+ "Ġclim": 11076,
+ "_ON": 11077,
+ "ĠCD": 11078,
+ "Ġteacher": 11079,
+ "_current": 11080,
+ "bn": 11081,
+ "_PL": 11082,
+ "isting": 11083,
+ "Enable": 11084,
+ "GEN": 11085,
+ "Ġtv": 11086,
+ "Ġsock": 11087,
+ "Ġplays": 11088,
+ "Ġdiscount": 11089,
+ "ĠKE": 11090,
+ "ĠDebug": 11091,
+ "Fore": 11092,
+ "ĠIraq": 11093,
+ "Ġappearance": 11094,
+ "Mon": 11095,
+ "Ġstyled": 11096,
+ "ĠHuman": 11097,
+ "iot": 11098,
+ "ĠHistory": 11099,
+ "Ġsac": 11100,
+ "ĠCollection": 11101,
+ "Ġrecommended": 11102,
+ ".Selected": 11103,
+ "Ġorganizations": 11104,
+ "Ġdiscovered": 11105,
+ "cohol": 11106,
+ "adas": 11107,
+ "ĠThomas": 11108,
+ "May": 11109,
+ "Ġconserv": 11110,
+ "Ġdomin": 11111,
+ "ĠFollow": 11112,
+ "ĠSection": 11113,
+ "ĠThanks": 11114,
+ "Username": 11115,
+ "Ġrecipe": 11116,
+ "Ġwonderful": 11117,
+ ".sleep": 11118,
+ "_if": 11119,
+ "ĉĊĉĊ": 11120,
+ "orno": 11121,
+ "Ġru": 11122,
+ "_target": 11123,
+ ".\"\"": 11124,
+ "à¦": 11125,
+ "EventArgs": 11126,
+ "Ġinputs": 11127,
+ "Ġfif": 11128,
+ "Ġvision": 11129,
+ "cy": 11130,
+ "ĠSeries": 11131,
+ ")(((": 11132,
+ "Ġtrading": 11133,
+ "Ġmarker": 11134,
+ "Begin": 11135,
+ "Ġtypically": 11136,
+ "Ġcauses": 11137,
+ "dropdown": 11138,
+ "_DEBUG": 11139,
+ "Ġdetect": 11140,
+ "country": 11141,
+ "!\");Ċ": 11142,
+ "ĉR": 11143,
+ "appy": 11144,
+ "Ġcref": 11145,
+ "('<": 11146,
+ "\"=>": 11147,
+ "ĠLE": 11148,
+ "reader": 11149,
+ "Ġadministr": 11150,
+ "õ": 11151,
+ "ucket": 11152,
+ "Ġfashion": 11153,
+ ".char": 11154,
+ "izar": 11155,
+ "Ġdisable": 11156,
+ "Ġsuc": 11157,
+ "ĠLive": 11158,
+ "issue": 11159,
+ "Ġmetadata": 11160,
+ "flags": 11161,
+ "ĠðŁ": 11162,
+ "Ġcommitted": 11163,
+ "Ġva": 11164,
+ "Ġrough": 11165,
+ "Ġ'''Ċ": 11166,
+ "Ġhighlight": 11167,
+ "_vars": 11168,
+ "VO": 11169,
+ "Ġencoding": 11170,
+ "-Z": 11171,
+ "_sign": 11172,
+ "$(\"#": 11173,
+ "Ġrain": 11174,
+ "reatest": 11175,
+ "ĠEND": 11176,
+ "Selection": 11177,
+ "Ġcandidates": 11178,
+ "Ġsav": 11179,
+ ".Empty": 11180,
+ "Ġdecisions": 11181,
+ "Ġcollabor": 11182,
+ "ridge": 11183,
+ "feed": 11184,
+ "ression": 11185,
+ "Ġpersons": 11186,
+ "VM": 11187,
+ "ega": 11188,
+ "_BIT": 11189,
+ "According": 11190,
+ "acked": 11191,
+ "Ġdollars": 11192,
+ "_loss": 11193,
+ "ĠCost": 11194,
+ "}\"Ċ": 11195,
+ "Notification": 11196,
+ "Ġprostit": 11197,
+ "Ġauthority": 11198,
+ ".rec": 11199,
+ "Ġspokes": 11200,
+ "ĠToday": 11201,
+ "istant": 11202,
+ "ĠHead": 11203,
+ "âĢĿ.": 11204,
+ "ertainment": 11205,
+ "cean": 11206,
+ "culate": 11207,
+ "Ġven": 11208,
+ "However": 11209,
+ "_arr": 11210,
+ "Ġtokens": 11211,
+ "Graph": 11212,
+ "ĠJud": 11213,
+ "ĠVirgin": 11214,
+ "ĠSerial": 11215,
+ "unning": 11216,
+ "Mutable": 11217,
+ "agers": 11218,
+ ".csv": 11219,
+ "Ġdeveloping": 11220,
+ "Ġinstructions": 11221,
+ "Ġpromise": 11222,
+ "Ġrequested": 11223,
+ "_encode": 11224,
+ "/\"": 11225,
+ "ĠIcon": 11226,
+ "uilt": 11227,
+ "-day": 11228,
+ "Ġintelligence": 11229,
+ ".IS": 11230,
+ "ĠObservable": 11231,
+ "ĠHard": 11232,
+ "Bool": 11233,
+ "idential": 11234,
+ ".Anchor": 11235,
+ "Ġselling": 11236,
+ "CI": 11237,
+ "AGES": 11238,
+ "tle": 11239,
+ "bur": 11240,
+ "UFFER": 11241,
+ "RY": 11242,
+ "Ġbigger": 11243,
+ "Ġrat": 11244,
+ "Ġfamous": 11245,
+ "Ġtypename": 11246,
+ "Ġexplained": 11247,
+ "}}Ċ": 11248,
+ "Ġnuclear": 11249,
+ "-N": 11250,
+ "Ġcrisis": 11251,
+ "ĠEnter": 11252,
+ "Ġanswers": 11253,
+ "/${": 11254,
+ "/pl": 11255,
+ "Ġsequ": 11256,
+ "_next": 11257,
+ "mask": 11258,
+ "Ġstanding": 11259,
+ "Ġplenty": 11260,
+ "ĠCross": 11261,
+ "ĉret": 11262,
+ "dro": 11263,
+ "ĠCast": 11264,
+ "=true": 11265,
+ "ĠChris": 11266,
+ "icio": 11267,
+ "ĠMike": 11268,
+ "Decimal": 11269,
+ "addComponent": 11270,
+ "Len": 11271,
+ "Ġcock": 11272,
+ "Ġ#{": 11273,
+ "URN": 11274,
+ "": 11403,
+ "Ġ*=": 11404,
+ "ĠPS": 11405,
+ "Ġdangerous": 11406,
+ "[p": 11407,
+ "OME": 11408,
+ "Other": 11409,
+ "ĠStringBuilder": 11410,
+ "Points": 11411,
+ "heading": 11412,
+ "Ġcurrency": 11413,
+ "Ġpercentage": 11414,
+ "_API": 11415,
+ "Ġclassic": 11416,
+ "thead": 11417,
+ "ĠMO": 11418,
+ "FE": 11419,
+ "Idx": 11420,
+ "await": 11421,
+ "Ġè": 11422,
+ "Ġaccident": 11423,
+ "Ġvariant": 11424,
+ "Ġmyst": 11425,
+ "ĠLand": 11426,
+ "ĠBre": 11427,
+ "Ġharm": 11428,
+ "ĠAcc": 11429,
+ "Ġcharged": 11430,
+ "iones": 11431,
+ "Visibility": 11432,
+ "arry": 11433,
+ "ĠLanguage": 11434,
+ "Ġwalking": 11435,
+ "\".ĊĊ": 11436,
+ "ifer": 11437,
+ "Ġleadership": 11438,
+ ".From": 11439,
+ "ynam": 11440,
+ "Ġtimestamp": 11441,
+ "ipt": 11442,
+ "ĠHas": 11443,
+ "REFER": 11444,
+ "ĠIts": 11445,
+ "Ġlistener": 11446,
+ "UTE": 11447,
+ "_description": 11448,
+ "Ġexperiences": 11449,
+ "Ġcreates": 11450,
+ "RS": 11451,
+ "cart": 11452,
+ "black": 11453,
+ "Ġchoices": 11454,
+ "war": 11455,
+ "Ġ'''": 11456,
+ "Ġordered": 11457,
+ "Ġevening": 11458,
+ "Ġpil": 11459,
+ "Ġtun": 11460,
+ "ĠBad": 11461,
+ "(app": 11462,
+ "random": 11463,
+ "Ġexplicit": 11464,
+ "Ġarrived": 11465,
+ "Ġfly": 11466,
+ "Ġeconom": 11467,
+ "-mail": 11468,
+ "Ġlists": 11469,
+ "Ġarchitect": 11470,
+ "ĠPay": 11471,
+ "Ġds": 11472,
+ "ĠSol": 11473,
+ "Ġvehicles": 11474,
+ "Hz": 11475,
+ "-com": 11476,
+ "Ġking": 11477,
+ "_equal": 11478,
+ "ĠHelp": 11479,
+ "Ġabuse": 11480,
+ "--;Ċ": 11481,
+ "Ġextr": 11482,
+ "Ġchemical": 11483,
+ "ä¿": 11484,
+ "Ġorient": 11485,
+ "Ġbreath": 11486,
+ "ĠSpace": 11487,
+ "(element": 11488,
+ "wait": 11489,
+ "DED": 11490,
+ "igma": 11491,
+ "Ġentr": 11492,
+ "Ġsob": 11493,
+ "-name": 11494,
+ "Ġaffected": 11495,
+ "ika": 11496,
+ "Ġcoal": 11497,
+ "_work": 11498,
+ "Ġhundreds": 11499,
+ "Ġpolitics": 11500,
+ "subject": 11501,
+ "Ġconsumer": 11502,
+ "ANGE": 11503,
+ "Ġrepeated": 11504,
+ "Send": 11505,
+ "Ġ#[": 11506,
+ "Ġprotocol": 11507,
+ "Ġleads": 11508,
+ "useum": 11509,
+ "Every": 11510,
+ "Import": 11511,
+ "(count": 11512,
+ "Ġchallenges": 11513,
+ "Ġnovel": 11514,
+ "Ġdepart": 11515,
+ "bits": 11516,
+ ".Current": 11517,
+ "Ġ`${": 11518,
+ "oting": 11519,
+ "(\\": 11520,
+ "Ġcreative": 11521,
+ "Ġbuff": 11522,
+ "Ġintroduced": 11523,
+ "usic": 11524,
+ "modules": 11525,
+ "Are": 11526,
+ "-doc": 11527,
+ "language": 11528,
+ "_cache": 11529,
+ "Ġtod": 11530,
+ "?>": 11531,
+ "omething": 11532,
+ "Ġhun": 11533,
+ "åº": 11534,
+ "aters": 11535,
+ "Intent": 11536,
+ "Ġimplemented": 11537,
+ "ĠCase": 11538,
+ "Children": 11539,
+ "Ġnotification": 11540,
+ "Renderer": 11541,
+ "Wrapper": 11542,
+ "Objects": 11543,
+ "tl": 11544,
+ ".Contains": 11545,
+ "Plugin": 11546,
+ ".row": 11547,
+ "Ġforg": 11548,
+ "Ġpermit": 11549,
+ "Ġtargets": 11550,
+ "ĠIF": 11551,
+ "Ġtip": 11552,
+ "sex": 11553,
+ "Ġsupports": 11554,
+ "Ġfold": 11555,
+ "photo": 11556,
+ "},čĊ": 11557,
+ "Ġgoogle": 11558,
+ "$('#": 11559,
+ "Ġsharing": 11560,
+ "Ġgoods": 11561,
+ "vs": 11562,
+ "ĠDan": 11563,
+ "Rate": 11564,
+ "ĠMartin": 11565,
+ "Ġmanner": 11566,
+ "lie": 11567,
+ ".The": 11568,
+ "Internal": 11569,
+ "ĠCONTR": 11570,
+ "Mock": 11571,
+ "RIGHT": 11572,
+ "Ġ'{": 11573,
+ "Ġcontrols": 11574,
+ "Mat": 11575,
+ "Ġmand": 11576,
+ "Ġextended": 11577,
+ "Ok": 11578,
+ "Ġembed": 11579,
+ "Ġplanet": 11580,
+ "ĠNon": 11581,
+ "-ch": 11582,
+ ")\",": 11583,
+ "epar": 11584,
+ "Ġbelieved": 11585,
+ "ĠEnvironment": 11586,
+ "ĠFriend": 11587,
+ "-res": 11588,
+ "Ġhandling": 11589,
+ "nic": 11590,
+ "-level": 11591,
+ "scri": 11592,
+ "Xml": 11593,
+ "BE": 11594,
+ "ungen": 11595,
+ "Ġalter": 11596,
+ "[idx": 11597,
+ "Pop": 11598,
+ "cam": 11599,
+ "Ġ(((": 11600,
+ "Ġshipping": 11601,
+ "Ġbattery": 11602,
+ "iddleware": 11603,
+ "MC": 11604,
+ "Ġimpl": 11605,
+ "otation": 11606,
+ "ĠLab": 11607,
+ "