Update README.md

README.md

@@ -53,4 +53,510 @@ print(f"Loaded {len(data['qa_pairs'])} QA pairs!")
    
 ## Credits
    
-Created by Lyon28
+Created by Lyon28
+
+
+<!-- 
+     HEADER SECTION 
+      -->
+
+<div align="center">
+  <picture>
+    <source 
+      media="(prefers-color-scheme: dark)" 
+      srcset="https://huggingface.co/Lyon28/caca-10m/resolve/main/logo-dark.png"
+      type="image/png"
+    />
+    <source 
+      media="(prefers-color-scheme: light)" 
+      srcset="https://huggingface.co/Lyon28/caca-10m/resolve/main/logo-light.png"
+      type="image/png"
+    />
+    <img 
+      src="https://huggingface.co/Lyon28/caca-10m/resolve/main/logo.png" 
+      alt="Caca Transformers Logo"
+      title="Caca - Modern Transformer Architecture"
+      width="60%"
+      height="auto"
+      loading="lazy"
+    />
+  </picture>
+</div>
+
+<!-- 
+     BADGES SECTION 
+      -->
+
+<div align="center">
+  
+  <!-- Social Links -->
+  <p>
+    <a href="https://huggingface.co/Lyon28" target="_blank" rel="noopener noreferrer">
+      <img 
+        src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Lyon28-ffc107?color=ffc107&logoColor=white" 
+        alt="Hugging Face Profile" 
+        title="Visit Hugging Face Profile"
+      />
+    </a>
+  </p>
+  
+  <!-- License Badge -->
+  <p>
+    <a 
+      href="https://github.com/Lyon-28/caca-transformers?tab=Apache-2.0-1-ov-file" 
+      target="_blank" 
+      rel="noopener noreferrer"
+      title="Apache 2.0 License"
+    >
+      <img 
+        src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" 
+        alt="License: Apache 2.0"
+        height="20"
+      />
+    </a>
+  </p>
+  
+  <!-- PyPI Badge -->
+  <p>
+    <a href="https://pypi.org/project/caca-transformers/" target="_blank" rel="noopener noreferrer">
+      <img 
+        src="https://img.shields.io/pypi/v/caca-transformers?color=blue&label=PyPI&logo=pypi&logoColor=white" 
+        alt="PyPI Version"
+        title="View on PyPI"
+      />
+    </a>
+  </p>
+  
+  <!-- GitHub Stars -->
+  <p>
+    <a href="https://github.com/Lyon-28/caca-transformers" target="_blank" rel="noopener noreferrer">
+      <img 
+        src="https://img.shields.io/github/stars/Lyon-28/caca-transformers?style=social&label=Star&maxAge=2592000" 
+        alt="GitHub Stars"
+        title="Star on GitHub"
+      />
+    </a>
+  </p>
+  
+  <!-- Description -->
+  <p>
+    <strong>Arsitektur Transformer Modern dengan GQA, RoPE, SwiGLU &amp; Flash Attention</strong>
+  </p>
+  
+</div>
+
+<!-- Horizontal Rule -->
+<hr/>
+
+<!-- 
+     WARNING/ALERT SECTION 
+      -->
+
+<blockquote>
+  <p>
+    <strong>🔬 RESEARCH PROJECT</strong>
+  </p>
+  <p>
+    <strong>⚠️ PERHATIAN: MODEL UNTRAINED</strong>
+  </p>
+  <p>
+    Model ini memiliki bobot random dan memerlukan pretraining sebelum digunakan. 
+    Tidak bisa langsung digunakan untuk inference!<br/>
+    Model ini adalah eksperimen arsitektur dan belum divalidasi untuk production use.
+  </p>
+</blockquote>
+
+<!-- 
+     MAIN TITLE 
+      -->
+
+<h1 align="center">
+  🐣 CACA-10M - TINY
+</h1>
+
+<p align="center">
+  <strong>🔢 10,485,760 Parameters (0.01B)</strong>
+</p>
+
+<p align="center">
+  <strong>💾 ~0.02GB (FP16) / ~0.04GB (FP32)</strong>
+</p>
+
+<p align="center">
+  <strong>📏 8,192 Context Length</strong>
+</p>
+
+<p align="center">
+  <strong>🎯 Use Case:</strong> Eksperimen cepat, edge devices, pembelajaran
+</p>
+
+<p align="center">
+  <strong>🖥️ Recommended GPU:</strong> GTX 1060 6GB or better
+</p>
+
+<!-- 
+     FEATURES SECTION 
+      -->
+
+<h2>🎯 Fitur Utama</h2>
+
+<p>
+  Arsitektur Caca menggabungkan teknik-teknik modern terbaik dari berbagai model state-of-the-art:
+</p>
+
+<ul>
+  <li>
+    <strong>🔄 Grouped Query Attention (GQA)</strong> - 
+    Keseimbangan optimal antara kecepatan inference dan kualitas output
+  </li>
+  <li>
+    <strong>🌀 RoPE (Rotary Positional Embeddings)</strong> - 
+    Encoding posisi yang terbukti efektif untuk sequence panjang
+  </li>
+  <li>
+    <strong>⚡ SwiGLU Activation</strong> - 
+    Performa superior dibanding ReLU/GELU dalam language modeling
+  </li>
+  <li>
+    <strong>📊 RMSNorm</strong> - 
+    Normalisasi yang lebih efisien dan stabil dibanding LayerNorm
+  </li>
+  <li>
+    <strong>🪟 Sliding Window Attention</strong> - 
+    Efisiensi memori untuk context window panjang (4,096 tokens)
+  </li>
+  <li>
+    <strong>💫 Flash Attention Compatible</strong> - 
+    Support untuk Flash Attention 2-4x lebih cepat (opsional)
+  </li>
+  <li>
+    <strong>🔄 KV Cache Support</strong> - 
+    Efficient autoregressive generation dengan caching
+  </li>
+</ul>
+
+<!-- 
+     TABLE SECTION - dengan semua atribut
+      -->
+
+<h2 align="center">🏗️ Spesifikasi Teknis</h2>
+
+<div align="center">
+
+<table>
+  <caption>
+    <strong>Model Configuration Parameters</strong>
+  </caption>
+  <colgroup>
+    <col style="width: 50%"/>
+    <col style="width: 50%"/>
+  </colgroup>
+  <thead>
+    <tr>
+      <th align="left">Parameter</th>
+      <th align="right">Nilai</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td align="left"><strong>Total Parameters</strong></td>
+      <td align="right"><code>10,485,760</code> (~0.01B)</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Vocab Size</strong></td>
+      <td align="right"><code>50,000</code></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Hidden Size</strong></td>
+      <td align="right"><code>256</code></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Num Layers</strong></td>
+      <td align="right"><code>8</code></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Attention Heads</strong></td>
+      <td align="right"><code>8</code></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>KV Heads (GQA)</strong></td>
+      <td align="right"><code>2</code></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>GQA Ratio</strong></td>
+      <td align="right"><code>4:1</code></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Intermediate Size</strong></td>
+      <td align="right"><code>682</code></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Context Length</strong></td>
+      <td align="right"><code>8,192</code> tokens</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Sliding Window</strong></td>
+      <td align="right"><code>4,096</code> tokens</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>RoPE Theta</strong></td>
+      <td align="right"><code>10,000</code></td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Memory (FP16)</strong></td>
+      <td align="right">~<code>0.02</code> GB</td>
+    </tr>
+    <tr>
+      <td align="left"><strong>Memory (FP32)</strong></td>
+      <td align="right">~<code>0.04</code> GB</td>
+    </tr>
+  </tbody>
+  <tfoot>
+    <tr>
+      <td colspan="2" align="center">
+        <small><em>All values are approximate and may vary based on implementation</em></small>
+      </td>
+    </tr>
+  </tfoot>
+</table>
+
+</div>
+
+<!-- 
+     DETAILS/SUMMARY - Collapsible sections
+      -->
+
+<h2>📚 Model Family</h2>
+
+<p>Kami menyediakan berbagai ukuran model untuk berbagai use case:</p>
+
+<details open>
+  <summary>
+    <strong>🐣 Tiny &amp; Small Models (10M - 500M)</strong>
+  </summary>
+  
+  <p>Cocok untuk: Eksperimen cepat, edge devices, pembelajaran</p>
+  
+  <table>
+    <thead>
+      <tr>
+        <th>Model</th>
+        <th>Params</th>
+        <th>Hidden</th>
+        <th>Layers</th>
+        <th>Heads</th>
+        <th>KV Heads</th>
+        <th>Context</th>
+        <th>Memory (FP16)</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr>
+        <td>
+          <a href="https://huggingface.co/Lyon28/caca-10m" target="_blank">caca-10M</a>
+        </td>
+        <td>10M</td>
+        <td>256</td>
+        <td>8</td>
+        <td>8</td>
+        <td>2</td>
+        <td>8K</td>
+        <td>~0.02 GB</td>
+      </tr>
+      <tr>
+        <td>
+          <a href="https://huggingface.co/Lyon28/caca-50m" target="_blank">caca-50M</a>
+        </td>
+        <td>50M</td>
+        <td>512</td>
+        <td>12</td>
+        <td>8</td>
+        <td>2</td>
+        <td>8K</td>
+        <td>~0.1 GB</td>
+      </tr>
+      <tr>
+        <td>
+          <a href="https://huggingface.co/Lyon28/caca-100m" target="_blank">caca-100M</a>
+        </td>
+        <td>100M</td>
+        <td>768</td>
+        <td>12</td>
+        <td>12</td>
+        <td>3</td>
+        <td>8K</td>
+        <td>~0.2 GB</td>
+      </tr>
+    </tbody>
+  </table>
+  
+</details>
+
+<details>
+  <summary>
+    <strong>🦅 Medium Models (1B - 10B)</strong>
+  </summary>
+  
+  <p>Cocok untuk: Aplikasi production, fine-tuning, domain-specific tasks</p>
+  
+  <p><em>Click to expand for model list...</em></p>
+  
+</details>
+
+<!-- 
+     CODE BLOCKS dengan syntax highlighting
+      -->
+
+<h2>🚀 Quick Start</h2>
+
+<h3>💻 Installation</h3>
+
+<pre><code class="language-bash"># Install dengan xFormers untuk speedup 3x
+pip install caca-transformers[xformers]
+
+# Atau manual
+pip install caca-transformers
+pip install xformers
+
+# Untuk Flash Attention (4x speedup) - opsional
+pip install flash-attn --no-build-isolation
+</code></pre>
+
+<h3>Penggunaan Dasar</h3>
+
+<pre><code class="language-python">from caca_transformers import CacaForCausalLM, CacaConfig
+import torch
+
+# Load model
+model = CacaForCausalLM.from_pretrained("Lyon28/caca-10m")
+
+# Atau buat dari scratch
+config = CacaConfig()
+model = CacaForCausalLM(config)
+
+# Info model
+print(f"Parameters: {model.num_parameters():,}")
+</code></pre>
+
+<!-- 
+     INLINE ELEMENTS
+      -->
+
+<h2>💡 Tips &amp; Best Practices</h2>
+
+<p>
+  Gunakan <kbd>Ctrl</kbd> + <kbd>C</kbd> untuk copy code. 
+  Parameter <code>learning_rate</code> sebaiknya <mark>3e-4</mark> untuk pretraining.
+  Formula RMSNorm: <code>x / RMS(x) * γ</code> dimana 
+  RMS(x) = <code>sqrt(mean(x<sup>2</sup>) + ε)</code>
+</p>
+
+<p>
+  <small>
+    <em>Note: Semua nilai adalah perkiraan dan dapat bervariasi</em>
+  </small>
+</p>
+
+<p>
+  Referensi: <cite>Attention is All You Need</cite> (Vaswani et al., 2017)
+</p>
+
+<!-- 
+     MIXED CONTENT TABLE
+      -->
+
+<h2>📊 Perbandingan dengan Arsitektur Lain</h2>
+
+<table>
+  <thead>
+    <tr>
+      <th rowspan="2">Feature</th>
+      <th colspan="2">Decoder-Only</th>
+      <th colspan="2">Others</th>
+    </tr>
+    <tr>
+      <th>Caca</th>
+      <th>LLaMA 2</th>
+      <th>GPT-3</th>
+      <th>BERT</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>GQA</td>
+      <td align="center">✅</td>
+      <td align="center">✅</td>
+      <td align="center">❌</td>
+      <td align="center">❌</td>
+    </tr>
+    <tr>
+      <td>RoPE</td>
+      <td align="center">✅</td>
+      <td align="center">✅</td>
+      <td align="center">❌</td>
+      <td align="center">❌</td>
+    </tr>
+    <tr>
+      <td>Open Source</td>
+      <td align="center">✅</td>
+      <td align="center">✅</td>
+      <td align="center">❌</td>
+      <td align="center">✅</td>
+    </tr>
+  </tbody>
+</table>
+
+<!-- 
+     FOOTER SECTION
+      -->
+
+<hr/>
+
+<div align="center">
+  
+  <h2>🌟 Star History</h2>
+  
+  <a href="https://star-history.com/#Lyon-28/caca-transformers&Date" target="_blank" rel="noopener noreferrer">
+    <img 
+      src="https://api.star-history.com/svg?repos=Lyon-28/caca-transformers&type=Date" 
+      alt="Star History Chart"
+      title="View Star History"
+      width="100%"
+      loading="lazy"
+    />
+  </a>
+  
+</div>
+
+<hr/>
+
+<div align="center">
+  
+  <p>
+    <strong>🚀 Built with ❤️ for the Indonesian AI Community</strong>
+  </p>
+  
+  <p>
+    <a href="https://github.com/Lyon-28/caca-transformers" target="_blank" rel="noopener noreferrer">GitHub</a>
+    •
+    <a href="https://huggingface.co/Lyon28" target="_blank" rel="noopener noreferrer">Hugging Face</a>
+  </p>
+  
+  <p>
+    <small>
+      <strong>Dibuat oleh 
+        <a href="https://huggingface.co/Lyon28" target="_blank" rel="noopener noreferrer">Lyon</a>
+      </strong>
+      <br/>
+      Apache 2.0 License | 2025
+    </small>
+  </p>
+  
+</div>
+
+<!-- 
+  TODO: 
+  - Add more model variants
+  - Include benchmark results
+  - Add training scripts
+-->