| <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1240 1363" width="1240" height="1363" font-size="14"> |
| <style> |
| :root { |
| --bg: #ffffff; --fg: #1b1f24; --muted: #6b7280; --panel: #f6f8fa; --grid: #e5e7eb; |
| --embed: #dbeafe; --embed-s: #3b82f6; |
| --attn: #cffafe; --attn-s: #06b6d4; |
| --mamba: #dcfce7; --mamba-s: #22c55e; |
| --linattn: #fce7f3; --linattn-s: #ec4899; |
| --recur: #ede9fe; --recur-s: #8b5cf6; |
| --moe: #ffedd5; --moe-s: #f97316; |
| --mlp: #ede9fe; --mlp-s: #8b5cf6; |
| --norm: #e5e7eb; --norm-s: #9ca3af; |
| --head: #fee2e2; --head-s: #ef4444; |
| --config: #f1f5f9; --config-s: #64748b; |
| --rope: #fef9c3; --rope-s: #eab308; |
| --layer: #f8fafc; --layer-s: #cbd5e1; |
| --io: #f1f5f9; --io-s: #94a3b8; |
| --soft: #fae8ff; --soft-s: #c026d3; |
| --add: #ffffff; --add-s: #475569; |
| --block-s: #94a3b8; |
| --residual: #f59e0b; |
| --added: #16a34a; --over: #d97706; --deleted: #dc2626; |
| --lt-full: #06b6d4; --lt-sliding: #3b82f6; --lt-chunked: #8b5cf6; |
| --lt-compressed: #f97316; --lt-heavy: #dc2626; --lt-linear: #ec4899; --lt-mamba: #22c55e; |
| --cell-on: #0ea5e9; --cell-off: #e5e7eb; |
| --vision: #dcfce7; --vision-s: #16a34a; --audio: #fae8ff; --audio-s: #c026d3; |
| --proj: #fef3c7; --proj-s: #d97706; --xattn: #db2777; |
| --conv: #d1fae5; --conv-s: #10b981; --act: #ecfccb; --act-s: #65a30d; |
| --pool: #e0f2fe; --pool-s: #0284c7; --quant: #fae8ff; --quant-s: #c026d3; |
| } |
| @media (prefers-color-scheme: dark) { |
| :root { |
| --bg: #0d1117; --fg: #e6edf3; --muted: #8b949e; --panel: #161b22; --grid: #30363d; |
| --embed: #172554; --attn: #083344; --mamba: #052e16; --linattn: #500724; |
| --recur: #2e1065; --moe: #431407; --mlp: #2e1065; --norm: #21262d; --head: #450a0a; |
| --config: #1e293b; --rope: #422006; --layer: #161b22; |
| --conv: #022c22; --act: #1a2e05; --pool: #082f49; --quant: #3b0764; --proj: #422006; |
| --io: #1e293b; --soft: #3b0764; --add: #0d1117; --cell-off: #21262d; |
| } |
| } |
| .bg { fill: var(--bg); } |
| text { font-family: ui-sans-serif, -apple-system, "Segoe UI", Roboto, sans-serif; fill: var(--fg); } |
| .title { font-size: 22px; font-weight: 700; } |
| .subtitle { font-size: 13px; fill: var(--muted); } |
| .box-label { font-size: 14px; font-weight: 600; } |
| .box-label.sm { font-size: 12.5px; } |
| .box-sub { font-size: 11px; fill: var(--muted); } |
| .glyph { font-size: 18px; font-weight: 700; fill: var(--add-s); } |
| .badge { font-size: 12px; font-weight: 700; fill: var(--fg); } |
| .facts-k { font-size: 11.5px; fill: var(--muted); } |
| .facts-v { font-size: 11.5px; font-weight: 600; } |
| .legend-t { font-size: 11.5px; fill: var(--fg); } |
| .panel { fill: var(--panel); stroke: var(--grid); } |
| rect.b { rx: 9; stroke-width: 1.6; } |
| .c-embed { fill: var(--embed); stroke: var(--embed-s); } |
| .c-attn { fill: var(--attn); stroke: var(--attn-s); } |
| .c-mamba { fill: var(--mamba); stroke: var(--mamba-s); } |
| .c-linattn { fill: var(--linattn); stroke: var(--linattn-s); } |
| .c-recur { fill: var(--recur); stroke: var(--recur-s); } |
| .c-moe { fill: var(--moe); stroke: var(--moe-s); } |
| .c-mlp { fill: var(--mlp); stroke: var(--mlp-s); } |
| .c-norm { fill: var(--norm); stroke: var(--norm-s); } |
| .c-head { fill: var(--head); stroke: var(--head-s); } |
| .c-config{ fill: var(--config);stroke: var(--config-s); } |
| .c-rope { fill: var(--rope); stroke: var(--rope-s); } |
| .c-proj { fill: var(--proj); stroke: var(--proj-s); } |
| .c-conv { fill: var(--conv); stroke: var(--conv-s); } |
| .c-act { fill: var(--act); stroke: var(--act-s); } |
| .c-pool { fill: var(--pool); stroke: var(--pool-s); } |
| .c-quant { fill: var(--quant); stroke: var(--quant-s); } |
| .c-layer { fill: var(--layer); stroke: var(--layer-s); } |
| .c-io { fill: var(--io); stroke: var(--io-s); } |
| .c-soft { fill: var(--soft); stroke: var(--soft-s); } |
| .c-add { fill: var(--add); stroke: var(--add-s); } |
| .c-block { fill: none; stroke: var(--block-s); stroke-width: 1.6; stroke-dasharray: 7 5; } |
| .c-lt-full { fill: var(--lt-full); stroke: var(--lt-full); } |
| .c-lt-sliding { fill: var(--lt-sliding); stroke: var(--lt-sliding); } |
| .c-lt-chunked { fill: var(--lt-chunked); stroke: var(--lt-chunked); } |
| .c-lt-compressed { fill: var(--lt-compressed); stroke: var(--lt-compressed); } |
| .c-lt-heavy { fill: var(--lt-heavy); stroke: var(--lt-heavy); } |
| .c-lt-linear { fill: var(--lt-linear); stroke: var(--lt-linear); } |
| .c-lt-mamba { fill: var(--lt-mamba); stroke: var(--lt-mamba); } |
| .cell-on { fill: var(--cell-on); } |
| .cell-off { fill: var(--cell-off); } |
| .grid-frame { fill: none; stroke: var(--grid); stroke-width: 1; } |
| .mask-bg { fill: var(--cell-off); } |
| .mask-on { fill: #22c55e; } |
| .mask-div { stroke: var(--fg); stroke-width: 1.5; stroke-dasharray: 3 2; } |
| .c-vision { fill: var(--vision); stroke: var(--vision-s); } |
| .c-audio { fill: var(--audio); stroke: var(--audio-s); } |
| .c-proj { fill: var(--proj); stroke: var(--proj-s); } |
| .c-sub { fill: var(--bg); stroke: var(--block-s); stroke-width: 1.2; } |
| .sec-h { font-size: 12px; font-weight: 700; } |
| .sec-hbar { fill: var(--bg); opacity: 0.82; } |
| .residual.xattn { stroke: var(--xattn); stroke-width: 2.4; } |
| .ghost { opacity: 0.32; stroke-dasharray: 4 3; } |
| .ch-added rect.b, rect.b.ch-added { stroke: var(--added); stroke-width: 3.2; } |
| .ch-over rect.b, rect.b.ch-over { stroke: var(--over); stroke-width: 3.2; } |
| .ch-deleted rect.b, rect.b.ch-deleted { stroke: var(--deleted); stroke-width: 3.2; } |
| .edge { stroke: var(--grid); stroke-width: 2; } |
| .flow { stroke: var(--grid); stroke-width: 2; fill: none; } |
| .residual { stroke: var(--residual); stroke-width: 2; fill: none; } |
| .rope { stroke: var(--rope-s); stroke-width: 2.2; fill: none; } |
| .xattn { stroke: var(--xattn); stroke-width: 2.4; fill: none; } |
| .cell-idx { font-size: 9px; fill: #ffffff; font-weight: 600; } |
| .sky { fill: #bae6fd; } .sun { fill: #fde047; } .hill { fill: #4ade80; } |
| </style> |
| <defs><marker id="ah-flow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="var(--grid)"/></marker><marker id="ah-residual" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="var(--residual)"/></marker><marker id="ah-rope" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="var(--rope-s)"/></marker><marker id="ah-xattn" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse"><path d="M0,0 L10,5 L0,10 z" fill="var(--xattn)"/></marker></defs> |
| <rect class="bg" x="0" y="0" width="1240" height="1363"/> |
| <text class="title" x="24" y="34">afmoe</text> |
| <text class="subtitle" x="24" y="54">diff vs llama (+qwen2_moe,gpt_oss) · 7 overridden · 21 added · 0 deleted · 5 new · 4 inherited-as-is</text> |
| <line class="edge" x1="485" y1="108" x2="485" y2="1321"/> |
| <g class=""><rect class="c-block" x="286" y="184" width="668" height="1003" rx="14"/><text class="box-sub" x="298" y="202">AfmoeModel · base model</text></g> |
| <g class=""><title>32 × AfmoeDecoderLayer</title><rect class="c-block" x="294" y="257" width="652" height="890" rx="14"/><text class="box-sub" x="306" y="275">AfmoeDecoderLayer</text><text class="badge" x="934" y="277" text-anchor="end">× 32</text></g> |
| <g class=""><rect class="c-block" x="286" y="1187" width="668" height="142" rx="14"/><text class="box-sub" x="298" y="1205">AfmoeForCausalLM · task head</text></g> |
| <g class=""><rect class="c-block" x="36" y="255" width="58" height="16" rx="14"/><text class="box-sub" x="48" y="273">layers (idx)</text></g> |
| <g class=""><rect class="b c-attn ch-over" x="320" y="321" width="600" height="530" rx="11"/><rect class="sec-hbar" x="325" y="325" width="590" height="18" rx="5"/><text class="sec-h" x="332" y="338">Self-Attention · AfmoeAttention (2 layer types — same modules, mask differs)</text></g> |
| <g class="ghost"><rect class="b c-moe " x="320" y="947" width="330" height="152" rx="11"/><rect class="sec-hbar" x="325" y="951" width="320" height="18" rx="5"/><text class="sec-h" x="332" y="964">Sparse MoE · LlamaMLP ↩ inherited</text><text class="badge" x="638" y="964" text-anchor="end">MoE</text></g> |
| <polyline class="rope" points="284,365 320,365" marker-end="url(#ah-rope)"/> |
| <polyline class="residual" points="320,279 286,279 286,878 320,878" stroke-dasharray="6 4" marker-end="url(#ah-residual)"/> |
| <polyline class="residual" points="320,878 270,878 270,1126 320,1126" stroke-dasharray="6 4" marker-end="url(#ah-residual)"/> |
| <g class=""><rect class="b c-io" x="320" y="108" width="330" height="32" rx="16"/><text class="box-label sm" x="485.0" y="122.0" text-anchor="middle">input_ids</text><text class="box-sub" x="485.0" y="137.0" text-anchor="middle">tokenize("Hey, how are you?") → [1, 6]</text></g> |
| <g class="ghost"><title>token 0: 'Hey'</title><rect class="b c-sub" x="320" y="154" width="50" height="26" rx="9"/><text class="box-label sm" x="345.0" y="165.0" text-anchor="middle">Hey</text><text class="box-sub" x="345.0" y="180.0" text-anchor="middle">0</text></g> |
| <g class="ghost"><title>token 1: ','</title><rect class="b c-sub" x="375" y="154" width="50" height="26" rx="9"/><text class="box-label sm" x="400.0" y="165.0" text-anchor="middle">,</text><text class="box-sub" x="400.0" y="180.0" text-anchor="middle">1</text></g> |
| <g class="ghost"><title>token 2: '␣how'</title><rect class="b c-sub" x="431" y="154" width="50" height="26" rx="9"/><text class="box-label sm" x="456.0" y="165.0" text-anchor="middle">␣how</text><text class="box-sub" x="456.0" y="180.0" text-anchor="middle">2</text></g> |
| <g class="ghost"><title>token 3: '␣are'</title><rect class="b c-sub" x="487" y="154" width="50" height="26" rx="9"/><text class="box-label sm" x="512.0" y="165.0" text-anchor="middle">␣are</text><text class="box-sub" x="512.0" y="180.0" text-anchor="middle">3</text></g> |
| <g class="ghost"><title>token 4: '␣you'</title><rect class="b c-sub" x="543" y="154" width="50" height="26" rx="9"/><text class="box-label sm" x="568.0" y="165.0" text-anchor="middle">␣you</text><text class="box-sub" x="568.0" y="180.0" text-anchor="middle">4</text></g> |
| <g class="ghost"><title>token 5: '?'</title><rect class="b c-sub" x="599" y="154" width="50" height="26" rx="9"/><text class="box-label sm" x="624.0" y="165.0" text-anchor="middle">?</text><text class="box-sub" x="624.0" y="180.0" text-anchor="middle">5</text></g> |
| <g class="ghost"><rect class="b c-embed" x="320" y="198" width="330" height="45" rx="9"/><text class="box-label sm" x="485.0" y="218.5" text-anchor="middle">Token Embedding</text><text class="box-sub" x="485.0" y="233.5" text-anchor="middle">weight [200192 × 2048] → [1, 6, 2048]</text></g> |
| <g class="ghost"><rect class="b c-norm" x="320" y="279" width="330" height="28" rx="9"/><text class="box-label sm" x="485.0" y="291.0" text-anchor="middle">AfmoeRMSNorm</text><text class="box-sub" x="485.0" y="306.0" text-anchor="middle">pre-attention [1, 6, 2048]</text></g> |
| <g class="ghost"><title>q_proj: Linear [2048→2048]</title><rect class="b c-proj" x="330" y="347" width="580" height="22" rx="9"/><text class="box-label sm" x="620.0" y="362.5" text-anchor="middle">q_proj [2048→2048]</text></g> |
| <g class="ghost"><title>k_proj: Linear [2048→2048]</title><rect class="b c-proj" x="330" y="377" width="580" height="22" rx="9"/><text class="box-label sm" x="620.0" y="392.5" text-anchor="middle">k_proj [2048→2048]</text></g> |
| <g class="ghost"><title>v_proj: Linear [2048→2048]</title><rect class="b c-proj" x="330" y="407" width="580" height="22" rx="9"/><text class="box-label sm" x="620.0" y="422.5" text-anchor="middle">v_proj [2048→2048]</text></g> |
| <g class="ghost"><title>o_proj: Linear [2048→2048]</title><rect class="b c-proj" x="330" y="437" width="580" height="22" rx="9"/><text class="box-label sm" x="620.0" y="452.5" text-anchor="middle">o_proj [2048→2048]</text></g> |
| <g class=""><title>q_norm: AfmoeRMSNorm 128</title><rect class="b c-norm ch-added" x="330" y="467" width="580" height="22" rx="9"/><text class="box-label sm" x="620.0" y="482.5" text-anchor="middle">q_norm RMSNorm 128</text><circle cx="342" cy="479" r="5" fill="var(--added)"/></g> |
| <g class=""><title>k_norm: AfmoeRMSNorm 128</title><rect class="b c-norm ch-added" x="330" y="497" width="580" height="22" rx="9"/><text class="box-label sm" x="620.0" y="512.5" text-anchor="middle">k_norm RMSNorm 128</text><circle cx="342" cy="509" r="5" fill="var(--added)"/></g> |
| <g class="ghost"><title>gate_proj: Linear [2048→2048]</title><rect class="b c-proj" x="330" y="527" width="580" height="22" rx="9"/><text class="box-label sm" x="620.0" y="542.5" text-anchor="middle">gate_proj [2048→2048]</text></g> |
| <g class="ghost"><rect class="b c-sub" x="330" y="557" width="580" height="30" rx="9"/><text class="box-label sm" x="620.0" y="570.0" text-anchor="middle">scaled dot-product attention</text><text class="box-sub" x="620.0" y="585.0" text-anchor="middle">16 heads · head_dim 128</text></g> |
| <g class=""><title>sliding_attention attention mask (q↓ × k→)</title><text class="box-sub" x="351" y="599">sliding mask</text><rect class="mask-bg" x="351" y="605" width="240.0" height="240.0"/><rect class="mask-on" x="351.0" y="605.0" width="10.0" height="10.0"/><rect class="mask-on" x="351.0" y="615.0" width="10.0" height="10.0"/><rect class="mask-on" x="361.0" y="615.0" width="10.0" height="10.0"/><rect class="mask-on" x="351.0" y="625.0" width="10.0" height="10.0"/><rect class="mask-on" x="361.0" y="625.0" width="10.0" height="10.0"/><rect class="mask-on" x="371.0" y="625.0" width="10.0" height="10.0"/><rect class="mask-on" x="351.0" y="635.0" width="10.0" height="10.0"/><rect class="mask-on" x="361.0" y="635.0" width="10.0" height="10.0"/><rect class="mask-on" x="371.0" y="635.0" width="10.0" height="10.0"/><rect class="mask-on" x="381.0" y="635.0" width="10.0" height="10.0"/><rect class="mask-on" x="351.0" y="645.0" width="10.0" height="10.0"/><rect class="mask-on" x="361.0" y="645.0" width="10.0" height="10.0"/><rect class="mask-on" x="371.0" y="645.0" width="10.0" height="10.0"/><rect class="mask-on" x="381.0" y="645.0" width="10.0" height="10.0"/><rect class="mask-on" x="391.0" y="645.0" width="10.0" height="10.0"/><rect class="mask-on" x="351.0" y="655.0" width="10.0" height="10.0"/><rect class="mask-on" x="361.0" y="655.0" width="10.0" height="10.0"/><rect class="mask-on" x="371.0" y="655.0" width="10.0" height="10.0"/><rect class="mask-on" x="381.0" y="655.0" width="10.0" height="10.0"/><rect class="mask-on" x="391.0" y="655.0" width="10.0" height="10.0"/><rect class="mask-on" x="401.0" y="655.0" width="10.0" height="10.0"/><rect class="mask-on" x="361.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="371.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="381.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="391.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="401.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="411.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="371.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="381.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="391.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="401.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="411.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="421.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="381.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="391.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="401.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="411.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="421.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="431.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="391.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="401.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="411.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="421.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="431.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="441.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="401.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="411.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="421.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="431.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="441.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="451.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="411.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="421.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="431.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="441.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="451.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="461.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="421.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="431.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="441.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="451.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="461.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="471.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="431.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="441.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="451.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="461.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="471.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="481.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="441.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="451.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="461.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="471.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="481.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="491.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="451.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="461.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="471.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="481.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="491.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="501.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="461.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="471.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="481.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="491.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="501.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="511.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="471.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="481.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="491.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="501.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="511.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="521.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="481.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="491.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="501.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="511.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="521.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="531.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="491.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="501.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="511.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="521.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="531.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="541.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="501.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="511.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="521.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="531.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="541.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="551.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="511.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="521.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="531.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="541.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="551.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="561.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="521.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="531.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="541.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="551.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="561.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="571.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="531.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="541.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="551.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="561.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="571.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="581.0" y="835.0" width="10.0" height="10.0"/><rect class="grid-frame" x="351" y="605" width="240.0" height="240.0"/></g> |
| <g class=""><title>full_attention attention mask (q↓ × k→)</title><text class="box-sub" x="649" y="599">full mask</text><rect class="mask-bg" x="649" y="605" width="240.0" height="240.0"/><rect class="mask-on" x="649.0" y="605.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="615.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="615.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="625.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="625.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="625.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="635.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="635.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="635.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="635.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="645.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="645.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="645.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="645.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="645.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="655.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="655.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="655.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="655.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="655.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="655.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="665.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="675.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="685.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="695.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="705.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="715.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="769.0" y="725.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="769.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="779.0" y="735.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="769.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="779.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="789.0" y="745.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="769.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="779.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="789.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="799.0" y="755.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="769.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="779.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="789.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="799.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="809.0" y="765.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="769.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="779.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="789.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="799.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="809.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="819.0" y="775.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="769.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="779.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="789.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="799.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="809.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="819.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="829.0" y="785.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="769.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="779.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="789.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="799.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="809.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="819.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="829.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="839.0" y="795.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="769.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="779.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="789.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="799.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="809.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="819.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="829.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="839.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="849.0" y="805.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="769.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="779.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="789.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="799.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="809.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="819.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="829.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="839.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="849.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="859.0" y="815.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="769.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="779.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="789.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="799.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="809.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="819.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="829.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="839.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="849.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="859.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="869.0" y="825.0" width="10.0" height="10.0"/><rect class="mask-on" x="649.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="659.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="669.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="679.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="689.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="699.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="709.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="719.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="729.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="739.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="749.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="759.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="769.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="779.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="789.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="799.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="809.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="819.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="829.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="839.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="849.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="859.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="869.0" y="835.0" width="10.0" height="10.0"/><rect class="mask-on" x="879.0" y="835.0" width="10.0" height="10.0"/><rect class="grid-frame" x="649" y="605" width="240.0" height="240.0"/></g> |
| <g class="ghost"><rect class="b c-rope" x="152" y="347" width="132" height="40" rx="9"/><text class="box-label sm" x="218.0" y="365.0" text-anchor="middle">RoPE</text><text class="box-sub" x="218.0" y="380.0" text-anchor="middle">rotary positions → Q, K</text></g> |
| <g class=""><title>residual add</title><circle class="b c-add " cx="485.0" cy="878" r="13"/><text class="glyph" x="485.0" y="884" text-anchor="middle">+</text></g> |
| <g class="ghost"><rect class="b c-norm" x="320" y="905" width="330" height="28" rx="9"/><text class="box-label sm" x="485.0" y="917.0" text-anchor="middle">AfmoeRMSNorm</text><text class="box-sub" x="485.0" y="932.0" text-anchor="middle">pre-FFN [1, 6, 2048]</text></g> |
| <g class="ghost"><title>gate_proj: Linear [2048→6144]</title><rect class="b c-proj" x="332" y="973" width="306" height="22" rx="9"/><text class="box-label sm" x="485.0" y="988.5" text-anchor="middle">gate_proj [2048→6144]</text></g> |
| <g class="ghost"><title>up_proj: Linear [2048→6144]</title><rect class="b c-proj" x="332" y="1003" width="306" height="22" rx="9"/><text class="box-label sm" x="485.0" y="1018.5" text-anchor="middle">up_proj [2048→6144]</text></g> |
| <g class="ghost"><title>down_proj: Linear [6144→2048]</title><rect class="b c-proj" x="332" y="1033" width="306" height="22" rx="9"/><text class="box-label sm" x="485.0" y="1048.5" text-anchor="middle">down_proj [6144→2048]</text></g> |
| <g class="ghost"><title>act_fn: SiLUActivation</title><rect class="b c-act" x="332" y="1063" width="306" height="22" rx="9"/><text class="box-label sm" x="485.0" y="1078.5" text-anchor="middle">act_fn SiLU</text></g> |
| <g class=""><title>residual add</title><circle class="b c-add " cx="485.0" cy="1126" r="13"/><text class="glyph" x="485.0" y="1132" text-anchor="middle">+</text></g> |
| <g class="ghost"><rect class="b c-norm" x="320" y="1153" width="330" height="28" rx="9"/><text class="box-label sm" x="485.0" y="1165.0" text-anchor="middle">Final RMSNorm</text><text class="box-sub" x="485.0" y="1180.0" text-anchor="middle">[1, 6, 2048]</text></g> |
| <g class=""><title>AfmoeForCausalLM</title><rect class="b c-head ch-over" x="320" y="1195" width="330" height="42" rx="9"/><text class="box-label sm" x="485.0" y="1214.0" text-anchor="middle">LM Head</text><text class="box-sub" x="485.0" y="1229.0" text-anchor="middle">Linear [2048→200192]</text><circle cx="332" cy="1207" r="5" fill="var(--over)"/></g> |
| <g class="ghost"><rect class="b c-soft" x="320" y="1251" width="330" height="26" rx="9"/><text class="box-label sm" x="485.0" y="1268.5" text-anchor="middle">Softmax</text></g> |
| <g class=""><rect class="b c-io" x="320" y="1291" width="330" height="30" rx="15"/><text class="box-label sm" x="485.0" y="1304.0" text-anchor="middle">logits</text><text class="box-sub" x="485.0" y="1319.0" text-anchor="middle">[1, 6, 200192]</text></g> |
| <g class=""><title>layer 0: sliding_attention</title><rect class="c-lt-sliding" x="40" y="279" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="294.5" text-anchor="middle">0</text></g> |
| <g class=""><title>layer 1: sliding_attention</title><rect class="c-lt-sliding" x="40" y="305" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="320.5" text-anchor="middle">1</text></g> |
| <g class=""><title>layer 2: sliding_attention</title><rect class="c-lt-sliding" x="40" y="332" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="347.5" text-anchor="middle">2</text></g> |
| <g class=""><title>layer 3: full_attention</title><rect class="c-lt-full" x="40" y="359" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="374.5" text-anchor="middle">3</text></g> |
| <g class=""><title>layer 4: sliding_attention</title><rect class="c-lt-sliding" x="40" y="386" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="401.5" text-anchor="middle">4</text></g> |
| <g class=""><title>layer 5: sliding_attention</title><rect class="c-lt-sliding" x="40" y="413" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="428.5" text-anchor="middle">5</text></g> |
| <g class=""><title>layer 6: sliding_attention</title><rect class="c-lt-sliding" x="40" y="440" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="455.5" text-anchor="middle">6</text></g> |
| <g class=""><title>layer 7: full_attention</title><rect class="c-lt-full" x="40" y="467" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="482.5" text-anchor="middle">7</text></g> |
| <g class=""><title>layer 8: sliding_attention</title><rect class="c-lt-sliding" x="40" y="494" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="509.5" text-anchor="middle">8</text></g> |
| <g class=""><title>layer 9: sliding_attention</title><rect class="c-lt-sliding" x="40" y="520" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="535.5" text-anchor="middle">9</text></g> |
| <g class=""><title>layer 10: sliding_attention</title><rect class="c-lt-sliding" x="40" y="547" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="562.5" text-anchor="middle">10</text></g> |
| <g class=""><title>layer 11: full_attention</title><rect class="c-lt-full" x="40" y="574" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="589.5" text-anchor="middle">11</text></g> |
| <g class=""><title>layer 12: sliding_attention</title><rect class="c-lt-sliding" x="40" y="601" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="616.5" text-anchor="middle">12</text></g> |
| <g class=""><title>layer 13: sliding_attention</title><rect class="c-lt-sliding" x="40" y="628" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="643.5" text-anchor="middle">13</text></g> |
| <g class=""><title>layer 14: sliding_attention</title><rect class="c-lt-sliding" x="40" y="655" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="670.5" text-anchor="middle">14</text></g> |
| <g class=""><title>layer 15: full_attention</title><rect class="c-lt-full" x="40" y="682" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="697.5" text-anchor="middle">15</text></g> |
| <g class=""><title>layer 16: sliding_attention</title><rect class="c-lt-sliding" x="40" y="709" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="724.5" text-anchor="middle">16</text></g> |
| <g class=""><title>layer 17: sliding_attention</title><rect class="c-lt-sliding" x="40" y="735" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="750.5" text-anchor="middle">17</text></g> |
| <g class=""><title>layer 18: sliding_attention</title><rect class="c-lt-sliding" x="40" y="762" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="777.5" text-anchor="middle">18</text></g> |
| <g class=""><title>layer 19: full_attention</title><rect class="c-lt-full" x="40" y="789" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="804.5" text-anchor="middle">19</text></g> |
| <g class=""><title>layer 20: sliding_attention</title><rect class="c-lt-sliding" x="40" y="816" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="831.5" text-anchor="middle">20</text></g> |
| <g class=""><title>layer 21: sliding_attention</title><rect class="c-lt-sliding" x="40" y="843" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="858.5" text-anchor="middle">21</text></g> |
| <g class=""><title>layer 22: sliding_attention</title><rect class="c-lt-sliding" x="40" y="870" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="885.5" text-anchor="middle">22</text></g> |
| <g class=""><title>layer 23: full_attention</title><rect class="c-lt-full" x="40" y="897" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="912.5" text-anchor="middle">23</text></g> |
| <g class=""><title>layer 24: sliding_attention</title><rect class="c-lt-sliding" x="40" y="924" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="939.5" text-anchor="middle">24</text></g> |
| <g class=""><title>layer 25: sliding_attention</title><rect class="c-lt-sliding" x="40" y="950" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="965.5" text-anchor="middle">25</text></g> |
| <g class=""><title>layer 26: sliding_attention</title><rect class="c-lt-sliding" x="40" y="977" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="992.5" text-anchor="middle">26</text></g> |
| <g class=""><title>layer 27: full_attention</title><rect class="c-lt-full" x="40" y="1004" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="1019.5" text-anchor="middle">27</text></g> |
| <g class=""><title>layer 28: sliding_attention</title><rect class="c-lt-sliding" x="40" y="1031" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="1046.5" text-anchor="middle">28</text></g> |
| <g class=""><title>layer 29: sliding_attention</title><rect class="c-lt-sliding" x="40" y="1058" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="1073.5" text-anchor="middle">29</text></g> |
| <g class=""><title>layer 30: sliding_attention</title><rect class="c-lt-sliding" x="40" y="1085" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="1100.5" text-anchor="middle">30</text></g> |
| <g class=""><title>layer 31: full_attention</title><rect class="c-lt-full" x="40" y="1112" width="46" height="24" rx="2"/><text class="cell-idx" x="63.0" y="1127.5" text-anchor="middle">31</text></g> |
| <rect class="panel" x="952" y="88" width="264" height="174" rx="8"/> |
| <text class="facts-k" x="966" y="112">model id</text> |
| <text class="facts-v" x="1202" y="112" text-anchor="end">arcee-ai/Trinity-Mini</text> |
| <text class="facts-k" x="966" y="134">parent</text> |
| <text class="facts-v" x="1202" y="134" text-anchor="end">llama</text> |
| <text class="facts-k" x="966" y="156">classes</text> |
| <text class="facts-v" x="1202" y="156" text-anchor="end">11</text> |
| <text class="facts-k" x="966" y="178">overridden</text> |
| <text class="facts-v" x="1202" y="178" text-anchor="end">7</text> |
| <text class="facts-k" x="966" y="200">added</text> |
| <text class="facts-v" x="1202" y="200" text-anchor="end">21</text> |
| <text class="facts-k" x="966" y="222">new classes</text> |
| <text class="facts-v" x="1202" y="222" text-anchor="end">5</text> |
| <text class="facts-k" x="966" y="244">inherited as-is</text> |
| <text class="facts-v" x="1202" y="244" text-anchor="end">4</text> |
| <text class="legend-t" x="954" y="282" font-weight="700">legend</text> |
| <rect x="956" y="293" width="16" height="12" rx="2" fill="none" stroke="var(--added)" stroke-width="3"/> |
| <text class="legend-t" x="980" y="303">new submodule (vs parent)</text> |
| <rect x="956" y="311" width="16" height="12" rx="2" fill="none" stroke="var(--over)" stroke-width="3"/> |
| <text class="legend-t" x="980" y="321">changed / redefined</text> |
| <rect x="956" y="329" width="16" height="12" rx="2" fill="none" stroke="var(--deleted)" stroke-width="3"/> |
| <text class="legend-t" x="980" y="339">deleted</text> |
| <rect x="956" y="347" width="16" height="12" rx="2" fill="none" stroke="var(--grid)" stroke-width="3"/> |
| <text class="legend-t" x="980" y="357">inherited / copy-pasted</text> |
| <text class="legend-t" x="954" y="380" font-weight="700">changes by class</text> |
| <circle cx="960" cy="393" r="4" fill="var(--over)"/> |
| <text class="facts-v" x="972" y="396">AfmoeAttention</text> |
| <text class="box-sub" x="972" y="409">ovr __init__,forward</text> |
| <circle cx="960" cy="421" r="4" fill="var(--over)"/> |
| <text class="facts-v" x="972" y="424">AfmoeForCausalLM</text> |
| <text class="box-sub" x="972" y="437">ovr __init__,forward; 3 attr</text> |
| <circle cx="960" cy="449" r="4" fill="var(--added)"/> |
| <text class="facts-v" x="972" y="452">AfmoeDecoderLayer</text> |
| <text class="box-sub" x="972" y="465">add __init__,forward</text> |
| <circle cx="960" cy="477" r="4" fill="var(--added)"/> |
| <text class="facts-v" x="972" y="480">AfmoeModel</text> |
| <text class="box-sub" x="972" y="493">add __init__,forward</text> |
| <circle cx="960" cy="505" r="4" fill="var(--added)"/> |
| <text class="facts-v" x="972" y="508">AfmoePreTrainedModel</text> |
| <text class="box-sub" x="972" y="521">add _init_weights; 12 attr</text> |
| <circle cx="960" cy="533" r="4" fill="var(--added)"/> |
| <text class="facts-v" x="972" y="536">AfmoeSparseMoeBlock</text> |
| <text class="box-sub" x="972" y="549">add __init__,forward</text> |
| <circle cx="960" cy="561" r="4" fill="var(--added)"/> |
| <text class="facts-v" x="972" y="564">AfmoeTokenChoiceRouter</text> |
| <text class="box-sub" x="972" y="577">add __init__,forward</text> |
| </svg> |
