|
|
<!DOCTYPE html> |
|
|
<html lang="en"> |
|
|
<head> |
|
|
<meta charset="UTF-8"> |
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"> |
|
|
<title>GPT Training - Complete Visualization</title> |
|
|
<style> |
|
|
* { |
|
|
margin: 0; |
|
|
padding: 0; |
|
|
box-sizing: border-box; |
|
|
} |
|
|
|
|
|
body { |
|
|
min-height: 100vh; |
|
|
background: linear-gradient(145deg, #0d0d0d 0%, #1a1a2e 50%, #0d1520 100%); |
|
|
font-family: 'Segoe UI', system-ui, sans-serif; |
|
|
padding: 15px 10px; |
|
|
display: flex; |
|
|
flex-direction: column; |
|
|
align-items: center; |
|
|
overflow-x: auto; |
|
|
} |
|
|
|
|
|
h1 { |
|
|
color: #fff; |
|
|
font-size: 1.6rem; |
|
|
margin-bottom: 4px; |
|
|
text-align: center; |
|
|
} |
|
|
|
|
|
.subtitle { |
|
|
color: rgba(255, 255, 255, 0.4); |
|
|
margin-bottom: 12px; |
|
|
font-size: 0.85rem; |
|
|
} |
|
|
|
|
|
|
|
|
.colormap-legend { |
|
|
display: flex; |
|
|
gap: 20px; |
|
|
padding: 12px 20px; |
|
|
background: rgba(0, 0, 0, 0.5); |
|
|
border-radius: 10px; |
|
|
margin-bottom: 15px; |
|
|
flex-wrap: wrap; |
|
|
justify-content: center; |
|
|
border: 1px solid rgba(255, 255, 255, 0.1); |
|
|
} |
|
|
|
|
|
.cmap-item { |
|
|
display: flex; |
|
|
flex-direction: column; |
|
|
align-items: center; |
|
|
gap: 4px; |
|
|
} |
|
|
|
|
|
.cmap-label { |
|
|
font-size: 0.7rem; |
|
|
color: rgba(255, 255, 255, 0.7); |
|
|
text-transform: uppercase; |
|
|
letter-spacing: 1px; |
|
|
} |
|
|
|
|
|
.cmap-bar { |
|
|
width: 100px; |
|
|
height: 12px; |
|
|
border-radius: 3px; |
|
|
} |
|
|
|
|
|
|
|
|
.cmap-viridis { |
|
|
background: linear-gradient(90deg, #440154, #482878, #3e4a89, #31688e, #26838f, #1f9e89, #35b779, #6ece58, #b5de2b, #fde725); |
|
|
} |
|
|
|
|
|
|
|
|
.cmap-plasma { |
|
|
background: linear-gradient(90deg, #0d0887, #46039f, #7201a8, #9c179e, #bd3786, #d8576b, #ed7953, #fb9f3a, #fdca26, #f0f921); |
|
|
} |
|
|
|
|
|
|
|
|
.cmap-coolwarm { |
|
|
background: linear-gradient(90deg, #3b4cc0, #6688ee, #88bbff, #b8d4eb, #dddddd, #f5cba7, #f4a582, #d6604d, #b40426); |
|
|
} |
|
|
|
|
|
|
|
|
.cmap-inferno { |
|
|
background: linear-gradient(90deg, #000004, #1b0c41, #4a0c6b, #781c6d, #a52c60, #cf4446, #ed6925, #fb9b06, #f7d13d, #fcffa4); |
|
|
} |
|
|
|
|
|
.cmap-range { |
|
|
display: flex; |
|
|
justify-content: space-between; |
|
|
width: 100px; |
|
|
font-size: 0.6rem; |
|
|
color: rgba(255, 255, 255, 0.4); |
|
|
} |
|
|
|
|
|
|
|
|
.main-layout { |
|
|
display: flex; |
|
|
gap: 15px; |
|
|
align-items: flex-start; |
|
|
flex-wrap: wrap; |
|
|
justify-content: center; |
|
|
width: 100%; |
|
|
max-width: 1600px; |
|
|
} |
|
|
|
|
|
|
|
|
.model-column { |
|
|
display: flex; |
|
|
flex-direction: column; |
|
|
align-items: center; |
|
|
gap: 8px; |
|
|
flex: 1; |
|
|
min-width: 500px; |
|
|
} |
|
|
|
|
|
|
|
|
.side-panel { |
|
|
background: rgba(0, 0, 0, 0.4); |
|
|
border: 1px solid rgba(255, 255, 255, 0.1); |
|
|
border-radius: 12px; |
|
|
padding: 14px; |
|
|
width: 260px; |
|
|
display: flex; |
|
|
flex-direction: column; |
|
|
gap: 10px; |
|
|
} |
|
|
|
|
|
.panel-title { |
|
|
color: rgba(255, 255, 255, 0.85); |
|
|
font-size: 0.85rem; |
|
|
text-align: center; |
|
|
padding-bottom: 8px; |
|
|
border-bottom: 1px solid rgba(255, 255, 255, 0.1); |
|
|
} |
|
|
|
|
|
|
|
|
.phase-row { |
|
|
display: flex; |
|
|
justify-content: center; |
|
|
gap: 6px; |
|
|
} |
|
|
|
|
|
.phase-pill { |
|
|
padding: 5px 12px; |
|
|
border-radius: 12px; |
|
|
font-size: 0.7rem; |
|
|
color: rgba(255, 255, 255, 0.3); |
|
|
background: rgba(255, 255, 255, 0.05); |
|
|
border: 1px solid transparent; |
|
|
transition: all 0.3s; |
|
|
} |
|
|
|
|
|
.phase-pill.active { |
|
|
color: white; |
|
|
font-weight: 600; |
|
|
} |
|
|
|
|
|
.phase-pill.active.fwd { background: rgba(53, 183, 121, 0.3); border-color: #35b779; } |
|
|
.phase-pill.active.bwd { background: rgba(237, 121, 83, 0.3); border-color: #ed7953; } |
|
|
.phase-pill.active.upd { background: rgba(59, 76, 192, 0.3); border-color: #3b4cc0; } |
|
|
|
|
|
|
|
|
.stats-row { |
|
|
display: grid; |
|
|
grid-template-columns: repeat(2, 1fr); |
|
|
gap: 6px; |
|
|
} |
|
|
|
|
|
.stat-box { |
|
|
padding: 8px; |
|
|
background: rgba(255, 255, 255, 0.03); |
|
|
border-radius: 6px; |
|
|
text-align: center; |
|
|
} |
|
|
|
|
|
.stat-val { |
|
|
color: #35b779; |
|
|
font-size: 1rem; |
|
|
font-weight: 700; |
|
|
font-family: monospace; |
|
|
} |
|
|
|
|
|
.stat-lbl { |
|
|
color: rgba(255, 255, 255, 0.4); |
|
|
font-size: 0.6rem; |
|
|
margin-top: 2px; |
|
|
} |
|
|
|
|
|
|
|
|
.loss-box { |
|
|
padding: 10px; |
|
|
background: rgba(237, 121, 83, 0.1); |
|
|
border: 1px solid rgba(237, 121, 83, 0.3); |
|
|
border-radius: 8px; |
|
|
display: flex; |
|
|
justify-content: space-between; |
|
|
align-items: center; |
|
|
} |
|
|
|
|
|
.loss-lbl { |
|
|
color: rgba(255, 255, 255, 0.6); |
|
|
font-size: 0.8rem; |
|
|
} |
|
|
|
|
|
.loss-val { |
|
|
color: #ed7953; |
|
|
font-size: 1.2rem; |
|
|
font-weight: 700; |
|
|
font-family: monospace; |
|
|
} |
|
|
|
|
|
.loss-val.good { color: #35b779; } |
|
|
|
|
|
|
|
|
.status-box { |
|
|
padding: 10px; |
|
|
background: rgba(0, 0, 0, 0.3); |
|
|
border-radius: 8px; |
|
|
text-align: center; |
|
|
} |
|
|
|
|
|
.status-main { |
|
|
color: #6ece58; |
|
|
font-size: 0.85rem; |
|
|
font-weight: 500; |
|
|
} |
|
|
|
|
|
.status-sub { |
|
|
color: rgba(255, 255, 255, 0.4); |
|
|
font-size: 0.7rem; |
|
|
margin-top: 3px; |
|
|
} |
|
|
|
|
|
|
|
|
.layer-container { |
|
|
display: flex; |
|
|
flex-direction: column; |
|
|
align-items: center; |
|
|
gap: 4px; |
|
|
width: 100%; |
|
|
} |
|
|
|
|
|
.layer-label { |
|
|
font-size: 0.65rem; |
|
|
color: rgba(255, 255, 255, 0.4); |
|
|
text-transform: uppercase; |
|
|
letter-spacing: 1px; |
|
|
} |
|
|
|
|
|
|
|
|
.activation-stream { |
|
|
display: flex; |
|
|
gap: 2px; |
|
|
padding: 6px 10px; |
|
|
background: rgba(0, 0, 0, 0.4); |
|
|
border-radius: 6px; |
|
|
border: 1px solid rgba(255, 255, 255, 0.1); |
|
|
min-height: 24px; |
|
|
align-items: center; |
|
|
flex-wrap: wrap; |
|
|
max-width: 450px; |
|
|
justify-content: center; |
|
|
} |
|
|
|
|
|
.act-cell { |
|
|
width: 10px; |
|
|
height: 16px; |
|
|
border-radius: 2px; |
|
|
background: rgba(255, 255, 255, 0.1); |
|
|
transition: background 0.3s; |
|
|
} |
|
|
|
|
|
|
|
|
.flow-arrow { |
|
|
width: 3px; |
|
|
height: 24px; |
|
|
background: rgba(255, 255, 255, 0.1); |
|
|
border-radius: 2px; |
|
|
position: relative; |
|
|
} |
|
|
|
|
|
.flow-arrow.fwd { |
|
|
background: linear-gradient(to top, #1f9e89, #35b779); |
|
|
} |
|
|
|
|
|
.flow-arrow.bwd { |
|
|
background: linear-gradient(to bottom, #bd3786, #ed7953); |
|
|
} |
|
|
|
|
|
.flow-arrow::after { |
|
|
content: ''; |
|
|
position: absolute; |
|
|
left: 50%; |
|
|
transform: translateX(-50%); |
|
|
border-left: 6px solid transparent; |
|
|
border-right: 6px solid transparent; |
|
|
} |
|
|
|
|
|
.flow-arrow.fwd::after { |
|
|
top: -6px; |
|
|
border-bottom: 7px solid #35b779; |
|
|
} |
|
|
|
|
|
.flow-arrow.bwd::after { |
|
|
bottom: -6px; |
|
|
border-top: 7px solid #ed7953; |
|
|
} |
|
|
|
|
|
|
|
|
.layer-box { |
|
|
padding: 12px; |
|
|
border-radius: 10px; |
|
|
background: rgba(255, 255, 255, 0.03); |
|
|
border: 1px solid rgba(255, 255, 255, 0.1); |
|
|
transition: all 0.3s; |
|
|
width: 100%; |
|
|
max-width: 480px; |
|
|
} |
|
|
|
|
|
.layer-box.active-fwd { |
|
|
border-color: rgba(53, 183, 121, 0.5); |
|
|
box-shadow: 0 0 20px rgba(53, 183, 121, 0.2); |
|
|
} |
|
|
|
|
|
.layer-box.active-bwd { |
|
|
border-color: rgba(237, 121, 83, 0.5); |
|
|
box-shadow: 0 0 20px rgba(237, 121, 83, 0.2); |
|
|
} |
|
|
|
|
|
.layer-header { |
|
|
display: flex; |
|
|
justify-content: space-between; |
|
|
align-items: center; |
|
|
margin-bottom: 10px; |
|
|
} |
|
|
|
|
|
.layer-title { |
|
|
font-size: 0.8rem; |
|
|
color: rgba(255, 255, 255, 0.8); |
|
|
font-weight: 500; |
|
|
} |
|
|
|
|
|
.layer-dims { |
|
|
font-size: 0.65rem; |
|
|
color: rgba(255, 255, 255, 0.35); |
|
|
font-family: monospace; |
|
|
} |
|
|
|
|
|
|
|
|
.weight-grad-container { |
|
|
display: flex; |
|
|
gap: 10px; |
|
|
justify-content: center; |
|
|
flex-wrap: wrap; |
|
|
} |
|
|
|
|
|
.matrix-group { |
|
|
display: flex; |
|
|
flex-direction: column; |
|
|
align-items: center; |
|
|
gap: 3px; |
|
|
} |
|
|
|
|
|
.matrix-label { |
|
|
font-size: 0.6rem; |
|
|
color: rgba(255, 255, 255, 0.5); |
|
|
} |
|
|
|
|
|
.matrix-grid { |
|
|
display: grid; |
|
|
gap: 1px; |
|
|
padding: 4px; |
|
|
background: rgba(0, 0, 0, 0.4); |
|
|
border-radius: 4px; |
|
|
} |
|
|
|
|
|
.matrix-grid.g6 { grid-template-columns: repeat(6, 1fr); } |
|
|
.matrix-grid.g8 { grid-template-columns: repeat(8, 1fr); } |
|
|
.matrix-grid.g10 { grid-template-columns: repeat(10, 1fr); } |
|
|
|
|
|
.m-cell { |
|
|
width: 8px; |
|
|
height: 8px; |
|
|
border-radius: 1px; |
|
|
background: rgba(255, 255, 255, 0.08); |
|
|
transition: background 0.2s; |
|
|
} |
|
|
|
|
|
|
|
|
.transformer-block { |
|
|
padding: 14px; |
|
|
background: rgba(50, 50, 80, 0.15); |
|
|
border: 2px solid rgba(100, 100, 150, 0.3); |
|
|
border-radius: 12px; |
|
|
position: relative; |
|
|
width: 100%; |
|
|
max-width: 500px; |
|
|
} |
|
|
|
|
|
.transformer-block.active-fwd { |
|
|
border-color: rgba(53, 183, 121, 0.5); |
|
|
} |
|
|
|
|
|
.transformer-block.active-bwd { |
|
|
border-color: rgba(237, 121, 83, 0.5); |
|
|
} |
|
|
|
|
|
.block-badge { |
|
|
position: absolute; |
|
|
top: -10px; |
|
|
left: 50%; |
|
|
transform: translateX(-50%); |
|
|
background: #1a1a2e; |
|
|
padding: 2px 10px; |
|
|
font-size: 0.7rem; |
|
|
color: rgba(255, 255, 255, 0.6); |
|
|
border-radius: 4px; |
|
|
border: 1px solid rgba(100, 100, 150, 0.4); |
|
|
} |
|
|
|
|
|
.nx-indicator { |
|
|
position: absolute; |
|
|
right: -28px; |
|
|
top: 50%; |
|
|
transform: translateY(-50%); |
|
|
color: #35b779; |
|
|
font-weight: 700; |
|
|
font-size: 0.9rem; |
|
|
} |
|
|
|
|
|
|
|
|
.sub-layer { |
|
|
padding: 10px; |
|
|
border-radius: 8px; |
|
|
margin: 6px 0; |
|
|
background: rgba(255, 255, 255, 0.02); |
|
|
border: 1px solid rgba(255, 255, 255, 0.08); |
|
|
} |
|
|
|
|
|
.sub-layer.attn-layer { |
|
|
border-color: rgba(124, 58, 237, 0.3); |
|
|
} |
|
|
|
|
|
.sub-layer.ffn-layer { |
|
|
border-color: rgba(14, 165, 233, 0.3); |
|
|
} |
|
|
|
|
|
.sub-layer.active-fwd { |
|
|
background: rgba(53, 183, 121, 0.1); |
|
|
border-color: rgba(53, 183, 121, 0.4); |
|
|
} |
|
|
|
|
|
.sub-layer.active-bwd { |
|
|
background: rgba(237, 121, 83, 0.1); |
|
|
border-color: rgba(237, 121, 83, 0.4); |
|
|
} |
|
|
|
|
|
.sub-header { |
|
|
font-size: 0.75rem; |
|
|
color: rgba(255, 255, 255, 0.7); |
|
|
margin-bottom: 8px; |
|
|
text-align: center; |
|
|
} |
|
|
|
|
|
|
|
|
.attention-content { |
|
|
display: flex; |
|
|
gap: 12px; |
|
|
justify-content: center; |
|
|
flex-wrap: wrap; |
|
|
align-items: flex-start; |
|
|
} |
|
|
|
|
|
.qkv-section { |
|
|
display: flex; |
|
|
flex-direction: column; |
|
|
gap: 6px; |
|
|
} |
|
|
|
|
|
.qkv-row { |
|
|
display: flex; |
|
|
gap: 6px; |
|
|
} |
|
|
|
|
|
.qkv-item { |
|
|
display: flex; |
|
|
flex-direction: column; |
|
|
align-items: center; |
|
|
gap: 2px; |
|
|
} |
|
|
|
|
|
.qkv-tag { |
|
|
font-size: 0.55rem; |
|
|
font-weight: 700; |
|
|
padding: 1px 5px; |
|
|
border-radius: 3px; |
|
|
color: white; |
|
|
} |
|
|
|
|
|
.tag-q { background: #ef4444; } |
|
|
.tag-k { background: #22c55e; } |
|
|
.tag-v { background: #3b82f6; } |
|
|
.tag-o { background: #a855f7; } |
|
|
|
|
|
|
|
|
.attn-matrix-section { |
|
|
display: flex; |
|
|
flex-direction: column; |
|
|
align-items: center; |
|
|
gap: 4px; |
|
|
} |
|
|
|
|
|
.attn-matrix-label { |
|
|
font-size: 0.6rem; |
|
|
color: rgba(255, 255, 255, 0.5); |
|
|
} |
|
|
|
|
|
.attn-matrix { |
|
|
display: grid; |
|
|
gap: 2px; |
|
|
padding: 4px; |
|
|
background: rgba(0, 0, 0, 0.5); |
|
|
border-radius: 4px; |
|
|
border: 1px solid rgba(255, 200, 100, 0.2); |
|
|
} |
|
|
|
|
|
.attn-matrix.g6 { grid-template-columns: repeat(6, 1fr); } |
|
|
|
|
|
.attn-cell { |
|
|
width: 12px; |
|
|
height: 12px; |
|
|
border-radius: 2px; |
|
|
background: rgba(255, 255, 255, 0.05); |
|
|
transition: background 0.2s; |
|
|
} |
|
|
|
|
|
.attn-cell.masked { |
|
|
background: rgba(0, 0, 0, 0.3); |
|
|
} |
|
|
|
|
|
|
|
|
.ffn-content { |
|
|
display: flex; |
|
|
gap: 10px; |
|
|
justify-content: center; |
|
|
align-items: center; |
|
|
flex-wrap: wrap; |
|
|
} |
|
|
|
|
|
.ffn-arrow { |
|
|
color: rgba(255, 255, 255, 0.3); |
|
|
font-size: 0.8rem; |
|
|
} |
|
|
|
|
|
.act-fn-box { |
|
|
padding: 6px 12px; |
|
|
background: rgba(255, 255, 255, 0.05); |
|
|
border-radius: 6px; |
|
|
color: rgba(255, 255, 255, 0.5); |
|
|
font-size: 0.7rem; |
|
|
} |
|
|
|
|
|
|
|
|
.input-tokens { |
|
|
display: flex; |
|
|
gap: 4px; |
|
|
padding: 8px 12px; |
|
|
background: rgba(0, 0, 0, 0.4); |
|
|
border-radius: 8px; |
|
|
} |
|
|
|
|
|
.token { |
|
|
padding: 4px 10px; |
|
|
background: rgba(53, 183, 121, 0.15); |
|
|
border: 1px solid rgba(53, 183, 121, 0.3); |
|
|
border-radius: 4px; |
|
|
color: rgba(255, 255, 255, 0.8); |
|
|
font-size: 0.7rem; |
|
|
font-family: monospace; |
|
|
} |
|
|
|
|
|
|
|
|
.controls { |
|
|
display: flex; |
|
|
gap: 10px; |
|
|
margin-top: 15px; |
|
|
flex-wrap: wrap; |
|
|
justify-content: center; |
|
|
} |
|
|
|
|
|
.btn { |
|
|
padding: 9px 20px; |
|
|
border: none; |
|
|
border-radius: 6px; |
|
|
font-size: 0.8rem; |
|
|
font-weight: 600; |
|
|
cursor: pointer; |
|
|
transition: all 0.3s; |
|
|
} |
|
|
|
|
|
.btn-train { |
|
|
background: linear-gradient(135deg, #35b779, #1f9e89); |
|
|
color: white; |
|
|
} |
|
|
|
|
|
.btn-train:hover:not(:disabled) { |
|
|
transform: translateY(-2px); |
|
|
box-shadow: 0 4px 15px rgba(53, 183, 121, 0.4); |
|
|
} |
|
|
|
|
|
.btn-train:disabled { |
|
|
opacity: 0.5; |
|
|
cursor: not-allowed; |
|
|
} |
|
|
|
|
|
.btn-reset { |
|
|
background: rgba(255, 255, 255, 0.1); |
|
|
color: rgba(255, 255, 255, 0.7); |
|
|
border: 1px solid rgba(255, 255, 255, 0.2); |
|
|
} |
|
|
|
|
|
.btn-reset:hover { |
|
|
background: rgba(255, 255, 255, 0.15); |
|
|
} |
|
|
|
|
|
.speed-ctrl { |
|
|
display: flex; |
|
|
align-items: center; |
|
|
gap: 6px; |
|
|
color: rgba(255, 255, 255, 0.5); |
|
|
font-size: 0.75rem; |
|
|
} |
|
|
|
|
|
.speed-ctrl input { |
|
|
width: 60px; |
|
|
accent-color: #35b779; |
|
|
} |
|
|
|
|
|
|
|
|
.ln-mini { |
|
|
padding: 4px 8px; |
|
|
background: rgba(255, 255, 255, 0.03); |
|
|
border: 1px dashed rgba(255, 255, 255, 0.15); |
|
|
border-radius: 4px; |
|
|
font-size: 0.65rem; |
|
|
color: rgba(255, 255, 255, 0.4); |
|
|
text-align: center; |
|
|
margin: 4px 0; |
|
|
} |
|
|
|
|
|
|
|
|
.matrix-grid { |
|
|
cursor: pointer; |
|
|
position: relative; |
|
|
transition: all 0.2s; |
|
|
} |
|
|
|
|
|
.matrix-grid:hover { |
|
|
transform: scale(1.05); |
|
|
box-shadow: 0 0 10px rgba(53, 183, 121, 0.3); |
|
|
} |
|
|
|
|
|
.matrix-grid::after { |
|
|
content: '🔍'; |
|
|
position: absolute; |
|
|
top: 2px; |
|
|
right: 2px; |
|
|
font-size: 8px; |
|
|
opacity: 0; |
|
|
transition: opacity 0.2s; |
|
|
} |
|
|
|
|
|
.matrix-grid:hover::after { |
|
|
opacity: 0.6; |
|
|
} |
|
|
|
|
|
|
|
|
.matrix-modal { |
|
|
display: none; |
|
|
position: fixed; |
|
|
top: 0; |
|
|
left: 0; |
|
|
width: 100%; |
|
|
height: 100%; |
|
|
background: rgba(0, 0, 0, 0.9); |
|
|
z-index: 1000; |
|
|
justify-content: center; |
|
|
align-items: center; |
|
|
padding: 20px; |
|
|
overflow: auto; |
|
|
} |
|
|
|
|
|
.matrix-modal.active { |
|
|
display: flex; |
|
|
} |
|
|
|
|
|
.matrix-modal-content { |
|
|
background: rgba(26, 26, 46, 0.95); |
|
|
border: 2px solid rgba(100, 100, 150, 0.5); |
|
|
border-radius: 12px; |
|
|
padding: 20px; |
|
|
max-width: 90%; |
|
|
max-height: 90%; |
|
|
overflow: auto; |
|
|
position: relative; |
|
|
} |
|
|
|
|
|
.matrix-modal-header { |
|
|
display: flex; |
|
|
justify-content: space-between; |
|
|
align-items: center; |
|
|
margin-bottom: 15px; |
|
|
padding-bottom: 10px; |
|
|
border-bottom: 1px solid rgba(255, 255, 255, 0.2); |
|
|
} |
|
|
|
|
|
.matrix-modal-title { |
|
|
color: rgba(255, 255, 255, 0.9); |
|
|
font-size: 1rem; |
|
|
font-weight: 600; |
|
|
} |
|
|
|
|
|
.matrix-modal-close { |
|
|
background: rgba(255, 255, 255, 0.1); |
|
|
border: 1px solid rgba(255, 255, 255, 0.2); |
|
|
color: rgba(255, 255, 255, 0.7); |
|
|
padding: 6px 12px; |
|
|
border-radius: 6px; |
|
|
cursor: pointer; |
|
|
font-size: 0.8rem; |
|
|
transition: all 0.2s; |
|
|
} |
|
|
|
|
|
.matrix-modal-close:hover { |
|
|
background: rgba(255, 255, 255, 0.2); |
|
|
color: white; |
|
|
} |
|
|
|
|
|
.matrix-modal-info { |
|
|
color: rgba(255, 255, 255, 0.5); |
|
|
font-size: 0.75rem; |
|
|
margin-bottom: 15px; |
|
|
text-align: center; |
|
|
} |
|
|
|
|
|
.full-matrix-container { |
|
|
display: flex; |
|
|
justify-content: center; |
|
|
align-items: center; |
|
|
} |
|
|
|
|
|
.full-matrix-grid { |
|
|
display: grid; |
|
|
gap: 1px; |
|
|
padding: 4px; |
|
|
background: rgba(0, 0, 0, 0.6); |
|
|
border-radius: 4px; |
|
|
} |
|
|
|
|
|
.full-matrix-grid .m-cell { |
|
|
width: 6px; |
|
|
height: 6px; |
|
|
} |
|
|
|
|
|
|
|
|
.tabs-container { |
|
|
width: 100%; |
|
|
max-width: 1400px; |
|
|
margin-bottom: 15px; |
|
|
} |
|
|
|
|
|
.tabs-nav { |
|
|
display: flex; |
|
|
gap: 8px; |
|
|
padding: 0 10px; |
|
|
border-bottom: 2px solid rgba(255, 255, 255, 0.1); |
|
|
} |
|
|
|
|
|
.tab-btn { |
|
|
padding: 10px 20px; |
|
|
background: rgba(0, 0, 0, 0.3); |
|
|
border: none; |
|
|
border-top-left-radius: 8px; |
|
|
border-top-right-radius: 8px; |
|
|
color: rgba(255, 255, 255, 0.5); |
|
|
font-size: 0.85rem; |
|
|
font-weight: 500; |
|
|
cursor: pointer; |
|
|
transition: all 0.2s ease; |
|
|
border-bottom: 2px solid transparent; |
|
|
margin-bottom: -2px; |
|
|
} |
|
|
|
|
|
.tab-btn:hover { |
|
|
background: rgba(0, 0, 0, 0.5); |
|
|
color: rgba(255, 255, 255, 0.8); |
|
|
} |
|
|
|
|
|
.tab-btn.active { |
|
|
background: rgba(30, 60, 100, 0.4); |
|
|
color: #fff; |
|
|
border-bottom: 2px solid #35b779; |
|
|
} |
|
|
|
|
|
.tab-content { |
|
|
display: none; |
|
|
} |
|
|
|
|
|
.tab-content.active { |
|
|
display: block; |
|
|
} |
|
|
|
|
|
|
|
|
.component-test-controls { |
|
|
margin-top: 20px; |
|
|
padding: 15px; |
|
|
background: rgba(0, 0, 0, 0.3); |
|
|
border-radius: 8px; |
|
|
display: flex; |
|
|
gap: 10px; |
|
|
justify-content: center; |
|
|
align-items: center; |
|
|
border: 1px solid rgba(255, 255, 255, 0.1); |
|
|
} |
|
|
|
|
|
.test-btn { |
|
|
padding: 10px 20px; |
|
|
background: rgba(30, 60, 100, 0.5); |
|
|
border: 1px solid rgba(255, 255, 255, 0.2); |
|
|
border-radius: 6px; |
|
|
color: #fff; |
|
|
font-size: 0.85rem; |
|
|
cursor: pointer; |
|
|
transition: all 0.2s ease; |
|
|
} |
|
|
|
|
|
.test-btn:hover { |
|
|
background: rgba(30, 60, 100, 0.7); |
|
|
border-color: rgba(255, 255, 255, 0.4); |
|
|
} |
|
|
|
|
|
.test-btn:disabled { |
|
|
opacity: 0.5; |
|
|
cursor: not-allowed; |
|
|
} |
|
|
|
|
|
.test-btn.fwd { |
|
|
background: rgba(31, 158, 137, 0.3); |
|
|
border-color: rgba(53, 183, 121, 0.5); |
|
|
} |
|
|
|
|
|
.test-btn.fwd:hover:not(:disabled) { |
|
|
background: rgba(31, 158, 137, 0.5); |
|
|
} |
|
|
|
|
|
.test-btn.bwd { |
|
|
background: rgba(189, 55, 134, 0.3); |
|
|
border-color: rgba(237, 121, 83, 0.5); |
|
|
} |
|
|
|
|
|
.test-btn.bwd:hover:not(:disabled) { |
|
|
background: rgba(189, 55, 134, 0.5); |
|
|
} |
|
|
</style> |
|
|
</head> |
|
|
<body> |
|
|
<h1>GPT Training Visualization</h1> |
|
|
<p class="subtitle">Weights · Gradients · Activations · Attention</p> |
|
|
|
|
|
|
|
|
<div class="colormap-legend"> |
|
|
<div class="cmap-item"> |
|
|
<span class="cmap-label">Weights (Viridis)</span> |
|
|
<div class="cmap-bar cmap-viridis"></div> |
|
|
<div class="cmap-range"><span>−</span><span>+</span></div> |
|
|
</div> |
|
|
<div class="cmap-item"> |
|
|
<span class="cmap-label">Gradients (Plasma)</span> |
|
|
<div class="cmap-bar cmap-plasma"></div> |
|
|
<div class="cmap-range"><span>−</span><span>+</span></div> |
|
|
</div> |
|
|
<div class="cmap-item"> |
|
|
<span class="cmap-label">Activations (Coolwarm)</span> |
|
|
<div class="cmap-bar cmap-coolwarm"></div> |
|
|
<div class="cmap-range"><span>−</span><span>+</span></div> |
|
|
</div> |
|
|
<div class="cmap-item"> |
|
|
<span class="cmap-label">Attention (Inferno)</span> |
|
|
<div class="cmap-bar cmap-inferno"></div> |
|
|
<div class="cmap-range"><span>0</span><span>1</span></div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="tabs-container"> |
|
|
<div class="tabs-nav"> |
|
|
<button class="tab-btn active" data-tab="full-view">Full Training View</button> |
|
|
<button class="tab-btn" data-tab="attention-component">Multi-Head Self-Attention</button> |
|
|
<button class="tab-btn" data-tab="ffn-component">Feed-Forward Network</button> |
|
|
<button class="tab-btn" data-tab="transformer-block">Transformer Block</button> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="tab-content active" id="tab-full-view"> |
|
|
<div class="main-layout"> |
|
|
|
|
|
<div class="model-column"> |
|
|
|
|
|
<div class="layer-container"> |
|
|
<span class="layer-label">Loss</span> |
|
|
<div class="layer-box" id="loss-box"> |
|
|
<div class="layer-header"> |
|
|
<span class="layer-title">Cross-Entropy Loss</span> |
|
|
<span class="layer-dims">scalar</span> |
|
|
</div> |
|
|
<div style="text-align: center; color: rgba(255,255,255,0.4); font-size: 0.7rem;"> |
|
|
L = −Σᵢ yᵢ log(ŷᵢ) |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="flow-arrow" id="arrow-0"></div> |
|
|
<div class="layer-container"> |
|
|
<span class="layer-label">Logits [B, V]</span> |
|
|
<div class="activation-stream" id="act-logits"></div> |
|
|
</div> |
|
|
|
|
|
<div class="flow-arrow" id="arrow-1"></div> |
|
|
|
|
|
|
|
|
<div class="layer-container"> |
|
|
<div class="layer-box" id="lm-head-box"> |
|
|
<div class="layer-header"> |
|
|
<span class="layer-title">LM Head (Linear)</span> |
|
|
<span class="layer-dims">[d_model, vocab]</span> |
|
|
</div> |
|
|
<div class="weight-grad-container"> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">Weights</span> |
|
|
<div class="matrix-grid g10" id="lm-w"></div> |
|
|
</div> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">Gradients</span> |
|
|
<div class="matrix-grid g10" id="lm-g"></div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="flow-arrow" id="arrow-2"></div> |
|
|
<div class="layer-container"> |
|
|
<span class="layer-label">Hidden [B, T, D]</span> |
|
|
<div class="activation-stream" id="act-hidden"></div> |
|
|
</div> |
|
|
|
|
|
<div class="flow-arrow" id="arrow-3"></div> |
|
|
|
|
|
|
|
|
<div class="transformer-block" id="xformer-block"> |
|
|
<span class="block-badge">Transformer Block</span> |
|
|
<span class="nx-indicator">×12</span> |
|
|
|
|
|
<div class="ln-mini">LayerNorm</div> |
|
|
|
|
|
|
|
|
<div class="sub-layer ffn-layer" id="ffn-layer"> |
|
|
<div class="sub-header">Feed-Forward Network</div> |
|
|
|
|
|
|
|
|
<div class="layer-container" style="margin-bottom: 8px;"> |
|
|
<span class="layer-label" style="font-size: 0.55rem;">FFN Output</span> |
|
|
<div class="activation-stream" id="act-ffn-out" style="max-width: 300px;"></div> |
|
|
</div> |
|
|
|
|
|
<div class="ffn-content"> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">W₁ weights</span> |
|
|
<div class="matrix-grid g8" id="ffn-w1-w"></div> |
|
|
</div> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">W₁ grads</span> |
|
|
<div class="matrix-grid g8" id="ffn-w1-g"></div> |
|
|
</div> |
|
|
<span class="ffn-arrow">→</span> |
|
|
<div class="act-fn-box">GELU</div> |
|
|
<span class="ffn-arrow">→</span> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">W₂ weights</span> |
|
|
<div class="matrix-grid g8" id="ffn-w2-w"></div> |
|
|
</div> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">W₂ grads</span> |
|
|
<div class="matrix-grid g8" id="ffn-w2-g"></div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="layer-container" style="margin-top: 8px;"> |
|
|
<span class="layer-label" style="font-size: 0.55rem;">FFN Input</span> |
|
|
<div class="activation-stream" id="act-ffn-in" style="max-width: 300px;"></div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
<div class="ln-mini">LayerNorm</div> |
|
|
|
|
|
|
|
|
<div class="sub-layer attn-layer" id="attn-layer"> |
|
|
<div class="sub-header">Multi-Head Self-Attention</div> |
|
|
|
|
|
|
|
|
<div class="layer-container" style="margin-bottom: 8px;"> |
|
|
<span class="layer-label" style="font-size: 0.55rem;">Attn Output</span> |
|
|
<div class="activation-stream" id="act-attn-out" style="max-width: 300px;"></div> |
|
|
</div> |
|
|
|
|
|
<div class="attention-content"> |
|
|
<div class="qkv-section"> |
|
|
<div class="qkv-row"> |
|
|
<div class="qkv-item"> |
|
|
<span class="qkv-tag tag-q">Wq</span> |
|
|
<div class="matrix-grid g6" id="wq-w"></div> |
|
|
<div class="matrix-grid g6" id="wq-g" style="margin-top: 2px;"></div> |
|
|
</div> |
|
|
<div class="qkv-item"> |
|
|
<span class="qkv-tag tag-k">Wk</span> |
|
|
<div class="matrix-grid g6" id="wk-w"></div> |
|
|
<div class="matrix-grid g6" id="wk-g" style="margin-top: 2px;"></div> |
|
|
</div> |
|
|
<div class="qkv-item"> |
|
|
<span class="qkv-tag tag-v">Wv</span> |
|
|
<div class="matrix-grid g6" id="wv-w"></div> |
|
|
<div class="matrix-grid g6" id="wv-g" style="margin-top: 2px;"></div> |
|
|
</div> |
|
|
<div class="qkv-item"> |
|
|
<span class="qkv-tag tag-o">Wo</span> |
|
|
<div class="matrix-grid g6" id="wo-w"></div> |
|
|
<div class="matrix-grid g6" id="wo-g" style="margin-top: 2px;"></div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="attn-matrix-section"> |
|
|
<span class="attn-matrix-label">Attention Matrix (QKᵀ/√d)</span> |
|
|
<div class="attn-matrix g6" id="attn-matrix"></div> |
|
|
<span class="attn-matrix-label" style="font-size: 0.5rem; opacity: 0.5;">causal mask applied</span> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="flow-arrow" id="arrow-4"></div> |
|
|
<div class="layer-container"> |
|
|
<span class="layer-label">Embedded [B, T, D]</span> |
|
|
<div class="activation-stream" id="act-emb"></div> |
|
|
</div> |
|
|
|
|
|
<div class="flow-arrow" id="arrow-5"></div> |
|
|
|
|
|
|
|
|
<div class="layer-container"> |
|
|
<div class="layer-box" id="emb-box"> |
|
|
<div class="layer-header"> |
|
|
<span class="layer-title">Token Embedding</span> |
|
|
<span class="layer-dims">[vocab, d_model]</span> |
|
|
</div> |
|
|
<div class="weight-grad-container"> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">Weights</span> |
|
|
<div class="matrix-grid g10" id="emb-w"></div> |
|
|
</div> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">Gradients</span> |
|
|
<div class="matrix-grid g10" id="emb-g"></div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
<div class="flow-arrow" id="arrow-6"></div> |
|
|
|
|
|
|
|
|
<div class="layer-container"> |
|
|
<span class="layer-label">Input Tokens</span> |
|
|
<div class="input-tokens"> |
|
|
<div class="token">The</div> |
|
|
<div class="token">quick</div> |
|
|
<div class="token">brown</div> |
|
|
<div class="token">fox</div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="side-panel"> |
|
|
<div class="panel-title">Training Monitor</div> |
|
|
|
|
|
<div class="phase-row"> |
|
|
<div class="phase-pill" id="ph-fwd">Forward</div> |
|
|
<div class="phase-pill" id="ph-bwd">Backward</div> |
|
|
<div class="phase-pill" id="ph-upd">Update</div> |
|
|
</div> |
|
|
|
|
|
<div class="loss-box"> |
|
|
<span class="loss-lbl">Loss</span> |
|
|
<span class="loss-val" id="loss-val">2.891</span> |
|
|
</div> |
|
|
|
|
|
<div class="stats-row"> |
|
|
<div class="stat-box"> |
|
|
<div class="stat-val" id="step-val">0</div> |
|
|
<div class="stat-lbl">Step</div> |
|
|
</div> |
|
|
<div class="stat-box"> |
|
|
<div class="stat-val" id="lr-val">3e-4</div> |
|
|
<div class="stat-lbl">LR</div> |
|
|
</div> |
|
|
<div class="stat-box"> |
|
|
<div class="stat-val" id="norm-val">0.00</div> |
|
|
<div class="stat-lbl">∇ Norm</div> |
|
|
</div> |
|
|
<div class="stat-box"> |
|
|
<div class="stat-val" id="tok-val">0</div> |
|
|
<div class="stat-lbl">Tokens</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
<div class="status-box"> |
|
|
<div class="status-main" id="status-main">Ready</div> |
|
|
<div class="status-sub" id="status-sub">Click Train to begin</div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
|
|
|
<div class="tab-content" id="tab-attention-component"> |
|
|
<div class="main-layout"> |
|
|
<div class="model-column" style="align-items: center;"> |
|
|
|
|
|
<div class="sub-layer attn-layer" id="attn-layer-tab"> |
|
|
<div class="sub-header">Multi-Head Self-Attention</div> |
|
|
|
|
|
|
|
|
<div class="layer-container" style="margin-bottom: 8px;"> |
|
|
<span class="layer-label" style="font-size: 0.55rem;">Attn Output</span> |
|
|
<div class="activation-stream" id="act-attn-out-tab" style="max-width: 300px;"></div> |
|
|
</div> |
|
|
|
|
|
<div class="attention-content"> |
|
|
<div class="qkv-section"> |
|
|
<div class="qkv-row"> |
|
|
<div class="qkv-item"> |
|
|
<span class="qkv-tag tag-q">Wq</span> |
|
|
<div class="matrix-grid g6" id="wq-w-tab"></div> |
|
|
<div class="matrix-grid g6" id="wq-g-tab" style="margin-top: 2px;"></div> |
|
|
</div> |
|
|
<div class="qkv-item"> |
|
|
<span class="qkv-tag tag-k">Wk</span> |
|
|
<div class="matrix-grid g6" id="wk-w-tab"></div> |
|
|
<div class="matrix-grid g6" id="wk-g-tab" style="margin-top: 2px;"></div> |
|
|
</div> |
|
|
<div class="qkv-item"> |
|
|
<span class="qkv-tag tag-v">Wv</span> |
|
|
<div class="matrix-grid g6" id="wv-w-tab"></div> |
|
|
<div class="matrix-grid g6" id="wv-g-tab" style="margin-top: 2px;"></div> |
|
|
</div> |
|
|
<div class="qkv-item"> |
|
|
<span class="qkv-tag tag-o">Wo</span> |
|
|
<div class="matrix-grid g6" id="wo-w-tab"></div> |
|
|
<div class="matrix-grid g6" id="wo-g-tab" style="margin-top: 2px;"></div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="attn-matrix-section"> |
|
|
<span class="attn-matrix-label">Attention Matrix (QKᵀ/√d)</span> |
|
|
<div class="attn-matrix g6" id="attn-matrix-tab"></div> |
|
|
<span class="attn-matrix-label" style="font-size: 0.5rem; opacity: 0.5;">causal mask applied</span> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="component-test-controls"> |
|
|
<button class="test-btn fwd" id="btn-attn-fwd">→ Forward Pass</button> |
|
|
<button class="test-btn bwd" id="btn-attn-bwd">← Backward Pass</button> |
|
|
<button class="test-btn" id="btn-attn-update">⟳ Update Weights</button> |
|
|
<button class="test-btn" id="btn-attn-reset">↺ Reset</button> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
|
|
|
<div class="tab-content" id="tab-ffn-component"> |
|
|
<div class="main-layout"> |
|
|
<div class="model-column" style="align-items: center;"> |
|
|
|
|
|
<div class="sub-layer ffn-layer" id="ffn-layer-tab"> |
|
|
<div class="sub-header">Feed-Forward Network</div> |
|
|
|
|
|
|
|
|
<div class="layer-container" style="margin-bottom: 8px;"> |
|
|
<span class="layer-label" style="font-size: 0.55rem;">FFN Output</span> |
|
|
<div class="activation-stream" id="act-ffn-out-tab" style="max-width: 300px;"></div> |
|
|
</div> |
|
|
|
|
|
<div class="ffn-content"> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">W₁ weights</span> |
|
|
<div class="matrix-grid g8" id="ffn-w1-w-tab"></div> |
|
|
</div> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">W₁ grads</span> |
|
|
<div class="matrix-grid g8" id="ffn-w1-g-tab"></div> |
|
|
</div> |
|
|
<span class="ffn-arrow">→</span> |
|
|
<div class="act-fn-box">GELU</div> |
|
|
<span class="ffn-arrow">→</span> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">W₂ weights</span> |
|
|
<div class="matrix-grid g8" id="ffn-w2-w-tab"></div> |
|
|
</div> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">W₂ grads</span> |
|
|
<div class="matrix-grid g8" id="ffn-w2-g-tab"></div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="layer-container" style="margin-top: 8px;"> |
|
|
<span class="layer-label" style="font-size: 0.55rem;">FFN Input</span> |
|
|
<div class="activation-stream" id="act-ffn-in-tab" style="max-width: 300px;"></div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="component-test-controls"> |
|
|
<button class="test-btn fwd" id="btn-ffn-fwd">→ Forward Pass</button> |
|
|
<button class="test-btn bwd" id="btn-ffn-bwd">← Backward Pass</button> |
|
|
<button class="test-btn" id="btn-ffn-update">⟳ Update Weights</button> |
|
|
<button class="test-btn" id="btn-ffn-reset">↺ Reset</button> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
|
|
|
<div class="tab-content" id="tab-transformer-block"> |
|
|
<div class="main-layout"> |
|
|
<div class="model-column" style="align-items: center;"> |
|
|
|
|
|
<div class="transformer-block" id="xformer-block-tab"> |
|
|
<span class="block-badge">Transformer Block</span> |
|
|
<span class="nx-indicator">×12</span> |
|
|
|
|
|
<div class="ln-mini">LayerNorm</div> |
|
|
|
|
|
|
|
|
<div class="sub-layer ffn-layer" id="ffn-layer-tab2"> |
|
|
<div class="sub-header">Feed-Forward Network</div> |
|
|
|
|
|
|
|
|
<div class="layer-container" style="margin-bottom: 8px;"> |
|
|
<span class="layer-label" style="font-size: 0.55rem;">FFN Output</span> |
|
|
<div class="activation-stream" id="act-ffn-out-tab2" style="max-width: 300px;"></div> |
|
|
</div> |
|
|
|
|
|
<div class="ffn-content"> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">W₁ weights</span> |
|
|
<div class="matrix-grid g8" id="ffn-w1-w-tab2"></div> |
|
|
</div> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">W₁ grads</span> |
|
|
<div class="matrix-grid g8" id="ffn-w1-g-tab2"></div> |
|
|
</div> |
|
|
<span class="ffn-arrow">→</span> |
|
|
<div class="act-fn-box">GELU</div> |
|
|
<span class="ffn-arrow">→</span> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">W₂ weights</span> |
|
|
<div class="matrix-grid g8" id="ffn-w2-w-tab2"></div> |
|
|
</div> |
|
|
<div class="matrix-group"> |
|
|
<span class="matrix-label">W₂ grads</span> |
|
|
<div class="matrix-grid g8" id="ffn-w2-g-tab2"></div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="layer-container" style="margin-top: 8px;"> |
|
|
<span class="layer-label" style="font-size: 0.55rem;">FFN Input</span> |
|
|
<div class="activation-stream" id="act-ffn-in-tab2" style="max-width: 300px;"></div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
<div class="ln-mini">LayerNorm</div> |
|
|
|
|
|
|
|
|
<div class="sub-layer attn-layer" id="attn-layer-tab2"> |
|
|
<div class="sub-header">Multi-Head Self-Attention</div> |
|
|
|
|
|
|
|
|
<div class="layer-container" style="margin-bottom: 8px;"> |
|
|
<span class="layer-label" style="font-size: 0.55rem;">Attn Output</span> |
|
|
<div class="activation-stream" id="act-attn-out-tab2" style="max-width: 300px;"></div> |
|
|
</div> |
|
|
|
|
|
<div class="attention-content"> |
|
|
<div class="qkv-section"> |
|
|
<div class="qkv-row"> |
|
|
<div class="qkv-item"> |
|
|
<span class="qkv-tag tag-q">Wq</span> |
|
|
<div class="matrix-grid g6" id="wq-w-tab2"></div> |
|
|
<div class="matrix-grid g6" id="wq-g-tab2" style="margin-top: 2px;"></div> |
|
|
</div> |
|
|
<div class="qkv-item"> |
|
|
<span class="qkv-tag tag-k">Wk</span> |
|
|
<div class="matrix-grid g6" id="wk-w-tab2"></div> |
|
|
<div class="matrix-grid g6" id="wk-g-tab2" style="margin-top: 2px;"></div> |
|
|
</div> |
|
|
<div class="qkv-item"> |
|
|
<span class="qkv-tag tag-v">Wv</span> |
|
|
<div class="matrix-grid g6" id="wv-w-tab2"></div> |
|
|
<div class="matrix-grid g6" id="wv-g-tab2" style="margin-top: 2px;"></div> |
|
|
</div> |
|
|
<div class="qkv-item"> |
|
|
<span class="qkv-tag tag-o">Wo</span> |
|
|
<div class="matrix-grid g6" id="wo-w-tab2"></div> |
|
|
<div class="matrix-grid g6" id="wo-g-tab2" style="margin-top: 2px;"></div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="attn-matrix-section"> |
|
|
<span class="attn-matrix-label">Attention Matrix (QKᵀ/√d)</span> |
|
|
<div class="attn-matrix g6" id="attn-matrix-tab2"></div> |
|
|
<span class="attn-matrix-label" style="font-size: 0.5rem; opacity: 0.5;">causal mask applied</span> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="component-test-controls"> |
|
|
<button class="test-btn fwd" id="btn-xformer-fwd">→ Forward Pass</button> |
|
|
<button class="test-btn bwd" id="btn-xformer-bwd">← Backward Pass</button> |
|
|
<button class="test-btn" id="btn-xformer-update">⟳ Update Weights</button> |
|
|
<button class="test-btn" id="btn-xformer-reset">↺ Reset</button> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="controls"> |
|
|
<button class="btn btn-train" id="btn-train">⚡ Train Step</button> |
|
|
<button class="btn btn-reset" id="btn-reset">↺ Reset</button> |
|
|
<div class="speed-ctrl"> |
|
|
<span>Speed:</span> |
|
|
<input type="range" id="speed" min="1" max="5" value="3"> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
|
|
|
<div class="matrix-modal" id="matrix-modal"> |
|
|
<div class="matrix-modal-content"> |
|
|
<div class="matrix-modal-header"> |
|
|
<span class="matrix-modal-title" id="modal-title">Full Weight Matrix</span> |
|
|
<button class="matrix-modal-close" id="modal-close">Close (Esc)</button> |
|
|
</div> |
|
|
<div class="matrix-modal-info" id="modal-info"> |
|
|
Full matrix dimensions: <span id="modal-dims"></span> |
|
|
</div> |
|
|
<div class="full-matrix-container"> |
|
|
<div class="full-matrix-grid" id="full-matrix"></div> |
|
|
</div> |
|
|
</div> |
|
|
</div> |
|
|
|
|
|
<script> |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const viridis = ['#440154','#482878','#3e4a89','#31688e','#26838f','#1f9e89','#35b779','#6ece58','#b5de2b','#fde725']; |
|
|
const plasma = ['#0d0887','#46039f','#7201a8','#9c179e','#bd3786','#d8576b','#ed7953','#fb9f3a','#fdca26','#f0f921']; |
|
|
const coolwarm = ['#3b4cc0','#5977e3','#7b9ff9','#9ebeff','#c0d4f5','#dddcdc','#f2cbb7','#f7ac8e','#ee8468','#d65244','#b40426']; |
|
|
const inferno = ['#000004','#1b0c41','#4a0c6b','#781c6d','#a52c60','#cf4446','#ed6925','#fb9b06','#f7d13d','#fcffa4']; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const matrixConfig = { |
|
|
|
|
|
'lm-w': { true: [100, 100], display: [5, 10], label: 'LM Head Weights' }, |
|
|
'lm-g': { true: [100, 100], display: [5, 10], label: 'LM Head Gradients' }, |
|
|
'ffn-w1-w': { true: [100, 100], display: [5, 8], label: 'FFN W₁ Weights' }, |
|
|
'ffn-w1-g': { true: [100, 100], display: [5, 8], label: 'FFN W₁ Gradients' }, |
|
|
'ffn-w2-w': { true: [100, 100], display: [5, 8], label: 'FFN W₂ Weights' }, |
|
|
'ffn-w2-g': { true: [100, 100], display: [5, 8], label: 'FFN W₂ Gradients' }, |
|
|
'wq-w': { true: [100, 100], display: [4, 6], label: 'Query Weights (Wq)' }, |
|
|
'wq-g': { true: [100, 100], display: [4, 6], label: 'Query Gradients (Wq)' }, |
|
|
'wk-w': { true: [100, 100], display: [4, 6], label: 'Key Weights (Wk)' }, |
|
|
'wk-g': { true: [100, 100], display: [4, 6], label: 'Key Gradients (Wk)' }, |
|
|
'wv-w': { true: [100, 100], display: [4, 6], label: 'Value Weights (Wv)' }, |
|
|
'wv-g': { true: [100, 100], display: [4, 6], label: 'Value Gradients (Wv)' }, |
|
|
'wo-w': { true: [100, 100], display: [4, 6], label: 'Output Weights (Wo)' }, |
|
|
'wo-g': { true: [100, 100], display: [4, 6], label: 'Output Gradients (Wo)' }, |
|
|
'emb-w': { true: [100, 100], display: [5, 10], label: 'Embedding Weights' }, |
|
|
'emb-g': { true: [100, 100], display: [5, 10], label: 'Embedding Gradients' }, |
|
|
|
|
|
|
|
|
'wq-w-tab': { true: [100, 100], display: [4, 6], label: 'Query Weights (Wq)' }, |
|
|
'wq-g-tab': { true: [100, 100], display: [4, 6], label: 'Query Gradients (Wq)' }, |
|
|
'wk-w-tab': { true: [100, 100], display: [4, 6], label: 'Key Weights (Wk)' }, |
|
|
'wk-g-tab': { true: [100, 100], display: [4, 6], label: 'Key Gradients (Wk)' }, |
|
|
'wv-w-tab': { true: [100, 100], display: [4, 6], label: 'Value Weights (Wv)' }, |
|
|
'wv-g-tab': { true: [100, 100], display: [4, 6], label: 'Value Gradients (Wv)' }, |
|
|
'wo-w-tab': { true: [100, 100], display: [4, 6], label: 'Output Weights (Wo)' }, |
|
|
'wo-g-tab': { true: [100, 100], display: [4, 6], label: 'Output Gradients (Wo)' }, |
|
|
|
|
|
|
|
|
'ffn-w1-w-tab': { true: [100, 100], display: [5, 8], label: 'FFN W₁ Weights' }, |
|
|
'ffn-w1-g-tab': { true: [100, 100], display: [5, 8], label: 'FFN W₁ Gradients' }, |
|
|
'ffn-w2-w-tab': { true: [100, 100], display: [5, 8], label: 'FFN W₂ Weights' }, |
|
|
'ffn-w2-g-tab': { true: [100, 100], display: [5, 8], label: 'FFN W₂ Gradients' }, |
|
|
|
|
|
|
|
|
'wq-w-tab2': { true: [100, 100], display: [4, 6], label: 'Query Weights (Wq)' }, |
|
|
'wq-g-tab2': { true: [100, 100], display: [4, 6], label: 'Query Gradients (Wq)' }, |
|
|
'wk-w-tab2': { true: [100, 100], display: [4, 6], label: 'Key Weights (Wk)' }, |
|
|
'wk-g-tab2': { true: [100, 100], display: [4, 6], label: 'Key Gradients (Wk)' }, |
|
|
'wv-w-tab2': { true: [100, 100], display: [4, 6], label: 'Value Weights (Wv)' }, |
|
|
'wv-g-tab2': { true: [100, 100], display: [4, 6], label: 'Value Gradients (Wv)' }, |
|
|
'wo-w-tab2': { true: [100, 100], display: [4, 6], label: 'Output Weights (Wo)' }, |
|
|
'wo-g-tab2': { true: [100, 100], display: [4, 6], label: 'Output Gradients (Wo)' }, |
|
|
'ffn-w1-w-tab2': { true: [100, 100], display: [5, 8], label: 'FFN W₁ Weights' }, |
|
|
'ffn-w1-g-tab2': { true: [100, 100], display: [5, 8], label: 'FFN W₁ Gradients' }, |
|
|
'ffn-w2-w-tab2': { true: [100, 100], display: [5, 8], label: 'FFN W₂ Weights' }, |
|
|
'ffn-w2-g-tab2': { true: [100, 100], display: [5, 8], label: 'FFN W₂ Gradients' } |
|
|
}; |
|
|
|
|
|
|
|
|
const CONFIG = { |
|
|
|
|
|
timing: { |
|
|
embedding: 200, |
|
|
embeddingActivation: 300, |
|
|
transformerBlock: 200, |
|
|
attention: 250, |
|
|
attentionEffect: 350, |
|
|
ffn: 250, |
|
|
ffnEffect: 300, |
|
|
hidden: 250, |
|
|
lmHead: 250, |
|
|
logits: 300, |
|
|
loss: 400, |
|
|
backwardDelay: 200, |
|
|
updateDelay: 300, |
|
|
cleanupDelay: 500 |
|
|
}, |
|
|
|
|
|
|
|
|
gradientIntensity: { |
|
|
lmHead: 0.75, |
|
|
ffn: 0.7, |
|
|
attention: 0.65, |
|
|
embedding: 0.4 |
|
|
}, |
|
|
|
|
|
|
|
|
activationIntensity: 0.8, |
|
|
|
|
|
|
|
|
streams: { |
|
|
|
|
|
'act-logits': 32, |
|
|
'act-hidden': 28, |
|
|
'act-ffn-in': 20, |
|
|
'act-ffn-out': 20, |
|
|
'act-attn-out': 20, |
|
|
'act-emb': 28, |
|
|
|
|
|
'act-attn-out-tab': 20, |
|
|
|
|
|
'act-ffn-in-tab': 20, |
|
|
'act-ffn-out-tab': 20, |
|
|
|
|
|
'act-attn-out-tab2': 20, |
|
|
'act-ffn-in-tab2': 20, |
|
|
'act-ffn-out-tab2': 20 |
|
|
}, |
|
|
|
|
|
|
|
|
attentionMatrixSize: 6, |
|
|
attentionMatrices: ['attn-matrix', 'attn-matrix-tab', 'attn-matrix-tab2'], |
|
|
|
|
|
|
|
|
loss: { |
|
|
minValue: 0.12, |
|
|
decreaseBase: 0.03, |
|
|
decreaseRange: 0.1 |
|
|
}, |
|
|
|
|
|
|
|
|
gradNorm: { |
|
|
min: 0.4, |
|
|
range: 1.2 |
|
|
}, |
|
|
|
|
|
|
|
|
tokensPerStep: 16, |
|
|
learningRate: 0.001, |
|
|
|
|
|
|
|
|
speedMultiplier: 6, |
|
|
|
|
|
|
|
|
attention: { |
|
|
decayCoefficient: 0.3, |
|
|
noiseFactor: 0.3 |
|
|
} |
|
|
}; |
|
|
|
|
|
|
|
|
const componentRegistry = {}; |
|
|
|
|
|
function registerComponent(instance) { |
|
|
Object.values(instance.ids).forEach(id => { |
|
|
if (id) componentRegistry[id] = instance; |
|
|
}); |
|
|
} |
|
|
|
|
|
function getColor(cmap, value) { |
|
|
|
|
|
const idx = Math.min(Math.floor(value * cmap.length), cmap.length - 1); |
|
|
return cmap[Math.max(0, idx)]; |
|
|
} |
|
|
|
|
|
function randomFromCmap(cmap) { |
|
|
return getColor(cmap, Math.random()); |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let running = false; |
|
|
let timer = null; |
|
|
let speed = 1; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
document.getElementById('speed').addEventListener('input', e => { |
|
|
speed = CONFIG.speedMultiplier - parseInt(e.target.value); |
|
|
}); |
|
|
|
|
|
function wait(ms) { |
|
|
return new Promise(r => timer = setTimeout(r, ms * speed)); |
|
|
} |
|
|
|
|
|
function setStatus(main, sub) { |
|
|
document.getElementById('status-main').textContent = main; |
|
|
document.getElementById('status-sub').textContent = sub; |
|
|
} |
|
|
|
|
|
function setPhase(phase) { |
|
|
['fwd', 'bwd', 'upd'].forEach(p => { |
|
|
const el = document.getElementById(`ph-${p}`); |
|
|
el.classList.remove('active', 'fwd', 'bwd', 'upd'); |
|
|
}); |
|
|
if (phase) { |
|
|
const el = document.getElementById(`ph-${phase}`); |
|
|
el.classList.add('active', phase); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function initGrid(id, rows, cols) { |
|
|
const el = document.getElementById(id); |
|
|
if (!el) return; |
|
|
el.innerHTML = ''; |
|
|
|
|
|
|
|
|
for (let i = 0; i < rows * cols; i++) { |
|
|
const cell = document.createElement('div'); |
|
|
cell.className = 'm-cell'; |
|
|
el.appendChild(cell); |
|
|
} |
|
|
|
|
|
|
|
|
if (matrixConfig[id]) { |
|
|
el.style.cursor = 'pointer'; |
|
|
el.addEventListener('click', () => openMatrixModal(id)); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
function initStream(id, count) { |
|
|
const el = document.getElementById(id); |
|
|
if (!el) return; |
|
|
el.innerHTML = ''; |
|
|
for (let i = 0; i < count; i++) { |
|
|
const cell = document.createElement('div'); |
|
|
cell.className = 'act-cell'; |
|
|
el.appendChild(cell); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
function initAttnMatrix() { |
|
|
|
|
|
CONFIG.attentionMatrices.forEach(matrixId => { |
|
|
const el = document.getElementById(matrixId); |
|
|
if (!el) return; |
|
|
|
|
|
el.innerHTML = ''; |
|
|
const size = CONFIG.attentionMatrixSize; |
|
|
for (let r = 0; r < size; r++) { |
|
|
for (let c = 0; c < size; c++) { |
|
|
const cell = document.createElement('div'); |
|
|
cell.className = 'attn-cell'; |
|
|
cell.dataset.row = r; |
|
|
cell.dataset.col = c; |
|
|
if (c > r) cell.classList.add('masked'); |
|
|
el.appendChild(cell); |
|
|
} |
|
|
} |
|
|
}); |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function openMatrixModal(id) { |
|
|
const config = matrixConfig[id]; |
|
|
if (!config) return; |
|
|
|
|
|
const modal = document.getElementById('matrix-modal'); |
|
|
const fullMatrix = document.getElementById('full-matrix'); |
|
|
const modalTitle = document.getElementById('modal-title'); |
|
|
const modalDims = document.getElementById('modal-dims'); |
|
|
|
|
|
const [rows, cols] = config.true; |
|
|
|
|
|
|
|
|
modalTitle.textContent = config.label; |
|
|
modalDims.textContent = `${rows} × ${cols}`; |
|
|
|
|
|
|
|
|
fullMatrix.innerHTML = ''; |
|
|
fullMatrix.style.gridTemplateColumns = `repeat(${cols}, 1fr)`; |
|
|
|
|
|
|
|
|
let data = null; |
|
|
const instance = componentRegistry[id]; |
|
|
if (instance) { |
|
|
|
|
|
const key = Object.keys(instance.ids).find(k => instance.ids[k] === id); |
|
|
if (key) { |
|
|
if (id.includes('-g')) { |
|
|
const weightKey = key.replace('_g', ''); |
|
|
if (instance.gradients && instance.gradients[weightKey]) { |
|
|
data = instance.gradients[weightKey]; |
|
|
} |
|
|
} else { |
|
|
if (instance.weights && instance.weights[key]) { |
|
|
data = instance.weights[key]; |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if (!data) data = new Float32Array(rows * cols).fill(0); |
|
|
|
|
|
const isGradient = id.includes('-g'); |
|
|
const cmap = isGradient ? plasma : viridis; |
|
|
|
|
|
for (let i = 0; i < rows * cols; i++) { |
|
|
const cell = document.createElement('div'); |
|
|
cell.className = 'm-cell'; |
|
|
|
|
|
let val = data[i]; |
|
|
if (!isGradient) val = (val + 1) / 2; |
|
|
if (isGradient) val = Math.min(Math.abs(val) * 5, 1); |
|
|
|
|
|
cell.style.background = getColor(cmap, val); |
|
|
fullMatrix.appendChild(cell); |
|
|
} |
|
|
|
|
|
|
|
|
modal.classList.add('active'); |
|
|
} |
|
|
|
|
|
function closeMatrixModal() { |
|
|
const modal = document.getElementById('matrix-modal'); |
|
|
modal.classList.remove('active'); |
|
|
} |
|
|
|
|
|
document.getElementById('modal-close').addEventListener('click', closeMatrixModal); |
|
|
document.getElementById('matrix-modal').addEventListener('click', (e) => { |
|
|
if (e.target.id === 'matrix-modal') closeMatrixModal(); |
|
|
}); |
|
|
document.addEventListener('keydown', (e) => { |
|
|
if (e.key === 'Escape') closeMatrixModal(); |
|
|
}); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class BaseComponent { |
|
|
constructor() { |
|
|
this.weights = {}; |
|
|
this.gradients = {}; |
|
|
this.ids = {}; |
|
|
this.dims = { row: 100, col: 100 }; |
|
|
} |
|
|
|
|
|
initWeights(keys) { |
|
|
keys.forEach(k => { |
|
|
this.weights[k] = new Float32Array(this.dims.row * this.dims.col).map(() => Math.random() * 2 - 1); |
|
|
this.gradients[k] = new Float32Array(this.dims.row * this.dims.col).fill(0); |
|
|
}); |
|
|
} |
|
|
|
|
|
renderWeights(keys) { |
|
|
keys.forEach(k => { |
|
|
const el = document.getElementById(this.ids[k]); |
|
|
if (!el) return; |
|
|
const cells = el.querySelectorAll('.m-cell'); |
|
|
cells.forEach((cell, i) => { |
|
|
if (i < this.weights[k].length) { |
|
|
const val = (this.weights[k][i] + 1) / 2; |
|
|
cell.style.background = getColor(viridis, val); |
|
|
} |
|
|
}); |
|
|
}); |
|
|
} |
|
|
|
|
|
renderGradients(keys, intensity = 1.0) { |
|
|
keys.forEach(k => { |
|
|
const el = document.getElementById(this.ids[k + '_g']); |
|
|
if (!el) return; |
|
|
const cells = el.querySelectorAll('.m-cell'); |
|
|
cells.forEach((cell, i) => { |
|
|
const grad = this.gradients[k][i]; |
|
|
if ((Math.random() < intensity && Math.abs(grad) > 0.0001) || Math.abs(grad) > 0.05) { |
|
|
const val = Math.min(Math.abs(grad) * 5, 1); |
|
|
cell.style.background = getColor(plasma, val); |
|
|
} else { |
|
|
cell.style.background = 'rgba(255,255,255,0.08)'; |
|
|
} |
|
|
}); |
|
|
}); |
|
|
} |
|
|
|
|
|
updateWeights(keys, lr) { |
|
|
keys.forEach(k => { |
|
|
for (let i = 0; i < this.weights[k].length; i++) { |
|
|
this.weights[k][i] -= lr * this.gradients[k][i]; |
|
|
} |
|
|
}); |
|
|
} |
|
|
|
|
|
clearGradients(keys) { |
|
|
keys.forEach(k => { |
|
|
this.gradients[k].fill(0); |
|
|
const el = document.getElementById(this.ids[k + '_g']); |
|
|
if (el) { |
|
|
el.querySelectorAll('.m-cell').forEach(c => c.style.background = 'rgba(255,255,255,0.08)'); |
|
|
} |
|
|
}); |
|
|
} |
|
|
|
|
|
visualizeActivation(id, intensity = 0.8) { |
|
|
const el = document.getElementById(id); |
|
|
if (el) { |
|
|
el.querySelectorAll('.act-cell').forEach(c => { |
|
|
c.style.background = Math.random() < intensity ? randomFromCmap(coolwarm) : 'rgba(255,255,255,0.1)'; |
|
|
}); |
|
|
} |
|
|
} |
|
|
|
|
|
clearActivation(id) { |
|
|
const el = document.getElementById(id); |
|
|
if (el) { |
|
|
el.querySelectorAll('.act-cell').forEach(c => c.style.background = 'rgba(255,255,255,0.1)'); |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function setArrow(id, state) { |
|
|
const el = document.getElementById(id); |
|
|
if (!el) return; |
|
|
el.classList.remove('fwd', 'bwd'); |
|
|
if (state) el.classList.add(state); |
|
|
} |
|
|
|
|
|
function clearArrows() { |
|
|
for (let i = 0; i <= 6; i++) setArrow(`arrow-${i}`, null); |
|
|
} |
|
|
|
|
|
function setLayerState(id, state) { |
|
|
const el = document.getElementById(id); |
|
|
if (!el) return; |
|
|
el.classList.remove('active-fwd', 'active-bwd'); |
|
|
if (state === 'fwd') el.classList.add('active-fwd'); |
|
|
if (state === 'bwd') el.classList.add('active-bwd'); |
|
|
} |
|
|
|
|
|
function clearLayerStates() { |
|
|
['loss-box', 'lm-head-box', 'xformer-block', 'ffn-layer', 'attn-layer', 'emb-box'].forEach(id => { |
|
|
setLayerState(id, null); |
|
|
}); |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TokenEmbedding extends BaseComponent { |
|
|
constructor(suffix = '') { |
|
|
super(); |
|
|
this.ids = { |
|
|
emb: `emb-w${suffix}`, |
|
|
emb_g: `emb-g${suffix}`, |
|
|
out: `act-emb${suffix}` |
|
|
}; |
|
|
registerComponent(this); |
|
|
} |
|
|
|
|
|
init() { |
|
|
this.initWeights(['emb']); |
|
|
this.renderWeights(['emb']); |
|
|
} |
|
|
|
|
|
async forward(animate = true) { |
|
|
if (animate) { |
|
|
setLayerState('emb-box', 'fwd'); |
|
|
setArrow('arrow-6', 'fwd'); |
|
|
await wait(CONFIG.timing.embedding); |
|
|
this.visualizeActivation(this.ids.out); |
|
|
setArrow('arrow-5', 'fwd'); |
|
|
await wait(CONFIG.timing.embeddingActivation); |
|
|
} |
|
|
} |
|
|
|
|
|
async backward(animate = true) { |
|
|
if (animate) { |
|
|
setStatus('Backward: Embedding', 'Sparse updates'); |
|
|
setLayerState('emb-box', 'bwd'); |
|
|
setArrow('arrow-4', 'bwd'); |
|
|
setArrow('arrow-5', 'bwd'); |
|
|
} |
|
|
|
|
|
this.gradients['emb'] = this.gradients['emb'].map(() => (Math.random() < 0.1 ? (Math.random() * 2 - 1) * 0.2 : 0)); |
|
|
|
|
|
if (animate) { |
|
|
this.renderGradients(['emb'], CONFIG.gradientIntensity.embedding); |
|
|
await wait(CONFIG.timing.ffnEffect); |
|
|
} |
|
|
} |
|
|
|
|
|
async update(lr) { |
|
|
this.updateWeights(['emb'], lr); |
|
|
this.renderWeights(['emb']); |
|
|
this.clearGradients(['emb']); |
|
|
} |
|
|
|
|
|
reset() { |
|
|
this.init(); |
|
|
this.clearActivation(this.ids.out); |
|
|
this.clearGradients(['emb']); |
|
|
} |
|
|
} |
|
|
|
|
|
class FeedForwardNetwork extends BaseComponent { |
|
|
constructor(suffix = '') { |
|
|
super(); |
|
|
this.ids = { |
|
|
w1: `ffn-w1-w${suffix}`, w2: `ffn-w2-w${suffix}`, |
|
|
w1_g: `ffn-w1-g${suffix}`, w2_g: `ffn-w2-g${suffix}`, |
|
|
in: `act-ffn-in${suffix}`, out: `act-ffn-out${suffix}` |
|
|
}; |
|
|
registerComponent(this); |
|
|
} |
|
|
|
|
|
init() { |
|
|
this.initWeights(['w1', 'w2']); |
|
|
this.renderWeights(['w1', 'w2']); |
|
|
} |
|
|
|
|
|
async forward(animate = true) { |
|
|
if (animate) { |
|
|
const layerId = this.ids.w1.includes('tab') ? (this.ids.w1.includes('tab2') ? 'ffn-layer-tab2' : 'ffn-layer-tab') : 'ffn-layer'; |
|
|
setLayerState(layerId, 'fwd'); |
|
|
this.visualizeActivation(this.ids.in); |
|
|
await wait(CONFIG.timing.ffn); |
|
|
this.visualizeActivation(this.ids.out); |
|
|
await wait(CONFIG.timing.ffnEffect); |
|
|
} |
|
|
} |
|
|
|
|
|
async backward(animate = true) { |
|
|
if (animate) { |
|
|
const layerId = this.ids.w1.includes('tab') ? (this.ids.w1.includes('tab2') ? 'ffn-layer-tab2' : 'ffn-layer-tab') : 'ffn-layer'; |
|
|
setLayerState(layerId, 'bwd'); |
|
|
} |
|
|
|
|
|
['w1', 'w2'].forEach(k => { |
|
|
this.gradients[k] = this.gradients[k].map(() => (Math.random() * 2 - 1) * 0.1); |
|
|
}); |
|
|
|
|
|
if (animate) { |
|
|
this.renderGradients(['w2', 'w1'], CONFIG.gradientIntensity.ffn); |
|
|
await wait(CONFIG.timing.attentionEffect); |
|
|
} |
|
|
} |
|
|
|
|
|
async update(lr) { |
|
|
this.updateWeights(['w1', 'w2'], lr); |
|
|
this.renderWeights(['w1', 'w2']); |
|
|
this.clearGradients(['w1', 'w2']); |
|
|
} |
|
|
|
|
|
reset() { |
|
|
this.init(); |
|
|
this.clearActivation(this.ids.in); |
|
|
this.clearActivation(this.ids.out); |
|
|
this.clearGradients(['w1', 'w2']); |
|
|
} |
|
|
} |
|
|
|
|
|
class MultiHeadAttention extends BaseComponent { |
|
|
constructor(suffix = '') { |
|
|
super(); |
|
|
this.suffix = suffix; |
|
|
this.ids = { |
|
|
wq: `wq-w${suffix}`, wk: `wk-w${suffix}`, wv: `wv-w${suffix}`, wo: `wo-w${suffix}`, |
|
|
wq_g: `wq-g${suffix}`, wk_g: `wk-g${suffix}`, wv_g: `wv-g${suffix}`, wo_g: `wo-g${suffix}`, |
|
|
attn_matrix: `attn-matrix${suffix}`, |
|
|
output: `act-attn-out${suffix}` |
|
|
}; |
|
|
registerComponent(this); |
|
|
} |
|
|
|
|
|
init() { |
|
|
this.initWeights(['wq', 'wk', 'wv', 'wo']); |
|
|
this.renderWeights(['wq', 'wk', 'wv', 'wo']); |
|
|
|
|
|
const attnEl = document.getElementById(this.ids.attn_matrix); |
|
|
if(attnEl) attnEl.querySelectorAll('.attn-cell').forEach(c => { |
|
|
const r = parseInt(c.dataset.row); |
|
|
const col = parseInt(c.dataset.col); |
|
|
c.style.background = col > r ? 'rgba(0,0,0,0.3)' : 'rgba(255,255,255,0.05)'; |
|
|
}); |
|
|
} |
|
|
|
|
|
async forward(animate = true) { |
|
|
if (animate) { |
|
|
const layerId = this.ids.wq.includes('tab') ? (this.ids.wq.includes('tab2') ? 'attn-layer-tab2' : 'attn-layer-tab') : 'attn-layer'; |
|
|
setLayerState(layerId, 'fwd'); |
|
|
await wait(CONFIG.timing.attention); |
|
|
} |
|
|
|
|
|
const attnEl = document.getElementById(this.ids.attn_matrix); |
|
|
if(attnEl) { |
|
|
const cells = attnEl.querySelectorAll('.attn-cell'); |
|
|
cells.forEach(cell => { |
|
|
const r = parseInt(cell.dataset.row); |
|
|
const c = parseInt(cell.dataset.col); |
|
|
if(c <= r) { |
|
|
const dist = r - c; |
|
|
const val = Math.exp(-0.3 * dist) + (Math.random() * 0.2); |
|
|
cell.style.background = getColor(inferno, Math.min(1, val)); |
|
|
} |
|
|
}); |
|
|
} |
|
|
|
|
|
if (animate) { |
|
|
this.visualizeActivation(this.ids.output); |
|
|
await wait(CONFIG.timing.attentionEffect); |
|
|
} |
|
|
} |
|
|
|
|
|
async backward(animate = true) { |
|
|
if (animate) { |
|
|
const layerId = this.ids.wq.includes('tab') ? (this.ids.wq.includes('tab2') ? 'attn-layer-tab2' : 'attn-layer-tab') : 'attn-layer'; |
|
|
setLayerState(layerId, 'bwd'); |
|
|
} |
|
|
|
|
|
['wq', 'wk', 'wv', 'wo'].forEach(k => { |
|
|
this.gradients[k] = this.gradients[k].map(() => (Math.random() * 2 - 1) * 0.1); |
|
|
}); |
|
|
|
|
|
if (animate) { |
|
|
this.renderGradients(['wq', 'wk', 'wv', 'wo'], CONFIG.gradientIntensity.attention); |
|
|
await wait(CONFIG.timing.attentionEffect); |
|
|
} |
|
|
} |
|
|
|
|
|
async update(lr) { |
|
|
this.updateWeights(['wq', 'wk', 'wv', 'wo'], lr); |
|
|
this.renderWeights(['wq', 'wk', 'wv', 'wo']); |
|
|
this.clearGradients(['wq', 'wk', 'wv', 'wo']); |
|
|
} |
|
|
|
|
|
reset() { |
|
|
this.init(); |
|
|
this.clearActivation(this.ids.output); |
|
|
this.clearGradients(['wq', 'wk', 'wv', 'wo']); |
|
|
} |
|
|
} |
|
|
|
|
|
class LanguageModelHead extends BaseComponent { |
|
|
constructor(suffix = '') { |
|
|
super(); |
|
|
this.ids = { |
|
|
lm: `lm-w${suffix}`, |
|
|
lm_g: `lm-g${suffix}`, |
|
|
in: `act-hidden${suffix}`, |
|
|
out: `act-logits${suffix}` |
|
|
}; |
|
|
registerComponent(this); |
|
|
} |
|
|
|
|
|
init() { |
|
|
this.initWeights(['lm']); |
|
|
this.renderWeights(['lm']); |
|
|
} |
|
|
|
|
|
async forward(animate = true) { |
|
|
if (animate) { |
|
|
setArrow('arrow-3', 'fwd'); |
|
|
this.visualizeActivation(this.ids.in); |
|
|
await wait(CONFIG.timing.hidden); |
|
|
|
|
|
setStatus('Forward: LM Head', 'Project to vocab'); |
|
|
setLayerState('lm-head-box', 'fwd'); |
|
|
setArrow('arrow-2', 'fwd'); |
|
|
await wait(CONFIG.timing.lmHead); |
|
|
|
|
|
setArrow('arrow-1', 'fwd'); |
|
|
this.visualizeActivation(this.ids.out); |
|
|
await wait(CONFIG.timing.logits); |
|
|
} |
|
|
} |
|
|
|
|
|
async backward(animate = true) { |
|
|
if (animate) { |
|
|
setArrow('arrow-0', 'bwd'); |
|
|
await wait(CONFIG.timing.backwardDelay); |
|
|
|
|
|
setStatus('Backward: LM Head', '∂L/∂W = xᵀ · ∂L/∂y'); |
|
|
setLayerState('lm-head-box', 'bwd'); |
|
|
setArrow('arrow-1', 'bwd'); |
|
|
} |
|
|
|
|
|
this.gradients['lm'] = this.gradients['lm'].map(() => (Math.random() * 2 - 1) * 0.1); |
|
|
|
|
|
if (animate) { |
|
|
this.renderGradients(['lm'], CONFIG.gradientIntensity.lmHead); |
|
|
await wait(CONFIG.timing.attentionEffect); |
|
|
setArrow('arrow-2', 'bwd'); |
|
|
await wait(CONFIG.timing.backwardDelay); |
|
|
} |
|
|
} |
|
|
|
|
|
async update(lr) { |
|
|
this.updateWeights(['lm'], lr); |
|
|
this.renderWeights(['lm']); |
|
|
this.clearGradients(['lm']); |
|
|
} |
|
|
|
|
|
reset() { |
|
|
this.init(); |
|
|
this.clearActivation(this.ids.in); |
|
|
this.clearActivation(this.ids.out); |
|
|
this.clearGradients(['lm']); |
|
|
} |
|
|
} |
|
|
|
|
|
class TransformerBlock { |
|
|
constructor(suffix = '') { |
|
|
this.attention = new MultiHeadAttention(suffix); |
|
|
this.ffn = new FeedForwardNetwork(suffix); |
|
|
this.suffix = suffix; |
|
|
} |
|
|
|
|
|
init() { |
|
|
this.attention.init(); |
|
|
this.ffn.init(); |
|
|
} |
|
|
|
|
|
async forward(animate = true) { |
|
|
if (animate && !this.suffix) { |
|
|
setLayerState('xformer-block', 'fwd'); |
|
|
setArrow('arrow-4', 'fwd'); |
|
|
await wait(CONFIG.timing.transformerBlock); |
|
|
} |
|
|
|
|
|
if (animate) setStatus('Forward: Attention', 'Computing Q, K, V'); |
|
|
await this.attention.forward(animate); |
|
|
|
|
|
if (animate) setStatus('Forward: FFN', 'GELU(xW₁)W₂'); |
|
|
await this.ffn.forward(animate); |
|
|
} |
|
|
|
|
|
async backward(animate = true) { |
|
|
if (animate && !this.suffix) { |
|
|
setLayerState('xformer-block', 'bwd'); |
|
|
setArrow('arrow-3', 'bwd'); |
|
|
} |
|
|
|
|
|
if (animate) setStatus('Backward: FFN', '∂L/∂W₁, ∂L/∂W₂'); |
|
|
await this.ffn.backward(animate); |
|
|
|
|
|
if (animate) setStatus('Backward: Attention', '∂L/∂Wq,k,v,o'); |
|
|
await this.attention.backward(animate); |
|
|
} |
|
|
|
|
|
async update(lr) { |
|
|
await this.attention.update(lr); |
|
|
await this.ffn.update(lr); |
|
|
} |
|
|
|
|
|
reset() { |
|
|
this.attention.reset(); |
|
|
this.ffn.reset(); |
|
|
} |
|
|
} |
|
|
|
|
|
class GPTModel { |
|
|
constructor() { |
|
|
this.embedding = new TokenEmbedding(); |
|
|
this.block = new TransformerBlock(); |
|
|
this.head = new LanguageModelHead(); |
|
|
|
|
|
this.step = 0; |
|
|
this.tokens = 0; |
|
|
this.loss = 2.891; |
|
|
this.running = false; |
|
|
} |
|
|
|
|
|
init() { |
|
|
this.embedding.init(); |
|
|
this.block.init(); |
|
|
this.head.init(); |
|
|
} |
|
|
|
|
|
async trainStep() { |
|
|
if (this.running) return; |
|
|
this.running = true; |
|
|
document.getElementById('btn-train').disabled = true; |
|
|
|
|
|
this.step++; |
|
|
this.tokens += CONFIG.tokensPerStep; |
|
|
document.getElementById('step-val').textContent = this.step; |
|
|
document.getElementById('tok-val').textContent = this.tokens; |
|
|
|
|
|
|
|
|
setPhase('fwd'); |
|
|
setStatus('Forward Pass', 'Computing activations'); |
|
|
|
|
|
await this.embedding.forward(); |
|
|
await this.block.forward(); |
|
|
await this.head.forward(); |
|
|
|
|
|
|
|
|
setStatus('Computing Loss', 'Cross-entropy'); |
|
|
setLayerState('loss-box', 'fwd'); |
|
|
setArrow('arrow-0', 'fwd'); |
|
|
await wait(CONFIG.timing.loss); |
|
|
|
|
|
|
|
|
setPhase('bwd'); |
|
|
setStatus('Backward Pass', 'Computing gradients'); |
|
|
clearArrows(); |
|
|
|
|
|
await this.head.backward(); |
|
|
await this.block.backward(); |
|
|
await this.embedding.backward(); |
|
|
|
|
|
|
|
|
const norm = (Math.random() * CONFIG.gradNorm.range + CONFIG.gradNorm.min).toFixed(2); |
|
|
document.getElementById('norm-val').textContent = norm; |
|
|
|
|
|
|
|
|
setPhase('upd'); |
|
|
setStatus('Weight Update', 'W ← W − lr·∇L'); |
|
|
await wait(CONFIG.timing.updateDelay); |
|
|
|
|
|
await this.head.update(CONFIG.learningRate); |
|
|
await this.block.update(CONFIG.learningRate); |
|
|
await this.embedding.update(CONFIG.learningRate); |
|
|
|
|
|
|
|
|
this.loss = Math.max(CONFIG.loss.minValue, this.loss - (Math.random() * CONFIG.loss.decreaseRange + CONFIG.loss.decreaseBase)); |
|
|
const lossEl = document.getElementById('loss-val'); |
|
|
lossEl.textContent = this.loss.toFixed(3); |
|
|
lossEl.classList.add('good'); |
|
|
await wait(CONFIG.timing.updateDelay); |
|
|
|
|
|
|
|
|
setPhase(null); |
|
|
clearArrows(); |
|
|
clearLayerStates(); |
|
|
setStatus('Step Complete', `Loss: ${this.loss.toFixed(3)} | ∇: ${norm}`); |
|
|
|
|
|
await wait(CONFIG.timing.cleanupDelay); |
|
|
document.getElementById('loss-val').classList.remove('good'); |
|
|
|
|
|
this.running = false; |
|
|
document.getElementById('btn-train').disabled = false; |
|
|
} |
|
|
|
|
|
reset() { |
|
|
if (timer) clearTimeout(timer); |
|
|
this.running = false; |
|
|
this.step = 0; |
|
|
this.tokens = 0; |
|
|
this.loss = 2.891; |
|
|
|
|
|
document.getElementById('step-val').textContent = '0'; |
|
|
document.getElementById('tok-val').textContent = '0'; |
|
|
document.getElementById('loss-val').textContent = '2.891'; |
|
|
document.getElementById('norm-val').textContent = '0.00'; |
|
|
document.getElementById('loss-val').classList.remove('good'); |
|
|
document.getElementById('btn-train').disabled = false; |
|
|
|
|
|
setPhase(null); |
|
|
clearArrows(); |
|
|
clearLayerStates(); |
|
|
setStatus('Ready', 'Click Train to begin'); |
|
|
|
|
|
this.embedding.reset(); |
|
|
this.block.reset(); |
|
|
this.head.reset(); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const gptModel = new GPTModel(); |
|
|
|
|
|
|
|
|
const attnComponent = new MultiHeadAttention('-tab'); |
|
|
const ffnComponent = new FeedForwardNetwork('-tab'); |
|
|
const blockComponent = new TransformerBlock('-tab2'); |
|
|
|
|
|
function initAll() { |
|
|
Object.entries(matrixConfig).forEach(([id, config]) => { |
|
|
const [rows, cols] = config.display; |
|
|
initGrid(id, rows, cols); |
|
|
}); |
|
|
|
|
|
Object.entries(CONFIG.streams).forEach(([id, count]) => { |
|
|
initStream(id, count); |
|
|
}); |
|
|
|
|
|
initAttnMatrix(); |
|
|
|
|
|
gptModel.init(); |
|
|
attnComponent.init(); |
|
|
ffnComponent.init(); |
|
|
blockComponent.init(); |
|
|
} |
|
|
|
|
|
|
|
|
document.getElementById('btn-train').addEventListener('click', () => gptModel.trainStep()); |
|
|
document.getElementById('btn-reset').addEventListener('click', () => gptModel.reset()); |
|
|
|
|
|
|
|
|
const toggleAttnBtns = (disabled) => { |
|
|
['btn-attn-fwd', 'btn-attn-bwd', 'btn-attn-update'].forEach(id => |
|
|
document.getElementById(id).disabled = disabled); |
|
|
}; |
|
|
document.getElementById('btn-attn-fwd').addEventListener('click', async () => { |
|
|
toggleAttnBtns(true); await attnComponent.forward(); toggleAttnBtns(false); |
|
|
}); |
|
|
document.getElementById('btn-attn-bwd').addEventListener('click', async () => { |
|
|
toggleAttnBtns(true); await attnComponent.backward(); toggleAttnBtns(false); |
|
|
}); |
|
|
document.getElementById('btn-attn-update').addEventListener('click', async () => { |
|
|
toggleAttnBtns(true); await attnComponent.update(CONFIG.learningRate); toggleAttnBtns(false); |
|
|
}); |
|
|
document.getElementById('btn-attn-reset').addEventListener('click', () => attnComponent.reset()); |
|
|
|
|
|
|
|
|
const toggleFFNBtns = (disabled) => { |
|
|
['btn-ffn-fwd', 'btn-ffn-bwd', 'btn-ffn-update'].forEach(id => |
|
|
document.getElementById(id).disabled = disabled); |
|
|
}; |
|
|
document.getElementById('btn-ffn-fwd').addEventListener('click', async () => { |
|
|
toggleFFNBtns(true); await ffnComponent.forward(); toggleFFNBtns(false); |
|
|
}); |
|
|
document.getElementById('btn-ffn-bwd').addEventListener('click', async () => { |
|
|
toggleFFNBtns(true); await ffnComponent.backward(); toggleFFNBtns(false); |
|
|
}); |
|
|
document.getElementById('btn-ffn-update').addEventListener('click', async () => { |
|
|
toggleFFNBtns(true); await ffnComponent.update(CONFIG.learningRate); toggleFFNBtns(false); |
|
|
}); |
|
|
document.getElementById('btn-ffn-reset').addEventListener('click', () => ffnComponent.reset()); |
|
|
|
|
|
|
|
|
const toggleBlockBtns = (disabled) => { |
|
|
['btn-xformer-fwd', 'btn-xformer-bwd', 'btn-xformer-update'].forEach(id => |
|
|
document.getElementById(id).disabled = disabled); |
|
|
}; |
|
|
document.getElementById('btn-xformer-fwd').addEventListener('click', async () => { |
|
|
toggleBlockBtns(true); await blockComponent.forward(); toggleBlockBtns(false); |
|
|
}); |
|
|
document.getElementById('btn-xformer-bwd').addEventListener('click', async () => { |
|
|
toggleBlockBtns(true); await blockComponent.backward(); toggleBlockBtns(false); |
|
|
}); |
|
|
document.getElementById('btn-xformer-update').addEventListener('click', async () => { |
|
|
toggleBlockBtns(true); await blockComponent.update(CONFIG.learningRate); toggleBlockBtns(false); |
|
|
}); |
|
|
document.getElementById('btn-xformer-reset').addEventListener('click', () => blockComponent.reset()); |
|
|
|
|
|
|
|
|
const tabButtons = document.querySelectorAll('.tab-btn'); |
|
|
const tabContents = document.querySelectorAll('.tab-content'); |
|
|
|
|
|
tabButtons.forEach(button => { |
|
|
button.addEventListener('click', () => { |
|
|
const targetTab = button.getAttribute('data-tab'); |
|
|
tabButtons.forEach(btn => btn.classList.remove('active')); |
|
|
tabContents.forEach(content => content.classList.remove('active')); |
|
|
button.classList.add('active'); |
|
|
document.getElementById(`tab-${targetTab}`).classList.add('active'); |
|
|
}); |
|
|
}); |
|
|
|
|
|
|
|
|
initAll(); |
|
|
</script> |
|
|
</body> |
|
|
</html> |
|
|
|
