| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| <title>Model Parameter Calculator</title> |
| <style> |
| body { |
| font-family: Arial, sans-serif; |
| margin: 0; |
| padding: 0; |
| display: flex; |
| flex-direction: row; |
| height: 100vh; |
| } |
| |
| .console { |
| width: 20%; |
| padding: 20px; |
| background-color: #f6b5b5; |
| overflow-y: auto; |
| border-right: 2px solid #ee4a4f; |
| } |
| |
| #additionalFieldsMOE { |
| display: none; |
| } |
| |
| .output { |
| width: 80%; |
| padding: 20px; |
| overflow-y: auto; |
| } |
| |
| table { |
| width: 100%; |
| border-collapse: collapse; |
| margin-bottom: 20px; |
| background-color: #fff7f9; |
| border: 1px solid #f6b5b5; |
| } |
| |
| th, td { |
| border: 1px solid #f6b5b5; |
| padding: 8px; |
| text-align: center; |
| } |
| |
| th { |
| background-color: #f6d4d4; |
| } |
| |
| .highlight { |
| background-color: #ffe8e8; |
| color: #ad0d0d; |
| padding: 10px; |
| margin: 20px 0; |
| border: 1px solid #f6b5b5; |
| border-radius: 4px; |
| text-align: center; |
| } |
| |
| h3 { |
| color: #ad0d0d; |
| } |
| |
| .section { |
| margin-bottom: 20px; |
| } |
| label { |
| font-weight: bold; |
| font-size: 0.9rem; |
| } |
| |
| input[type="number"], select { |
| width: 100%; |
| padding: 8px; |
| border: 1px solid #ddd; |
| border-radius: 5px; |
| box-sizing: border-box; |
| font-size: 0.9rem; |
| } |
| |
| input[type="number"]:focus, select:focus { |
| border-color: #ff6666; |
| outline: none; |
| box-shadow: 0px 0px 5px rgba(255, 102, 102, 0.5); |
| } |
| |
| button { |
| background-color: #ff6666; |
| color: white; |
| border: none; |
| border-radius: 5px; |
| padding: 5px; |
| width: 100%; |
| font-size: 1rem; |
| cursor: pointer; |
| transition: background-color 0.3s ease; |
| } |
| |
| button:hover { |
| background-color: #e60000; |
| } |
| </style> |
| <script> |
| function handleSelectChange() { |
| // Récupère la valeur sélectionnée |
| const isMoe = document.getElementById('is_moe').value; |
| const additionalFieldsMOE = document.getElementById('additionalFieldsMOE'); |
| |
| // Affiche ou cache les champs supplémentaires |
| if (isMoe === 'yes') { |
| additionalFieldsMOE.style.display = 'block'; |
| } else { |
| additionalFieldsMOE.style.display = 'none'; |
| } |
| } |
| |
| function calculateParameters() { |
| const hiddenSize = parseInt(document.getElementById('hidden_size').value); |
| const num_kv_heads = parseInt(document.getElementById('num_key_value_heads').value); |
| const num_heads = parseInt(document.getElementById('num_heads').value); |
| const intermediateSize = parseInt(document.getElementById('intermediate_size').value); |
| const vocabSize = parseInt(document.getElementById('vocab_size').value); |
| const numHiddenLayers = parseInt(document.getElementById('num_hidden_layers').value); |
| const active_expert_number = parseInt(document.getElementById('active_expert_number').value); |
| const expert_number = parseInt(document.getElementById('expert_number').value); |
| const includeBias = document.getElementById('include_bias').value === 'yes'; |
| const isMoe = document.getElementById('is_moe').value === 'yes'; |
| |
| const calcTotal = (input, output) => input * output; |
| const calcKVdim = (hiddenSize, num_heads,num_kv_heads) => hiddenSize * num_kv_heads / num_heads; |
| |
| // Attention calculations |
| const attention = [ |
| { name: 'Query', input: hiddenSize, output: hiddenSize }, |
| { name: 'Key', input: hiddenSize, output: calcKVdim(hiddenSize, num_heads,num_kv_heads) }, |
| { name: 'Value', input: hiddenSize, output: calcKVdim(hiddenSize, num_heads,num_kv_heads) }, |
| { name: 'Projection', input: hiddenSize, output: hiddenSize } |
| ].map(entry => ({ |
| ...entry, |
| wTotal: calcTotal(entry.input, entry.output), |
| bTotal: includeBias ? entry.output : 0 |
| })); |
| |
| const attentionTotal = attention.reduce((sum, entry) => sum + entry.wTotal + entry.bTotal, 0); |
| |
| // Feed Forward calculations |
| const switchW = calcTotal(hiddenSize, intermediateSize); |
| const switchB = includeBias ? intermediateSize : 0; |
| const luW = calcTotal(hiddenSize, intermediateSize); |
| const luB = includeBias ? intermediateSize : 0; |
| const projW = calcTotal(intermediateSize, hiddenSize); |
| const projB = includeBias ? hiddenSize : 0; |
| let feedForwardTotal; |
| |
| feedForwardTotal = switchW + switchB + luW + luB + projW + projB; |
| const feedForwardTotalCache = feedForwardTotal |
| |
| let feedForwardTotalActive; |
| |
| if (isMoe) { |
| feedForwardTotalActive = feedForwardTotalCache * active_expert_number; |
| feedForwardTotal = feedForwardTotalCache * expert_number; |
| console.log(feedForwardTotal) |
| } |
| // Embedding |
| const embeddingTotal = calcTotal(vocabSize, hiddenSize); |
| |
| // 1 layer parameters |
| const oneLayerParams = attentionTotal + feedForwardTotal; |
| |
| let oneLayerParamsActive; |
| if (isMoe) { |
| oneLayerParamsActive = attentionTotal + feedForwardTotalActive; |
| } |
| |
| // Full layers parameters |
| const fullLayersParams = oneLayerParams * numHiddenLayers; |
| |
| let fullLayersParamsMOEActive; |
| if (isMoe) { |
| fullLayersParamsMOEActive = oneLayerParamsActive * numHiddenLayers; |
| } |
| |
| // Full size (includes embedding) |
| const fullSize = fullLayersParams + embeddingTotal; |
| |
| let fullSizeActive; |
| if (isMoe) { |
| fullSizeActive = fullLayersParamsMOEActive + embeddingTotal; |
| } |
| |
| // Display results |
| const outputDiv = document.getElementById('output'); |
| outputDiv.innerHTML = ` |
| <h1>Model Parameter Calculator</h1> |
| <div class="section"> |
| <h3>Attention</h3> |
| <table> |
| <tr> |
| <th>Parameter</th> |
| <th>Input Size</th> |
| <th>Output Size</th> |
| <th>Total parameters (input*ouput)</th> |
| </tr> |
| ${attention.map(entry => ` |
| <tr> |
| <td>${entry.name} - W</td> |
| <td>${entry.input.toLocaleString()}</td> |
| <td>${entry.output.toLocaleString()}</td> |
| <td>${entry.wTotal.toLocaleString()}</td> |
| </tr> |
| ${includeBias ? ` |
| <tr> |
| <td>${entry.name} - b</td> |
| <td>-</td> |
| <td>${entry.output.toLocaleString()}</td> |
| <td>${entry.bTotal.toLocaleString()}</td> |
| </tr> |
| ` : ''} |
| `).join('')} |
| <tr> |
| <th colspan="3">Total attention parameters</th> |
| <td colspan="2">${attentionTotal.toLocaleString()}</td> |
| </tr> |
| </table> |
| </div> |
| <div class="section"> |
| <h3>Feed Forward</h3> |
| <table> |
| <tr> |
| <th>Parameter</th> |
| <th>Input Size</th> |
| <th>Output Size</th> |
| <th>Total (input*output)</th> |
| </tr> |
| <tr> |
| <td>Swish - W</td> |
| <td>${hiddenSize.toLocaleString()}</td> |
| <td>${intermediateSize.toLocaleString()}</td> |
| <td>${switchW.toLocaleString()}</td> |
| </tr> |
| ${includeBias ? `<tr> |
| <td>Swish - b</td> |
| <td>-</td> |
| <td>${intermediateSize.toLocaleString()}</td> |
| <td>${switchB.toLocaleString()}</td> |
| </tr>` : ''} |
| <tr> |
| <td>GLU W</td> |
| <td>${hiddenSize.toLocaleString()}</td> |
| <td>${intermediateSize.toLocaleString()}</td> |
| <td>${luW.toLocaleString()}</td> |
| </tr> |
| ${includeBias ? `<tr> |
| <td>GLU - b</td> |
| <td>-</td> |
| <td>${intermediateSize.toLocaleString()}</td> |
| <td>${luB.toLocaleString()}</td> |
| </tr>` : ''} |
| <tr> |
| <td>Projection - W</td> |
| <td>${intermediateSize.toLocaleString()}</td> |
| <td>${hiddenSize.toLocaleString()}</td> |
| <td>${projW.toLocaleString()}</td> |
| </tr> |
| ${includeBias ? `<tr> |
| <td>Projection - b</td> |
| <td>-</td> |
| <td>${hiddenSize.toLocaleString()}</td> |
| <td>${projB.toLocaleString()}</td> |
| </tr>` : ''} |
| <tr> |
| <th colspan="3">Total Feed Forward parameters</th> |
| <td>${feedForwardTotal.toLocaleString()}</td> |
| </tr> |
| ${isMoe ?` |
| <tr> |
| <th colspan="3">Total active Feed Forward parameters</th> |
| <td>${feedForwardTotalActive.toLocaleString()}</td> |
| </tr>` : ''} |
| </table> |
| </div> |
| <div class="highlight"> |
| <strong>1 Layer Parameters (Attention + FFN):</strong> ${oneLayerParams.toLocaleString()}<br> |
| <strong>Full Layers Parameters (1 layer parameters * num layers):</strong> ${fullLayersParams.toLocaleString()}<br><br> |
| ${isMoe ? `<tr> |
| <strong>1 Layer Parameters Active(Attention + FFN):</strong> ${oneLayerParamsActive.toLocaleString()}<br> |
| <strong>Full Layers Parameters Active(1 layer active parameters * num layers):</strong> ${fullLayersParamsMOEActive.toLocaleString()}<br> |
| </tr>` : ''} |
| </div> |
| <div class="section"> |
| <h3>Embedding</h3> |
| <table> |
| <tr> |
| <th>Parameter</th> |
| <th>Vocab Size</th> |
| <th>Hidden Size</th> |
| <th>Total (vocab * hidden)</th> |
| </tr> |
| <tr> |
| <td>Embedding</td> |
| <td>${vocabSize.toLocaleString()}</td> |
| <td>${hiddenSize.toLocaleString()}</td> |
| <td>${embeddingTotal.toLocaleString()}</td> |
| </tr> |
| </table> |
| </div> |
| <div class="highlight"> |
| <strong>Complete Model Parmeters (embedding size + full layers size):</strong> ${fullSize.toLocaleString()}<br><br> |
| ${isMoe ? `<strong>Complete Model Parmeters Active:</strong> ${fullSizeActive.toLocaleString()}` : ''} |
| </div> |
| `; |
| } |
| </script> |
| </head> |
| <body> |
|
|
| <div class="console"> |
| <h3>Input Parameters</h3> |
| <label for="hidden_size">Hidden size:</label><br> |
| <input type="number" id="hidden_size" value="896"><br><br> |
|
|
| <label for="intermediate_size">Intermediate size:</label><br> |
| <input type="number" id="intermediate_size" value="4864"><br><br> |
|
|
| <label for="vocab_size">Vocab size:</label><br> |
| <input type="number" id="vocab_size" value="151646"><br><br> |
|
|
| <label for="num_key_value_heads">Number of key-value heads:</label><br> |
| <input type="number" id="num_key_value_heads" value="2"><br><br> |
|
|
| <label for="num_heads">Number of attention (query) heads:</label><br> |
| <input type="number" id="num_heads" value="14"><br><br> |
|
|
| <label for="num_hidden_layers">Number of hidden layers:</label><br> |
| <input type="number" id="num_hidden_layers" value="24"><br><br> |
|
|
| <label for="include_bias">Include bias?</label><br> |
| <select id="include_bias"> |
| <option value="no">No</option> |
| <option value="yes">Yes</option> |
| </select><br><br> |
|
|
| <label for="is_moe">Is MOE ?</label><br> |
| <select id="is_moe" onchange="handleSelectChange()"> |
| <option value="no">No</option> |
| <option value="yes">Yes</option> |
| </select><br><br> |
|
|
| <div id="additionalFieldsMOE"> |
| <label for="expert_number">Total expert number :</label><br> |
| <input type="number" id="expert_number" name="expert_number"><br><br> |
|
|
| <label for="active_expert_number">Total active experts (shared + specifics):</label><br> |
| <input type="number" id="active_expert_number" name="active_expert_number"><br><br> |
| </div> |
|
|
| <button onclick="calculateParameters()">Calculate</button> |
| </div> |
|
|
| <div class="output" id="output"> |
| <h1>Transformer total number of parameters Calculator</h1> |
| <h3>Enter model hyperparameters in the console and press calculate (curently working for classic transformer/LLM architecture with GQA and GLU)</h3> |
| </div> |
|
|
| </body> |
| </html> |
|
|
