Hugging Face's logo

Spaces:
NyxKrage
/
LLM-Model-VRAM-Calculator
Running

App Files Community
11
LLM-Model-VRAM-Calculator / index.html
AliceThirty's picture
AliceThirty
Support gated models
19538ae verified
raw
history blame
25.9 kB
<!DOCTYPE html>
<html lang="en">
 <head>
 <meta charset="UTF-8" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 <script>
 function strToHtml(str) {
 let parser = new DOMParser();
 return parser.parseFromString(str, "text/html");
 }
 //Short, jQuery-independent function to read html table and write them into an Array.
 //Kudos to RobG at StackOverflow
 function tableToObj(table) {
 var rows = table.rows;
 var propCells = rows[0].cells;
 var propNames = [];
 var results = [];
 var obj, row, cells;
 // Use the first row for the property names
 // Could use a header section but result is the same if
 // there is only one header row
 for (var i = 0, iLen = propCells.length; i < iLen; i++) {
 propNames.push(
 (propCells[i].textContent || propCells[i].innerText).trim()
 );
 }
 // Use the rows for data
 // Could use tbody rows here to exclude header & footer
 // but starting from 1 gives required result
 for (var j = 1, jLen = rows.length; j < jLen; j++) {
 cells = rows[j].cells;
 obj = {};
 for (var k = 0; k < iLen; k++) {
 obj[propNames[k]] = (
 cells[k].textContent || cells[k].innerText
 ).trim();
 }
 results.push(obj);
 }
 return results;
 }
 function formatGpu(gpus) {
 return gpus.map(
 (g) => `${g["Product Name"]} - ${g["Memory"].split(",")[0]}`
 );
 }
 const gguf_quants = {
 "IQ1_S": 1.56,
 "IQ2_XXS": 2.06,
 "IQ2_XS": 2.31,
 "IQ2_S": 2.5,
 "IQ2_M": 2.7,
 "IQ3_XXS": 3.06,
 "IQ3_XS": 3.3,
 "Q2_K": 3.35,
 "Q3_K_S": 3.5,
 "IQ3_S": 3.5,
 "IQ3_M": 3.7,
 "Q3_K_M": 3.91,
 "Q3_K_L": 4.27,
 "IQ4_XS": 4.25,
 "IQ4_NL": 4.5,
 "Q4_0": 4.55,
 "Q4_K_S": 4.58,
 "Q4_K_M": 4.85,
 "Q5_0": 5.54,
 "Q5_K_S": 5.54,
 "Q5_K_M": 5.69,
 "Q6_K": 6.59,
 "Q8_0": 8.5,
 }
async function modelConfig(hf_model, hf_token) {
 auth = hf_token == "" ? {} : {
 headers: {
 'Authorization': `Bearer ${hf_token}`
 }
 }
 let config = await fetch(
 `https://huggingface.co/${hf_model}/raw/main/config.json`, auth
 ).then(r => r.json())
 let model_size = 0
 try {
 model_size = (await fetch(`https://huggingface.co/${hf_model}/resolve/main/model.safetensors.index.json`, auth).then(r => r.json()))["metadata"]["total_size"] / 2
 if (isNaN(model_size)) {
 throw new Erorr("no size in safetensors metadata")
 }
 } catch (e) {
 try {
 model_size = (await fetch(`https://huggingface.co/${hf_model}/resolve/main/pytorch_model.bin.index.json`, auth).then(r => r.json()))["metadata"]["total_size"] / 2
 if (isNaN(model_size)) {
 throw new Erorr("no size in pytorch metadata")
 }
 } catch {
 let model_page = await fetch(
 "https://corsproxy.io/?" + encodeURIComponent(`https://huggingface.co/${hf_model}`)
 ).then(r => r.text())
 let el = document.createElement( 'html' );
 el.innerHTML = model_page
 let params_el = el.querySelector('div[data-target="ModelSafetensorsParams"]')
 if (params_el !== null) {
 model_size = JSON.parse(params_el.attributes.getNamedItem("data-props").value)["safetensors"]["total"]
 } else {
 params_el = el.querySelector('div[data-target="ModelHeader"]')
 model_size = JSON.parse(params_el.attributes.getNamedItem("data-props").value)["model"]["safetensors"]["total"]
 }
 }
 }
 config.parameters = model_size
 return config
 }
 function inputBuffer(context=8192, model_config, bsz=512) {
 /* Calculation taken from github:ggerganov/llama.cpp/llama.cpp:11248
 ctx->inp_tokens = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);
 ctx->inp_embd = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, hparams.n_embd, cparams.n_batch);
 ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);
 ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx, cparams.n_batch);
 ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_ctx);
 ctx->inp_sum = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, 1, cparams.n_batch);
 n_embd is hidden size (github:ggeranov/llama.cpp/convert.py:248)
 */
 const inp_tokens = bsz
 const inp_embd = model_config["hidden_size"] * bsz
 const inp_pos = bsz
 const inp_KQ_mask = context * bsz
 const inp_K_shift = context
 const inp_sum = bsz
 return inp_tokens + inp_embd + inp_pos + inp_KQ_mask + inp_K_shift + inp_sum
 }
 function computeBuffer(context=8192, model_config, bsz=512) {
 if (bsz != 512) {
 alert("batch size other than 512 is currently not supported for the compute buffer, using batchsize 512 for compute buffer calculation, end result result will be an overestimatition")
 }
 return (context / 1024 * 2 + 0.75) * model_config["num_attention_heads"] * 1024 * 1024
 }
 function kvCache(context=8192, model_config, cache_bit=16) {
 const n_gqa = model_config["num_attention_heads"] / model_config["num_key_value_heads"]
 const n_embd_gqa = model_config["hidden_size"] / n_gqa
 const n_elements = n_embd_gqa * (model_config["num_hidden_layers"] * context)
 const size = 2 * n_elements
 return size * (cache_bit / 8)
 }
 function contextSize(context=8192, model_config, bsz=512, cache_bit=16) {
 return Number.parseFloat((inputBuffer(context, model_config, bsz) + kvCache(context, model_config, cache_bit) + computeBuffer(context, model_config, bsz)).toFixed(2))
 }
 function modelSize(model_config, bpw=4.5) {
 return Number.parseFloat((model_config["parameters"] * bpw / 8).toFixed(2))
 }
 async function calculateSizes(format) {
 try {
 const model_config = await modelConfig(document.getElementById("modelsearch").value, document.getElementById("hf_token").value)
 const context = parseInt(document.getElementById("contextsize").value)
 let bsz = 512
 let cache_bit = 16
 let bpw = 0
 if (format === "gguf") {
 bsz = parseInt(document.getElementById("batchsize").value)
 bpw = gguf_quants[document.getElementById("quantsize").innerText]
} else if (format == "exl2") {
 cache_bit = Number.parseInt(document.getElementById("kvCache").value)
 bpw = Number.parseFloat(document.getElementById("bpw").value)
 }
const model_size = modelSize(model_config, bpw)
 const context_size = contextSize(context, model_config, bsz, cache_bit)
 const total_size = ((model_size + context_size) / 2**30)
 document.getElementById("resultmodel").innerText = (model_size / 2**30).toFixed(2)
 document.getElementById("resultcontext").innerText = (context_size / 2**30).toFixed(2)
 const result_total_el = document.getElementById("resulttotal");
 result_total_el.innerText = total_size.toFixed(2)
const gpu = document.getElementById("gpusearch").value
 if (gpu !== "") {
 const vram = parseFloat(gpu.split("-")[1].replace("GB", "").trim())
 if (vram - total_size > 0.5) {
 result_total_el.style.backgroundColor = "#bef264"
 } else if (vram - total_size > 0) {
 result_total_el.style.backgroundColor = "#facc15"
 } else {
 result_total_el.style.backgroundColor = "#ef4444"
 }
 }
 } catch(e) {
 alert(e);
 }
 }
 </script>
 <link href="./styles.css" rel="stylesheet">
 <title>Can I run it? - LLM VRAM Calculator</title>
 </head>
 <body class="p-8">
 <div x-data="{ format: 'gguf' }" class="flex flex-col max-h-screen items-center mt-16 gap-10">
 <h1 class="text-xl font-semibold leading-6 text-gray-900">
 LLM Model, Can I run it?
 </h1>
 <p>
 To support gated or private repos, you need to <a href="https://huggingface.co/settings/tokens" style="color: #4444ff"><b>create an authentification token</b></a>, to check the box <span style="color: #6e1818"><b>"Read access to contents of all public gated repos you can access"</b></span> and then enter the token in the field below.
 </p>
<div class="flex flex-col gap-10">
 <div class="w-auto flex flex-col gap-4">
 <!-- Huggingface Authentification Token -->
 <div
 class="relative"
 x-data="{
 results: null,
 query: null
 }"
 >
 <label
 for="gpusearch"
 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
 >Huggingface Token (optional)</label
 >
 <input
 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
 id="hf_token"
 />
 </div>
 <!-- GPU Selector -->
 <div
 class="relative"
 x-data="{
 results: null,
 query: null
 }"
 >
 <label
 for="gpusearch"
 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
 >GPU (optional)</label
 >
 <input
 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
 placeholder="GeForce RTX 3090 - 24 GB"
 id="gpusearch"
 name="gpusearch"
 list="gpulist"
 x-model="query"
 @keypress.debounce.150ms="results = query === '' ? [] : formatGpu(tableToObj(strToHtml(await fetch('https://corsproxy.io/?https://www.techpowerup.com/gpu-specs/?ajaxsrch=' + query).then(r => r.text())).querySelector('table')))"
 />
 <datalist id="gpulist">
 <template x-for="item in results">
 <option :value="item" x-text="item"></option>
 </template>
 </datalist>
 </div>
 <!-- Model Selector -->
<div class="flex flex-row gap-4 relative">
 <label
 for="contextsize"
 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
 >
 Model (unquantized)
 </label>
 <div
 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
 x-data="{
 open: false,
 value: 'Nexusflow/Starling-LM-7B-beta',
 results: null,
 toggle() {
 if (this.open) {
 return this.close()
 }
 this.$refs.input.focus()
this.open = true
 },
 close(focusAfter) {
 if (! this.open) return
this.open = false
focusAfter && focusAfter.focus()
 }
 }"
 x-on:keydown.escape.prevent.stop="close($refs.input)"
 x-id="['model-typeahead']"
 class="relative"
 >
 <!-- Input -->
 <input
 id="modelsearch"
 x-ref="input"
 x-on:click="toggle()"
 @keypress.debounce.150ms="results = (await
 fetch('https://huggingface.co/api/quicksearch?type=model&q=' +
 encodeURIComponent(value)).then(r => r.json())).models.filter(m => !m.id.includes('GGUF') && !m.id.includes('AWQ') && !m.id.includes('GPTQ') && !m.id.includes('exl2'));"
 :aria-expanded="open"
 :aria-controls="$id('model-typeahead')"
 x-model="value"
 class="flex justify-between items-center gap-2 w-full"
 />
<!-- Panel -->
 <div
 x-ref="panel"
 x-show="open"
 x-transition.origin.top.left
 x-on:click.outside="close($refs.input)"
 :id="$id('model-typeahead')"
 style="display: none"
 class="absolute left-0 mt-4 w-full rounded-md bg-white shadow-sm ring-1 ring-inset ring-gray-300 z-10"
 >
 <template x-for="result in results">
 <a
 @click="value = result.id; close($refs.input)"
 x-text="result.id"
 class="flex cursor-pointer items-center gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm hover:bg-gray-500/5 disabled:text-gray-500"
 ></a>
 </template>
 </div>
 </div>
 </div>
<!-- Context Size Selector -->
 <div class="relative">
 <label
 for="contextsize"
 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
 >
 Context Size
 </label>
 <input
 value="8192"
 type="number"
 name="contextsize"
 id="contextsize"
 step="1024"
 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
 />
 </div>
 <!-- Quant Format Selector -->
 <div class="relative">
 <label
 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
 >Quant Format</label
 >
 <fieldset
 x-model="format"
 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
 >
 <legend class="sr-only">Quant format</legend>
 <div
 class="space-y-4 sm:flex sm:items-center sm:space-x-10 sm:space-y-0"
 >
 <div class="flex items-center">
 <input
 id="gguf-format"
 name="quant-format"
 type="radio"
 value="gguf"
 checked
 class="h-4 w-4 border-gray-300 text-indigo-600 focus:ring-indigo-600"
 />
 <label
 for="gguf-format"
 class="ml-3 block text-sm font-medium leading-6 text-gray-900"
 >GGUF</label
 >
 </div>
 <div class="flex items-center">
 <input
 id="exl2-format"
 name="quant-format"
 type="radio"
 value="exl2"
 class="h-4 w-4 border-gray-300 text-indigo-600 focus:ring-indigo-600"
 />
 <label
 for="exl2-format"
 class="ml-3 block text-sm font-medium leading-6 text-gray-900"
 >EXL2</label
 >
 </div>
 <div class="flex items-center">
 <input
 id="gptq-format"
 name="quant-format"
 type="radio"
 disabled
 value="gptq"
 class="h-4 w-4 border-gray-300 text-indigo-600 focus:ring-indigo-600"
 />
 <label
 for="gptq-format"
 class="ml-3 block text-sm font-medium leading-6 text-gray-900"
 >GPTQ (coming soon)</label
 >
 </div>
 </div>
 </fieldset>
 </div>
 <!-- EXL2 Options -->
 <div x-show="format === 'exl2'" class="flex flex-row gap-4">
 <div class="relative flex-grow">
 <label
 for="bpw"
 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
 >
 BPW
 </label>
 <input
 value="4.5"
 type="number"
 step="0.01"
 id="bpw"
 name="bpw"
 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
 />
 </div>
 <div
 class="flex-shrink relative rounded-md"
 >
 <div
 class="w-fit p-3 h-full flex items-center gap-2 justify-center rounded-md border-0 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
 >
 <label
 for="kvCache"
 class="inline-block bg-white text-xs font-medium text-gray-900"
 >
 KV Cache
 </label>
 <select id="kvCache" name="kvCache">
 <option value="16">16 bit</option>
 <option value="8">8 bit</option>
 <option value="4">4 bit</option>
 </select>
 </div>
 </div>
 </div>
 <!-- GGUF Options -->
 <div x-show="format === 'gguf'" class="relative">
 <div class="flex flex-row gap-4">
 <label
 for="contextsize"
 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
 >
 Quantization Size
 </label>
 <div
 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
 x-data="{
 open: false,
 value: '',
 toggle() {
 if (this.open) {
 return this.close()
 }
 this.$refs.button.focus()
this.open = true
 },
 close(focusAfter) {
 if (! this.open) return
this.open = false
focusAfter && focusAfter.focus()
 }
 }"
 x-on:keydown.escape.prevent.stop="close($refs.button)"
 x-id="['dropdown-button']"
 class="relative"
 >
 <!-- Button -->
 <button
 x-ref="button"
 x-on:click="toggle()"
 :aria-expanded="open"
 :aria-controls="$id('dropdown-button')"
 type="button"
 id="quantsize"
 x-text="value.length === 0 ? 'Q4_K_S' : value"
 class="flex justify-between items-center gap-2 w-full"
 >
 Q4_K_S
<!-- Heroicon: chevron-down -->
 <svg
 xmlns="http://www.w3.org/2000/svg"
 class="h-5 w-5 text-gray-400"
 viewBox="0 0 20 20"
 fill="currentColor"
 >
 <path
 fill-rule="evenodd"
 d="M5.293 7.293a1 1 0 011.414 0L10 10.586l3.293-3.293a1 1 0 111.414 1.414l-4 4a1 1 0 01-1.414 0l-4-4a1 1 0 010-1.414z"
 clip-rule="evenodd"
 />
 </svg>
 </button>
<!-- Panel -->
 <div
 x-data="{ quants: [
 'IQ1_S',
 'IQ2_XXS',
 'IQ2_XS',
 'IQ2_S',
 'IQ2_M',
 'IQ3_XXS',
 'IQ3_XS',
 'Q2_K',
 'Q3_K_S',
 'IQ3_S',
 'IQ3_M',
 'Q3_K_M',
 'Q3_K_L',
 'IQ4_XS',
 'IQ4_NL',
 'Q4_0',
 'Q4_K_S',
 'Q4_K_M',
 'Q5_0',
 'Q5_K_S',
 'Q5_K_M',
 'Q6_K',
 'Q8_0'
 ]}"
 x-ref="panel"
 x-show="open"
 x-transition.origin.top.left
 x-on:click.outside="close($refs.button)"
 :id="$id('dropdown-button')"
 style="display: none"
 class="absolute left-0 mt-4 w-full rounded-md bg-white shadow-sm ring-1 ring-inset ring-gray-300 z-10"
 >
 <template x-for="quant in quants">
 <a
 @click="value = quant; close($refs.button)"
 x-text="quant"
 class="flex cursor-pointer items-center gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm hover:bg-gray-500/5 disabled:text-gray-500"
 ></a>
 </template>
 </div>
 </div>
 <div class="relative">
 <label
 for="batchsize"
 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
 >
 Batch Size
 </label>
 <input
 value="512"
 type="number"
 step="128"
 id="batchsize"
 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
 />
 </div>
 </div>
 </div>
 <button
 type="button"
 class="rounded-md bg-slate-800 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-slate-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600"
 @click="calculateSizes(format)"
 >
 Submit
 </button>
 </div>
 <div class="w-auto flex flex-col gap-4">
 <div class="relative">
 <label
 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
 >
 Model Size (GB)
 </label>
 <div
 id="resultmodel"
 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
 >4.20</div>
 </div>
 <div class="relative">
 <label
 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
 >
 Context Size (GB)
 </label>
 <div
 id="resultcontext"
 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
 >6.90</div>
 </div>
 <div class="relative">
 <label
 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"
 >
 Total Size (GB)
 </label>
 <div
 id="resulttotal"
 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"
 >420.69</div>
 </div>
 </div>
 </div>
 </div>
 <script
 src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"
 ></script>
 <script defer>
 calculateSizes("gguf")
 </script>
 </body>
</html>