File size: 53,225 Bytes
d02aea6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 | <!DOCTYPE html>
<html lang="en" class="scroll-smooth">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>llama.cpp - Complete Guide, Tips & Forks</title>
<script src="https://cdn.tailwindcss.com"></script>
<script src="https://unpkg.com/lucide@latest"></script>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&family=JetBrains+Mono:wght@400;500;600&display=swap" rel="stylesheet">
<script>
tailwind.config = {
theme: {
extend: {
fontFamily: {
sans: ['Inter', 'sans-serif'],
mono: ['JetBrains Mono', 'monospace'],
},
colors: {
primary: '#10b981',
secondary: '#3b82f6',
accent: '#f59e0b',
dark: '#0f172a',
darker: '#020617',
code: '#1e293b',
}
}
}
}
</script>
<style>
body {
background-color: #020617;
color: #e2e8f0;
}
.glass {
background: rgba(30, 41, 59, 0.7);
backdrop-filter: blur(12px);
border: 1px solid rgba(255, 255, 255, 0.1);
}
.glass-hover:hover {
background: rgba(51, 65, 85, 0.8);
border-color: rgba(16, 185, 129, 0.5);
transform: translateY(-4px);
box-shadow: 0 20px 40px -15px rgba(16, 185, 129, 0.3);
}
.code-block {
background: #0d1117;
border: 1px solid #30363d;
border-radius: 0.5rem;
overflow-x: auto;
}
.copy-btn {
position: absolute;
top: 0.5rem;
right: 0.5rem;
opacity: 0;
transition: opacity 0.2s;
}
.code-block:hover .copy-btn {
opacity: 1;
}
.pre-wrap {
white-space: pre-wrap;
word-wrap: break-word;
}
.hero-gradient {
background: linear-gradient(135deg, rgba(16, 185, 129, 0.1) 0%, rgba(59, 130, 246, 0.1) 100%);
}
</style>
</head>
<body class="antialiased">
<!-- Navigation -->
<nav class="fixed w-full z-50 glass border-b border-slate-800">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<div class="flex justify-between items-center h-16">
<div class="flex items-center space-x-2">
<i data-lucide="brain-circuit" class="w-8 h-8 text-primary"></i>
<span class="text-xl font-bold text-white">llama.cpp</span>
</div>
<div class="hidden md:flex space-x-8">
<a href="#overview" class="text-slate-300 hover:text-primary transition">Overview</a>
<a href="#installation" class="text-slate-300 hover:text-primary transition">Installation</a>
<a href="#usage" class="text-slate-300 hover:text-primary transition">Usage</a>
<a href="#forks" class="text-slate-300 hover:text-primary transition">Forks</a>
<a href="#optimization" class="text-slate-300 hover:text-primary transition">Tips</a>
</div>
<div class="md:hidden">
<button id="mobile-menu-btn" class="text-slate-300 hover:text-white">
<i data-lucide="menu" class="w-6 h-6"></i>
</button>
</div>
</div>
</div>
<div id="mobile-menu" class="hidden md:hidden glass border-t border-slate-800">
<div class="px-2 pt-2 pb-3 space-y-1">
<a href="#overview" class="block px-3 py-2 text-slate-300 hover:text-primary">Overview</a>
<a href="#installation" class="block px-3 py-2 text-slate-300 hover:text-primary">Installation</a>
<a href="#usage" class="block px-3 py-2 text-slate-300 hover:text-primary">Usage</a>
<a href="#forks" class="block px-3 py-2 text-slate-300 hover:text-primary">Forks</a>
<a href="#optimization" class="block px-3 py-2 text-slate-300 hover:text-primary">Tips</a>
</div>
</div>
</nav>
<!-- Hero Section -->
<section class="relative pt-32 pb-20 lg:pt-48 lg:pb-32 hero-gradient overflow-hidden">
<div class="absolute inset-0 bg-[radial-gradient(ellipse_at_center,_var(--tw-gradient-stops))] from-primary/20 via-darker to-darker"></div>
<div class="relative max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 text-center">
<h1 class="text-5xl md:text-7xl font-extrabold mb-6 bg-gradient-to-r from-primary to-secondary bg-clip-text text-transparent">
llama.cpp
</h1>
<p class="text-2xl md:text-3xl text-slate-300 mb-8 font-light">
Your Ultimate Guide to Running LLMs Locally
</p>
<p class="text-lg text-slate-400 max-w-3xl mx-auto mb-10">
Complete documentation, installation guides, optimization tips, and forks comparison for the most efficient C++ implementation of LLaMA and other large language models.
</p>
<div class="flex flex-col sm:flex-row gap-4 justify-center">
<a href="#installation" class="px-8 py-4 bg-primary hover:bg-primary/90 text-white font-semibold rounded-lg transition transform hover:scale-105 flex items-center justify-center gap-2">
<i data-lucide="download" class="w-5 h-5"></i>
Quick Start
</a>
<a href="https://github.com/ggerganov/llama.cpp" target="_blank" class="px-8 py-4 glass hover:bg-slate-800 text-white font-semibold rounded-lg transition flex items-center justify-center gap-2">
<i data-lucide="github" class="w-5 h-5"></i>
View on GitHub
</a>
</div>
</div>
</section>
<!-- Overview Section -->
<section id="overview" class="py-20 bg-slate-900/50">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<div class="text-center mb-16">
<h2 class="text-3xl md:text-4xl font-bold mb-4 text-white">What is llama.cpp?</h2>
<p class="text-slate-400 text-lg max-w-2xl mx-auto">
A high-performance C++ port of Facebook's LLaMA model, enabling efficient inference on consumer hardware.
</p>
</div>
<div class="grid md:grid-cols-3 gap-8">
<div class="glass p-6 rounded-xl glass-hover transition duration-300">
<div class="w-12 h-12 bg-primary/20 rounded-lg flex items-center justify-center mb-4">
<i data-lucide="cpu" class="w-6 h-6 text-primary"></i>
</div>
<h3 class="text-xl font-semibold mb-2 text-white">Zero Dependencies</h3>
<p class="text-slate-400">Pure C++ implementation with no dependencies. Just compile and run on CPU or GPU.</p>
</div>
<div class="glass p-6 rounded-xl glass-hover transition duration-300">
<div class="w-12 h-12 bg-secondary/20 rounded-lg flex items-center justify-center mb-4">
<i data-lucide="zap" class="w-6 h-6 text-secondary"></i>
</div>
<h3 class="text-xl font-semibold mb-2 text-white">Optimized Inference</h3>
<p class="text-slate-400">Highly optimized for 4-bit and 5-bit quantized models. Supports GPU acceleration via CUDA, Metal, and Vulkan.</p>
</div>
<div class="glass p-6 rounded-xl glass-hover transition duration-300">
<div class="w-12 h-12 bg-accent/20 rounded-lg flex items-center justify-center mb-4">
<i data-lucide="layers" class="w-6 h-6 text-accent"></i>
</div>
<h3 class="text-xl font-semibold mb-2 text-white">Multiple Models</h3>
<p class="text-slate-400">Supports LLaMA, LLaMA 2, Falcon, Wizard, Vicuna, and many more GGUF models.</p>
</div>
</div>
<!-- Key Stats -->
<div class="mt-16 grid grid-cols-2 md:grid-cols-4 gap-8">
<div class="text-center p-6 glass rounded-xl">
<div class="text-3xl font-bold text-primary mb-1">60k+</div>
<div class="text-sm text-slate-400">GitHub Stars</div>
</div>
<div class="text-center p-6 glass rounded-xl">
<div class="text-3xl font-bold text-secondary mb-1">Q4_K_M</div>
<div class="text-sm text-slate-400">Recommended Quant</div>
</div>
<div class="text-center p-6 glass rounded-xl">
<div class="text-3xl font-bold text-accent mb-1">CUDA</div>
<div class="text-sm text-slate-400">GPU Support</div>
</div>
<div class="text-center p-6 glass rounded-xl">
<div class="text-3xl font-bold text-purple-400 mb-1">8GB+</div>
<div class="text-sm text-slate-400">RAM Required</div>
</div>
</div>
</div>
</section>
<!-- Installation Section -->
<section id="installation" class="py-20">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<div class="text-center mb-12">
<h2 class="text-3xl md:text-4xl font-bold mb-4 text-white">Installation Guide</h2>
<p class="text-slate-400">Multiple ways to get llama.cpp running on your machine.</p>
</div>
<div class="space-y-8">
<!-- Pre-built Binaries -->
<div class="glass p-6 rounded-xl">
<h3 class="text-xl font-semibold mb-4 text-white flex items-center gap-2">
<i data-lucide="package" class="w-5 h-5 text-primary"></i>
Option 1: Pre-built Releases (Windows)
</h3>
<div class="relative code-block p-4 mb-4">
<button class="copy-btn p-2 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-4 h-4"></i>
</button>
<code class="text-slate-300 font-mono text-sm">
# Download from:<br>
https://github.com/ggerganov/llama.cpp/releases<br><br>
# Look for: llama-b[BUILD]-bin-win-[ARCH]-[BUILD_TYPE].zip<br>
# llama-b[BUILD]-macOS-[ARCH].zip<br>
# llama-b[BUILD]-bin-ubuntu-[ARCH].[EXT]
</code>
</div>
</div>
<!-- Build from Source -->
<div class="glass p-6 rounded-xl">
<h3 class="text-xl font-semibold mb-4 text-white flex items-center gap-2">
<i data-lucide="terminal" class="w-5 h-5 text-secondary"></i>
Option 2: Build from Source
</h3>
<div class="space-y-4">
<div>
<h4 class="text-sm font-semibold text-slate-300 mb-2">Mac/Linux</h4>
<div class="relative code-block p-4">
<button class="copy-btn p-2 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-4 h-4"></i>
</button>
<pre class="font-mono text-sm text-slate-300 pre-wrap">git clone https://github.com/ggerganov/llama.cpp.git
cd llama.cpp
make</pre>
</div>
</div>
<div>
<h4 class="text-sm font-semibold text-slate-300 mb-2">Windows (CMake)</h4>
<div class="relative code-block p-4">
<button class="copy-btn p-2 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-4 h-4"></i>
</button>
<pre class="font-mono text-sm text-slate-300 pre-wrap">git clone https://github.com/ggerganov/llama.cpp.git
cd llama.cpp
cmake .
cmake --build . --config Release</pre>
</div>
</div>
<div>
<h4 class="text-sm font-semibold text-slate-300 mb-2">With CUDA Support (GPU)</h4>
<div class="relative code-block p-4">
<button class="copy-btn p-2 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-4 h-4"></i>
</button>
<pre class="font-mono text-sm text-slate-300 pre-wrap">make LLAMA_CUDA=1
# or for Windows
cmake -DLLAMA_CUDA=ON .
cmake --build . --config Release</pre>
</div>
</div>
</div>
</div>
<!-- Docker -->
<div class="glass p-6 rounded-xl">
<h3 class="text-xl font-semibold mb-4 text-white flex items-center gap-2">
<i data-lucide="container" class="w-5 h-5 text-accent"></i>
Option 3: Docker
</h3>
<div class="relative code-block p-4">
<button class="copy-btn p-2 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-4 h-4"></i>
</button>
<pre class="font-mono text-sm text-slate-300 pre-wrap">docker pull ghcr.io/ggerganov/llama.cpp:latest
docker run -v /path/to/models:/models ghcr.io/ggerganov/llama.cpp:latest --api -m /models/your-model.gguf</pre>
</div>
</div>
</div>
</div>
</section>
<!-- Usage Section -->
<section id="usage" class="py-20 bg-slate-900/50">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<div class="text-center mb-12">
<h2 class="text-3xl md:text-4xl font-bold mb-4 text-white">Usage & Commands</h2>
<p class="text-slate-400">Master the CLI and server modes.</p>
</div>
<div class="grid lg:grid-cols-2 gap-8">
<!-- Basic Commands -->
<div class="glass p-6 rounded-xl">
<h3 class="text-xl font-semibold mb-4 text-white">Basic Commands</h3>
<div class="space-y-4">
<div>
<h4 class="text-sm font-semibold text-slate-300 mb-2">Simple Inference</h4>
<div class="relative code-block p-3">
<button class="copy-btn p-1.5 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-3 h-3"></i>
</button>
<code class="text-xs font-mono text-slate-300">./main -m models/model.gguf -p "Your prompt here"</code>
</div>
</div>
<div>
<h4 class="text-sm font-semibold text-slate-300 mb-2">Interactive Chat</h4>
<div class="relative code-block p-3">
<button class="copy-btn p-1.5 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-3 h-3"></i>
</button>
<code class="text-xs font-mono text-slate-300">./main -m models/model.gguf --interactive-server</code>
</div>
</div>
<div>
<h4 class="text-sm font-semibold text-slate-300 mb-2">Server Mode (API)</h4>
<div class="relative code-block p-3">
<button class="copy-btn p-1.5 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-3 h-3"></i>
</button>
<code class="text-xs font-mono text-slate-300">./server -m models/model.gguf -c 2048 --port 8080</code>
</div>
</div>
</div>
</div>
<!-- Command Parameters -->
<div class="glass p-6 rounded-xl">
<h3 class="text-xl font-semibold mb-4 text-white">Important Flags</h3>
<div class="space-y-3">
<div class="flex items-start gap-3">
<code class="text-primary font-mono text-sm">-c, --ctx_size</code>
<span class="text-slate-400 text-sm">Context size (e.g., 2048, 4096, 8192)</span>
</div>
<div class="flex items-start gap-3">
<code class="text-primary font-mono text-sm">-n, --n_predict</code>
<span class="text-slate-400 text-sm">Number of tokens to predict (-1 = infinity)</span>
</div>
<div class="flex items-start gap-3">
<code class="text-primary font-mono text-sm">-t, --threads</code>
<span class="text-slate-400 text-sm">Number of CPU threads (recommend: physical cores)</span>
</div>
<div class="flex items-start gap-3">
<code class="text-primary font-mono text-sm">--temp</code>
<span class="text-slate-400 text-sm">Temperature (0.8 is standard)</span>
</div>
<div class="flex items-start gap-3">
<code class="text-primary font-mono text-sm">--gpu_layers/-ngl</code>
<span class="text-slate-400 text-sm">Number of layers to offload to GPU</span>
</div>
</div>
</div>
<!-- API Example -->
<div class="glass p-6 rounded-xl lg:col-span-2">
<h3 class="text-xl font-semibold mb-4 text-white">API Example (OpenAI-compatible)</h3>
<div class="relative code-block p-4">
<button class="copy-btn p-2 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-4 h-4"></i>
</button>
<pre class="font-mono text-sm text-slate-300 pre-wrap"># Start server
./server -m models/llama-2-7b-chat.Q4_K_M.gguf -c 4096
# Send request
curl -X POST http://localhost:8080/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"messages": [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello!"}
]
}'</pre>
</div>
</div>
</div>
</div>
</section>
<!-- Forks Section -->
<section id="forks" class="py-20">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<div class="text-center mb-12">
<h2 class="text-3xl md:text-4xl font-bold mb-4 text-white">Popular Forks & Projects</h2>
<p class="text-slate-400">Specialized versions and wrappers built on llama.cpp</p>
</div>
<div class="grid md:grid-cols-2 lg:grid-cols-3 gap-6">
<!-- Ollama -->
<div class="glass p-6 rounded-xl glass-hover transition duration-300 border-l-4 border-primary">
<div class="flex items-center justify-between mb-3">
<h3 class="text-xl font-bold text-white">Ollama</h3>
<span class="px-2 py-1 bg-primary/20 text-primary text-xs rounded font-mono">Top Pick</span>
</div>
<p class="text-slate-400 mb-4 text-sm">The easiest way to run LLaMA, Mistral, and other models locally. Provides a CLI and API for running models.</p>
<ul class="text-xs text-slate-500 space-y-1 mb-4">
<li>• macOS, Linux, Windows</li>
<li>• Model library available</li>
<li>• REST API included</li>
<li>• One-liner install</li>
</ul>
<a href="https://ollama.ai" target="_blank" class="text-primary hover:text-primary/80 text-sm flex items-center gap-1">
ollama.ai <i data-lucide="external-link" class="w-3 h-3"></i>
</a>
</div>
<!-- Oobabooga -->
<div class="glass p-6 rounded-xl glass-hover transition duration-300 border-l-4 border-secondary">
<div class="flex items-center justify-between mb-3">
<h3 class="text-xl font-bold text-white">TextGen WebUI</h3>
<span class="px-2 py-1 bg-secondary/20 text-secondary text-xs rounded font-mono">GUI</span>
</div>
<p class="text-slate-400 mb-4 text-sm">A web interface for LLMs. Supports multiple backends including llama.cpp with extensive character/persona features.</p>
<ul class="text-xs text-slate-500 space-y-1 mb-4">
<li>• Web-based interface</li>
<li>• Chat/Completions modes</li>
<li>• LoRA support</li>
<li>• Extensions support</li>
</ul>
<a href="https://github.com/oobabooga/text-generation-webui" target="_blank" class="text-secondary hover:text-secondary/80 text-sm flex items-center gap-1">
GitHub <i data-lucide="external-link" class="w-3 h-3"></i>
</a>
</div>
<!-- Koboldcpp -->
<div class="glass p-6 rounded-xl glass-hover transition duration-300 border-l-4 border-pink-500">
<div class="flex items-center justify-between mb-3">
<h3 class="text-xl font-bold text-white">KoboldCpp</h3>
<span class="px-2 py-1 bg-pink-500/20 text-pink-500 text-xs rounded font-mono">Gaming</span>
</div>
<p class="text-slate-400 mb-4 text-sm">A user-friendly wrapper for llama.cpp optimized for story writing and text adventure gaming.</p>
<ul class="text-xs text-slate-500 space-y-1 mb-4">
<li>• Kobold AI compatibility</li>
<li>• Streamlined UI</li>
<li>• Adventure mode</li>
<li>• Easy single-binary setup</li>
</ul>
<a href="https://github.com/LostRuins/koboldcpp" target="_blank" class="text-pink-500 hover:text-pink-400 text-sm flex items-center gap-1">
GitHub <i data-lucide="external-link" class="w-3 h-3"></i>
</a>
</div>
<!-- llamafile -->
<div class="glass p-6 rounded-xl glass-hover transition duration-300 border-l-4 border-accent">
<div class="flex items-center justify-between mb-3">
<h3 class="text-xl font-bold text-white">llamafile</h3>
<span class="px-2 py-1 bg-accent/20 text-accent text-xs rounded font-mono">Portable</span>
</div>
<p class="text-slate-400 mb-4 text-sm">Mozilla's project. LLMs packaged as single executable files that run on most computers without dependencies.</p>
<ul class="text-xs text-slate-500 space-y-1 mb-4">
<li>• Single-file executables</li>
<li>• No installation needed</li>
<li>• Cross-platform</li>
<li>• Embeddable</li>
</ul>
<a href="https://github.com/Mozilla-Ocho/llamafile" target="_blank" class="text-accent hover:text-accent/80 text-sm flex items-center gap-1">
GitHub <i data-lucide="external-link" class="w-3 h-3"></i>
</a>
</div>
<!-- LocalAI -->
<div class="glass p-6 rounded-xl glass-hover transition duration-300 border-l-4 border-purple-500">
<div class="flex items-center justify-between mb-3">
<h3 class="text-xl font-bold text-white">LocalAI</h3>
<span class="px-2 py-1 bg-purple-500/20 text-purple-500 text-xs rounded font-mono">API</span>
</div>
<p class="text-slate-400 mb-4 text-sm">Drop-in OpenAI API replacement. Self-hosted with llama.cpp backend. Supports text generation, images, and audio.</p>
<ul class="text-xs text-slate-500 space-y-1 mb-4">
<li>• OpenAI API compatible</li>
<li>• Docker ready</li>
<li>• Model hot-reloading</li>
<li>• Multiple backends</li>
</ul>
<a href="https://localai.io" target="_blank" class="text-purple-500 hover:text-purple-400 text-sm flex items-center gap-1">
localai.io <i data-lucide="external-link" class="w-3 h-3"></i>
</a>
</div>
<!-- LM Studio -->
<div class="glass p-6 rounded-xl glass-hover transition duration-300 border-l-4 border-cyan-500">
<div class="flex items-center justify-between mb-3">
<h3 class="text-xl font-bold text-white">LM Studio</h3>
<span class="px-2 py-1 bg-cyan-500/20 text-cyan-500 text-xs rounded font-mono">App</span>
</div>
<p class="text-slate-400 mb-4 text-sm">Desktop application for running local LLMs with a beautiful interface. Easy model downloading and chatting.</p>
<ul class="text-xs text-slate-500 space-y-1 mb-4">
<li>• Desktop GUI</li>
<li>• Chat interface</li>
<li>• HuggingFace integration</li>
<li>• macOS/Windows</li>
</ul>
<a href="https://lmstudio.ai" target="_blank" class="text-cyan-500 hover:text-cyan-400 text-sm flex items-center gap-1">
lmstudio.ai <i data-lucide="external-link" class="w-3 h-3"></i>
</a>
</div>
</div>
</div>
</section>
<!-- Optimization Tips -->
<section id="optimization" class="py-20 bg-slate-900/50">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<div class="text-center mb-12">
<h2 class="text-3xl md:text-4xl font-bold mb-4 text-white">Optimization Tips</h2>
<p class="text-slate-400">Get the best performance from your models.</p>
</div>
<div class="grid md:grid-cols-2 gap-8">
<!-- Quantization Guide -->
<div class="glass p-6 rounded-xl">
<h3 class="text-xl font-semibold mb-4 text-white flex items-center gap-2">
<i data-lucide="scale" class="w-5 h-5 text-primary"></i>
Quantization Guide (GGUF)
</h3>
<div class="space-y-3">
<div class="p-3 bg-slate-800/50 rounded-lg border border-slate-700">
<div class="flex justify-between mb-1">
<span class="font-semibold text-primary font-mono text-sm">Q4_K_M</span>
<span class="text-slate-400 text-xs">Best balance</span>
</div>
<p class="text-slate-400 text-xs">Recommended for most models. ~4.7GB for 7B model. Quality slightly better than Q4_0.</p>
</div>
<div class="p-3 bg-slate-800/50 rounded-lg border border-slate-700">
<div class="flex justify-between mb-1">
<span class="font-semibold text-slate-300 font-mono text-sm">Q5_K_M</span>
<span class="text-slate-400 text-xs">High quality</span>
</div>
<p class="text-slate-400 text-xs">For quality-critical tasks. ~5.8GB for 7B model.</p>
</div>
<div class="p-3 bg-slate-800/50 rounded-lg border border-slate-700">
<div class="flex justify-between mb-1">
<span class="font-semibold text-slate-300 font-mono text-sm">Q8_0</span>
<span class="text-slate-400 text-xs">Maximum quality</span>
</div>
<p class="text-slate-400 text-xs">Almost unnoticeable loss. ~7GB for 7B model.</p>
</div>
</div>
</div>
<!-- Performance Tips -->
<div class="glass p-6 rounded-xl">
<h3 class="text-xl font-semibold mb-4 text-white flex items-center gap-2">
<i data-lucide="gauge" class="w-5 h-5 text-secondary"></i>
Performance Tips
</h3>
<ul class="space-y-3 text-slate-300 text-sm">
<li class="flex items-start gap-2">
<span class="text-primary">•</span>
<span><strong>GPU Offloading:</strong> Use <code class="bg-slate-800 px-1 rounded text-primary">-ngl 35</code> on macOS (Metal) or <code class="bg-slate-800 px-1 rounded text-primary">-ngl 33</code> on NVIDIA (CUDA) for 7B models</span>
</li>
<li class="flex items-start gap-2">
<span class="text-primary">•</span>
<span><strong>Context Size:</strong> Start with 2048, increase based on your needs (uses more VRAM)</span>
</li>
<li class="flex items-start gap-2">
<span class="text-primary">•</span>
<span><strong>Threads:</strong> Set to your physical CPU cores count with <code class="bg-slate-800 px-1 rounded text-primary">-t [cores]</code></span>
</li>
<li class="flex items-start gap-2">
<span class="text-primary">•</span>
<span><strong>Memory Mapping:</strong> Enable with <code class="bg-slate-800 px-1 rounded text-primary">--mlock</code> to keep model in RAM</span>
</li>
<li class="flex items-start gap-2">
<span class="text-primary">•</span>
<span><strong>Batch Size:</strong> Increase <code class="bg-slate-800 px-1 rounded text-primary">-b 512</code> for higher throughput</span>
</li>
</ul>
</div>
<!-- Hardware Recommendations -->
<div class="glass p-6 rounded-xl md:col-span-2">
<h3 class="text-xl font-semibold mb-4 text-white flex items-center gap-2">
<i data-lucide="hard-drive" class="w-5 h-5 text-accent"></i>
Hardware Requirements
</h3>
<div class="grid grid-cols-2 md:grid-cols-4 gap-4">
<div class="text-center p-4 bg-slate-800/50 rounded-lg">
<div class="text-lg font-bold text-white mb-1">7B Model</div>
<div class="text-xs text-slate-400">~4-8GB RAM</div>
<div class="text-xs text-slate-500 mt-1">M1 Mac Minimum</div>
</div>
<div class="text-center p-4 bg-slate-800/50 rounded-lg">
<div class="text-lg font-bold text-white mb-1">13B Model</div>
<div class="text-xs text-slate-400">~8-12GB RAM</div>
<div class="text-xs text-slate-500 mt-1">Requires GPU</div>
</div>
<div class="text-center p-4 bg-slate-800/50 rounded-lg">
<div class="text-lg font-bold text-white mb-1">30B Model</div>
<div class="text-xs text-slate-400">~20GB RAM</div>
<div class="text-xs text-slate-500 mt-1">32GB System + GPU</div>
</div>
<div class="text-center p-4 bg-slate-800/50 rounded-lg">
<div class="text-lg font-bold text-white mb-1">70B Model</div>
<div class="text-xs text-slate-400">~40GB+ RAM</div>
<div class="text-xs text-slate-500 mt-1">High-end GPU</div>
</div>
</div>
</div>
</div>
</div>
</section>
<!-- Convert Models Section -->
<section class="py-20">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<div class="glass p-8 rounded-xl border border-slate-700">
<h2 class="text-2xl font-bold mb-4 text-white">Converting Models</h2>
<p class="text-slate-400 mb-4">Converting Safetensors/PyTorch models to GGUF format for llama.cpp:</p>
<div class="relative code-block p-4">
<button class="copy-btn p-2 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-4 h-4"></i>
</button>
<pre class="font-mono text-sm text-slate-300 pre-wrap"># Install dependencies
python -m pip install gguf protobuf
# Convert HuggingFace model to GGUF
python convert-hf-to-gguf.py /path/to/model \
--outfile /path/to/output/model.gguf \
--outtype q4_k_m</pre>
</div>
<p class="text-xs text-slate-500 mt-3">Available outtypes: f32, f16, bf16, q8_0, q4_0, q4_1, q4_k_s, q4_k_m, q5_k_s, q5_k_m, q6_k</p>
</div>
</div>
</section>
<!-- Model Sources Section -->
<section class="py-20 bg-slate-900/50">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<div class="text-center mb-12">
<h2 class="text-3xl md:text-4xl font-bold mb-4 text-white">Where to Download Models</h2>
<p class="text-slate-400">Pre-converted GGUF models ready to use.</p>
</div>
<div class="grid md:grid-cols-3 gap-6">
<div class="glass p-6 rounded-xl glass-hover transition duration-300">
<div class="flex items-center gap-3 mb-4">
<div class="w-10 h-10 bg-yellow-500/20 rounded-lg flex items-center justify-center">
<i data-lucide="database" class="w-5 h-5 text-yellow-500"></i>
</div>
<h3 class="text-lg font-bold text-white">TheBloke</h3>
</div>
<p class="text-slate-400 text-sm mb-4">The most popular source for quantized GGUF models. Hundreds of models including Llama 2, Mistral, CodeLlama, and more.</p>
<a href="https://huggingface.co/TheBloke" target="_blank" class="text-primary hover:text-primary/80 text-sm flex items-center gap-1">
View on HuggingFace <i data-lucide="external-link" class="w-3 h-3"></i>
</a>
</div>
<div class="glass p-6 rounded-xl glass-hover transition duration-300">
<div class="flex items-center gap-3 mb-4">
<div class="w-10 h-10 bg-blue-500/20 rounded-lg flex items-center justify-center">
<i data-lucide="layers" class="w-5 h-5 text-blue-500"></i>
</div>
<h3 class="text-lg font-bold text-white">NousResearch</h3>
</div>
<p class="text-slate-400 text-sm mb-4">Research-focused models including Hermes, Synthia, and other fine-tuned versions with GGUF support.</p>
<a href="https://huggingface.co/NousResearch" target="_blank" class="text-blue-500 hover:text-blue-400 text-sm flex items-center gap-1">
View on HuggingFace <i data-lucide="external-link" class="w-3 h-3"></i>
</a>
</div>
<div class="glass p-6 rounded-xl glass-hover transition duration-300">
<div class="flex items-center gap-3 mb-4">
<div class="w-10 h-10 bg-purple-500/20 rounded-lg flex items-center justify-center">
<i data-lucide="cloud" class="w-5 h-5 text-purple-500"></i>
</div>
<h3 class="text-lg font-bold text-white">LWDW (RunPod)</h3>
</div>
<p class="text-slate-400 text-sm mb-4">Specialized in large model variants (70B+) and unique quantizations. Great for GPU cloud inference.</p>
<a href="https://huggingface.co/LWDW" target="_blank" class="text-purple-500 hover:text-purple-400 text-sm flex items-center gap-1">
View on HuggingFace <i data-lucide="external-link" class="w-3 h-3"></i>
</a>
</div>
</div>
<div class="mt-8 glass p-6 rounded-xl">
<h3 class="text-lg font-semibold mb-3 text-white">Popular Models to Try</h3>
<div class="grid md:grid-cols-2 lg:grid-cols-4 gap-4 text-sm">
<div class="p-3 bg-slate-800/50 rounded border border-slate-700">
<code class="text-primary font-mono">mistral-7b-instruct</code>
<p class="text-slate-500 text-xs mt-1">Fast, great quality</p>
</div>
<div class="p-3 bg-slate-800/50 rounded border border-slate-700">
<code class="text-primary font-mono">llama-2-7b/13b-chat</code>
<p class="text-slate-500 text-xs mt-1">All-purpose, balanced</p>
</div>
<div class="p-3 bg-slate-800/50 rounded border border-slate-700">
<code class="text-primary font-mono">codellama-7b/13b</code>
<p class="text-slate-500 text-xs mt-1">Code generation</p>
</div>
<div class="p-3 bg-slate-800/50 rounded border border-slate-700">
<code class="text-primary font-mono">neural-chat-7b</code>
<p class="text-slate-500 text-xs mt-1">Conversations</p>
</div>
</div>
</div>
</div>
</section>
<!-- Advanced Features Section -->
<section class="py-20">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<div class="text-center mb-12">
<h2 class="text-3xl md:text-4xl font-bold mb-4 text-white">Advanced Features</h2>
<p class="text-slate-400">Unlock the full potential of llama.cpp.</p>
</div>
<div class="space-y-6">
<div class="grid md:grid-cols-2 gap-6">
<div class="glass p-6 rounded-xl">
<h3 class="text-xl font-semibold mb-3 text-white flex items-center gap-2">
<i data-lucide="zap" class="w-5 h-5 text-yellow-500"></i>
Speculative Decoding
</h3>
<p class="text-slate-400 text-sm mb-3">Use a smaller draft model to speed up token generation. Can achieve 2-3x speedup on supported hardware.</p>
<div class="relative code-block p-3">
<button class="copy-btn p-1.5 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-3 h-3"></i>
</button>
<code class="text-xs font-mono text-slate-300">./main -m large_model.gguf --draft small_model.gguf -ngl 35 --draft 10</code>
</div>
</div>
<div class="glass p-6 rounded-xl">
<h3 class="text-xl font-semibold mb-3 text-white flex items-center gap-2">
<i data-lucide="terminal" class="w-5 h-5 text-green-500"></i>
Grammar-Based Sampling
</h3>
<p class="text-slate-400 text-sm mb-3">Force JSON output or specific formats using GBNF grammar files. Perfect for structured output.</p>
<div class="relative code-block p-3">
<button class="copy-btn p-1.5 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-3 h-3"></i>
</button>
<code class="text-xs font-mono text-slate-300">./main -m model.gguf --grammar-file json.gbnf -p "Generate JSON:"</code>
</div>
</div>
</div>
<div class="grid md:grid-cols-2 gap-6">
<div class="glass p-6 rounded-xl">
<h3 class="text-xl font-semibold mb-3 text-white flex items-center gap-2">
<i data-lucide="cpu" class="w-5 h-5 text-blue-500"></i>
Continuous Batching
</h3>
<p class="text-slate-400 text-sm">Process multiple prompts simultaneously in server mode for higher throughput in production environments.</p>
</div>
<div class="glass p-6 rounded-xl">
<h3 class="text-xl font-semibold mb-3 text-white flex items-center gap-2">
<i data-lucide="layers" class="w-5 h-5 text-purple-500"></i>
LoRA Support
</h3>
<p class="text-slate-400 text-sm mb-3">Load LoRA adapters on top of base models without merging. Hot-swap adapters at runtime.</p>
<div class="relative code-block p-3">
<button class="copy-btn p-1.5 bg-slate-700 rounded hover:bg-slate-600 text-white" onclick="copyCode(this)">
<i data-lucide="copy" class="w-3 h-3"></i>
</button>
<code class="text-xs font-mono text-slate-300">./main -m base.gguf --lora adapter.bin --lora-scale 0.8</code>
</div>
</div>
</div>
</div>
</div>
</section>
<!-- Troubleshooting Section -->
<section class="py-20 bg-slate-900/50">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<div class="text-center mb-12">
<h2 class="text-3xl md:text-4xl font-bold mb-4 text-white">Troubleshooting</h2>
<p class="text-slate-400">Common issues and solutions.</p>
</div>
<div class="space-y-4">
<div class="glass p-6 rounded-xl border-l-4 border-red-500">
<h3 class="text-lg font-semibold text-white mb-2">CUDA Out of Memory</h3>
<p class="text-slate-400 text-sm mb-2">Error: <code class="bg-red-900/30 text-red-300 px-1 rounded">CUDA out of memory</code></p>
<p class="text-slate-400 text-sm">Reduce GPU layers or use a smaller model. Try <code class="text-primary">-ngl 20</code> instead of <code class="text-primary">-ngl 35</code>, or use a Q4_K_M quantized model instead of Q5.</p>
</div>
<div class="glass p-6 rounded-xl border-l-4 border-yellow-500">
<h3 class="text-lg font-semibold text-white mb-2">Slow Token Generation</h3>
<p class="text-slate-400 text-sm mb-2">Model is running on CPU instead of GPU.</p>
<p class="text-slate-400 text-sm">Ensure you built with CUDA/Metal support. Check <code class="text-primary">nvidia-smi</code> or Activity Monitor to verify GPU usage. Increase <code class="text-primary">-ngl</code> to offload more layers.</p>
</div>
<div class="glass p-6 rounded-xl border-l-4 border-blue-500">
<h3 class="text-lg font-semibold text-white mb-2">GGUF Format Errors</h3>
<p class="text-slate-400 text-sm mb-2">Error: <code class="bg-blue-900/30 text-blue-300 px-1 rounded">invalid magic</code></p>
<p class="text-slate-400 text-sm">Your llama.cpp version is too old for this GGUF file. Pull latest changes and rebuild, or download an older GGUF version (v1 or v2).</p>
</div>
<div class="glass p-6 rounded-xl border-l-4 border-green-500">
<h3 class="text-lg font-semibold text-white mb-2">Model Output is Gibberish</h3>
<p class="text-slate-400 text-sm mb-2">Random characters or nonsensical output.</p>
<p class="text-slate-400 text-sm">Usually indicates wrong tokenizer or incompatible model. Ensure you're using the correct prompt template for the model (e.g., ChatML for Mistral).</p>
</div>
</div>
</div>
</section>
<!-- Community Section -->
<section class="py-20">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<div class="text-center mb-12">
<h2 class="text-3xl md:text-4xl font-bold mb-4 text-white">Community & Resources</h2>
<p class="text-slate-400">Get help and stay updated.</p>
</div>
<div class="grid md:grid-cols-4 gap-6">
<a href="https://github.com/ggerganov/llama.cpp/discussions" target="_blank" class="glass p-6 rounded-xl glass-hover transition duration-300 text-center">
<i data-lucide="message-square" class="w-8 h-8 text-primary mx-auto mb-3"></i>
<h3 class="text-lg font-semibold text-white mb-1">GitHub Discussions</h3>
<p class="text-slate-400 text-sm">Community support</p>
</a>
<a href="https://discord.gg/llama-cpp" target="_blank" class="glass p-6 rounded-xl glass-hover transition duration-300 text-center">
<i data-lucide="message-circle" class="w-8 h-8 text-indigo-500 mx-auto mb-3"></i>
<h3 class="text-lg font-semibold text-white mb-1">Discord</h3>
<p class="text-slate-400 text-sm">Real-time chat</p>
</a>
<a href="https://www.reddit.com/r/LocalLLaMA/" target="_blank" class="glass p-6 rounded-xl glass-hover transition duration-300 text-center">
<i data-lucide="users" class="w-8 h-8 text-orange-500 mx-auto mb-3"></i>
<h3 class="text-lg font-semibold text-white mb-1">r/LocalLLaMA</h3>
<p class="text-slate-400 text-sm">Reddit community</p>
</a>
<a href="https://github.com/ggerganov/llama.cpp/blob/master/README.md" target="_blank" class="glass p-6 rounded-xl glass-hover transition duration-300 text-center">
<i data-lucide="book" class="w-8 h-8 text-blue-500 mx-auto mb-3"></i>
<h3 class="text-lg font-semibold text-white mb-1">Documentation</h3>
<p class="text-slate-400 text-sm">Official wiki</p>
</a>
</div>
</div>
</section>
<!-- Footer -->
<footer class="bg-slate-900 border-t border-slate-800 py-12">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
<div class="flex flex-col md:flex-row justify-between items-center">
<div class="mb-4 md:mb-0">
<div class="flex items-center space-x-2 mb-2">
<i data-lucide="brain-circuit" class="w-6 h-6 text-primary"></i>
<span class="text-lg font-bold text-white">llama.cpp</span>
</div>
<p class="text-slate-400 text-sm">The ultimate guide to running LLMs locally</p>
</div>
<div class="flex space-x-6">
<a href="https://github.com/ggerganov/llama.cpp" target="_blank" class="text-slate-400 hover:text-white transition">
<i data-lucide="github" class="w-6 h-6"></i>
</a>
<a href="https://huggingface.co/TheBloke" target="_blank" class="text-slate-400 hover:text-white transition">
<i data-lucide="database" class="w-6 h-6"></i>
</a>
<a href="#" class="text-slate-400 hover:text-white transition">
<i data-lucide="twitter" class="w-6 h-6"></i>
</a>
</div>
</div>
<div class="mt-8 pt-8 border-t border-slate-800 text-center text-slate-500 text-sm">
<p>Not affiliated with Meta or Facebook. llama.cpp is created by Georgi Gerganov and community.</p>
</div>
</div>
</footer>
<script>
// Initialize Lucide icons
lucide.createIcons();
// Mobile menu toggle
const mobileMenuBtn = document.getElementById('mobile-menu-btn');
const mobileMenu = document.getElementById('mobile-menu');
mobileMenuBtn.addEventListener('click', () => {
mobileMenu.classList.toggle('hidden');
});
// Close mobile menu when clicking a link
mobileMenu.querySelectorAll('a').forEach(link => {
link.addEventListener('click', () => {
mobileMenu.classList.add('hidden');
});
});
// Copy code functionality
function copyCode(btn) {
const codeBlock = btn.parentElement.querySelector('code, pre');
const text = codeBlock.textContent;
navigator.clipboard.writeText(text).then(() => {
const originalIcon = btn.innerHTML;
btn.innerHTML = '<i data-lucide="check" class="w-4 h-4"></i>';
lucide.createIcons();
setTimeout(() => {
btn.innerHTML = originalIcon;
lucide.createIcons();
}, 2000);
});
}
// Smooth scroll for anchor links
document.querySelectorAll('a[href^="#"]').forEach(anchor => {
anchor.addEventListener('click', function (e) {
e.preventDefault();
const target = document.querySelector(this.getAttribute('href'));
if (target) {
target.scrollIntoView({ behavior: 'smooth', block: 'start' });
}
});
});
</script>
<script src="https://deepsite.hf.co/deepsite-badge.js"></script>
</body>
</html> |