Update inference.cpp
Browse files- inference.cpp +676 -318
inference.cpp
CHANGED
|
@@ -1,329 +1,687 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
<
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
<div id="stat-kv" class="text-teal-400 text-sm font-bold mt-0.5">0 / 1024</div>
|
| 85 |
-
</div>
|
| 86 |
-
|
| 87 |
-
</div>
|
| 88 |
-
</div>
|
| 89 |
-
</header>
|
| 90 |
-
|
| 91 |
-
<!-- Main Content -->
|
| 92 |
-
<div class="flex-1 flex overflow-hidden">
|
| 93 |
-
|
| 94 |
-
<!-- Chat Messages -->
|
| 95 |
-
<main class="flex-1 flex flex-col">
|
| 96 |
-
<div id="chat-box" class="flex-1 overflow-y-auto p-6 space-y-5">
|
| 97 |
-
<div class="max-w-3xl mx-auto bg-slate-900/90 border border-slate-800 rounded-xl p-4 text-sm text-slate-300">
|
| 98 |
-
π <b>OmegaMind (152.8M)</b> is live. Every response tracks <b>TTFT (ms)</b>, <b>Total Output Time</b>, <b>Avg Speed per Token (tok/s & ms/tok)</b>, <b>CPU %</b>, and <b>RAM %</b> in real time.
|
| 99 |
-
</div>
|
| 100 |
-
</div>
|
| 101 |
-
|
| 102 |
-
<!-- Input Bar -->
|
| 103 |
-
<div class="border-t border-slate-800 bg-slate-900/60 p-4">
|
| 104 |
-
<form onsubmit="sendMessage(event)" class="max-w-3xl mx-auto flex gap-3">
|
| 105 |
-
<input
|
| 106 |
-
id="user-input"
|
| 107 |
-
type="text"
|
| 108 |
-
placeholder="Ask OmegaMind anything (e.g., Write a Python function to reverse a string)..."
|
| 109 |
-
class="flex-1 bg-slate-950 border border-slate-800 rounded-xl px-4 py-3 text-sm text-white focus:outline-none focus:border-sky-500"
|
| 110 |
-
autocomplete="off"
|
| 111 |
-
/>
|
| 112 |
-
<button
|
| 113 |
-
id="send-btn"
|
| 114 |
-
type="submit"
|
| 115 |
-
class="bg-sky-600 hover:bg-sky-500 text-white font-medium px-6 py-3 rounded-xl text-sm transition disabled:opacity-40"
|
| 116 |
-
>
|
| 117 |
-
Send
|
| 118 |
-
</button>
|
| 119 |
-
</form>
|
| 120 |
-
</div>
|
| 121 |
-
</main>
|
| 122 |
-
|
| 123 |
-
<!-- Sampling Sidebar -->
|
| 124 |
-
<aside class="w-72 border-l border-slate-800 bg-slate-900/50 p-5 hidden lg:flex flex-col gap-5 text-xs">
|
| 125 |
-
<h2 class="font-semibold text-slate-200 uppercase tracking-wider">Sampling Controls</h2>
|
| 126 |
-
|
| 127 |
-
<div>
|
| 128 |
-
<div class="flex justify-between mb-1"><span>Temperature</span><span id="val-temp" class="font-mono text-sky-400">0.25</span></div>
|
| 129 |
-
<input id="inp-temp" type="range" min="0.05" max="1.0" step="0.05" value="0.25" oninput="document.getElementById('val-temp').innerText=this.value" class="w-full accent-sky-500" />
|
| 130 |
-
</div>
|
| 131 |
-
|
| 132 |
-
<div>
|
| 133 |
-
<div class="flex justify-between mb-1"><span>Top-K</span><span id="val-topk" class="font-mono text-sky-400">8</span></div>
|
| 134 |
-
<input id="inp-topk" type="range" min="1" max="50" step="1" value="8" oninput="document.getElementById('val-topk').innerText=this.value" class="w-full accent-sky-500" />
|
| 135 |
-
</div>
|
| 136 |
-
|
| 137 |
-
<div>
|
| 138 |
-
<div class="flex justify-between mb-1"><span>Top-P (Nucleus)</span><span id="val-topp" class="font-mono text-sky-400">0.85</span></div>
|
| 139 |
-
<input id="inp-topp" type="range" min="0.10" max="1.0" step="0.05" value="0.85" oninput="document.getElementById('val-topp').innerText=this.value" class="w-full accent-sky-500" />
|
| 140 |
-
</div>
|
| 141 |
-
|
| 142 |
-
<div>
|
| 143 |
-
<div class="flex justify-between mb-1"><span>Repetition Penalty</span><span id="val-rep" class="font-mono text-sky-400">1.18</span></div>
|
| 144 |
-
<input id="inp-rep" type="range" min="1.00" max="1.40" step="0.02" value="1.18" oninput="document.getElementById('val-rep').innerText=this.value" class="w-full accent-sky-500" />
|
| 145 |
-
</div>
|
| 146 |
-
|
| 147 |
-
<div>
|
| 148 |
-
<div class="flex justify-between mb-1"><span>Max Output Tokens</span><span id="val-max" class="font-mono text-sky-400">256</span></div>
|
| 149 |
-
<input id="inp-max" type="range" min="64" max="512" step="32" value="256" oninput="document.getElementById('val-max').innerText=this.value" class="w-full accent-sky-500" />
|
| 150 |
-
</div>
|
| 151 |
-
|
| 152 |
-
<div class="mt-auto p-3 rounded-lg bg-slate-950 border border-slate-800 text-slate-400 space-y-1">
|
| 153 |
-
<div><b>Repo:</b> NOT-OMEGA/OmegaMind</div>
|
| 154 |
-
<div><b>Context:</b> 1024 tokens</div>
|
| 155 |
-
<div><b>SIMD:</b> AVX2 + FMA3 + OpenMP</div>
|
| 156 |
-
</div>
|
| 157 |
-
</aside>
|
| 158 |
-
</div>
|
| 159 |
-
|
| 160 |
-
<script>
|
| 161 |
-
const sessionId = "sess_" + Math.random().toString(36).substring(2, 10);
|
| 162 |
-
let isFirstTurn = true;
|
| 163 |
-
|
| 164 |
-
function updateDashboard(d) {
|
| 165 |
-
if (d.ttft_ms !== undefined) {
|
| 166 |
-
document.getElementById("stat-ttft").innerText = `${d.ttft_ms} ms`;
|
| 167 |
-
}
|
| 168 |
-
if (d.total_ms !== undefined) {
|
| 169 |
-
const sec = (d.total_ms / 1000).toFixed(2);
|
| 170 |
-
document.getElementById("stat-total-time").innerText = `${d.total_ms} ms (${sec}s)`;
|
| 171 |
-
}
|
| 172 |
-
if (d.tok_s !== undefined) {
|
| 173 |
-
document.getElementById("stat-speed").innerText = `${d.tok_s} tok/s`;
|
| 174 |
-
}
|
| 175 |
-
if (d.ms_per_tok !== undefined) {
|
| 176 |
-
document.getElementById("stat-ms-tok").innerText = `${d.ms_per_tok} ms / token`;
|
| 177 |
-
}
|
| 178 |
-
if (d.cpu_pct !== undefined) {
|
| 179 |
-
document.getElementById("stat-cpu").innerText = `${d.cpu_pct}%`;
|
| 180 |
-
document.getElementById("bar-cpu").style.width = `${Math.min(d.cpu_pct, 100)}%`;
|
| 181 |
-
}
|
| 182 |
-
if (d.ram_pct !== undefined) {
|
| 183 |
-
document.getElementById("stat-ram").innerText = `${d.ram_pct}%`;
|
| 184 |
-
document.getElementById("bar-ram").style.width = `${Math.min(d.ram_pct, 100)}%`;
|
| 185 |
-
}
|
| 186 |
-
if (d.ram_used_gb !== undefined && d.ram_total_gb !== undefined) {
|
| 187 |
-
document.getElementById("stat-ram-gb").innerText = `${d.ram_used_gb} / ${d.ram_total_gb} GB`;
|
| 188 |
-
}
|
| 189 |
-
if (d.kv_pos !== undefined) {
|
| 190 |
-
document.getElementById("stat-kv").innerText = `${d.kv_pos} / 1024`;
|
| 191 |
-
}
|
| 192 |
}
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 232 |
}
|
| 233 |
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
let buffer = "";
|
| 268 |
-
|
| 269 |
-
while (true) {
|
| 270 |
-
const { value, done } = await reader.read();
|
| 271 |
-
if (done) break;
|
| 272 |
-
buffer += decoder.decode(value, { stream: true });
|
| 273 |
-
|
| 274 |
-
const lines = buffer.split("\n\n");
|
| 275 |
-
buffer = lines.pop();
|
| 276 |
-
|
| 277 |
-
for (const line of lines) {
|
| 278 |
-
if (!line.startsWith("data: ")) continue;
|
| 279 |
-
const data = JSON.parse(line.slice(6));
|
| 280 |
-
|
| 281 |
-
if (data.error) {
|
| 282 |
-
bBubble.innerHTML = `<span class="text-rose-400">Error: ${data.error}</span>`;
|
| 283 |
-
break;
|
| 284 |
}
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 290 |
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 303 |
}
|
| 304 |
-
}
|
| 305 |
}
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
btn.disabled = false;
|
| 310 |
-
input.focus();
|
| 311 |
-
}
|
| 312 |
}
|
| 313 |
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
isFirstTurn = true;
|
| 321 |
-
document.getElementById("stat-kv").innerText = "0 / 1024";
|
| 322 |
-
document.getElementById("chat-box").innerHTML = `
|
| 323 |
-
<div class="max-w-3xl mx-auto bg-slate-900/90 border border-slate-800 rounded-xl p-4 text-sm text-emerald-400">
|
| 324 |
-
β
KV Cache cleared! Ready for a new conversation.
|
| 325 |
-
</div>`;
|
| 326 |
}
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/*
|
| 2 |
+
* ============================================================
|
| 3 |
+
* OmegaMind β Zero-Dependency C++17 AVX2/FMA Inference Engine
|
| 4 |
+
* Compatible with NOT-OMEGA/OmegaMind (model.bin & tokenizer.bin)
|
| 5 |
+
* Features:
|
| 6 |
+
* - Exact 5x int32 header mapping (n_layer, n_head, n_embd, block_size, vocab_size)
|
| 7 |
+
* - Batch Prefill (OpenMP + AVX2/FMA)
|
| 8 |
+
* - Multi-session LRU KV-Cache with auto-sliding window
|
| 9 |
+
* - Repetition Penalty + Temperature + Top-K + Top-P (Nucleus) Sampling
|
| 10 |
+
* ============================================================
|
| 11 |
+
*/
|
| 12 |
+
#include <iostream>
|
| 13 |
+
#include <cstdio>
|
| 14 |
+
#include <cstdlib>
|
| 15 |
+
#include <cmath>
|
| 16 |
+
#include <cstring>
|
| 17 |
+
#include <ctime>
|
| 18 |
+
#include <cstdint>
|
| 19 |
+
#include <algorithm>
|
| 20 |
+
#include <string>
|
| 21 |
+
#include <vector>
|
| 22 |
+
#include <unordered_map>
|
| 23 |
+
#include <unordered_set>
|
| 24 |
+
#include <immintrin.h>
|
| 25 |
+
|
| 26 |
+
#ifdef _OPENMP
|
| 27 |
+
#include <omp.h>
|
| 28 |
+
#endif
|
| 29 |
+
|
| 30 |
+
#ifdef _WIN32
|
| 31 |
+
#include <windows.h>
|
| 32 |
+
static double get_ms() {
|
| 33 |
+
LARGE_INTEGER f, c;
|
| 34 |
+
QueryPerformanceFrequency(&f); QueryPerformanceCounter(&c);
|
| 35 |
+
return (double)c.QuadPart / f.QuadPart * 1000.0;
|
| 36 |
+
}
|
| 37 |
+
#else
|
| 38 |
+
#include <sys/time.h>
|
| 39 |
+
#include <sys/mman.h>
|
| 40 |
+
#include <sys/stat.h>
|
| 41 |
+
#include <fcntl.h>
|
| 42 |
+
#include <unistd.h>
|
| 43 |
+
static double get_ms() {
|
| 44 |
+
struct timeval tv; gettimeofday(&tv, NULL);
|
| 45 |
+
return tv.tv_sec * 1000.0 + tv.tv_usec / 1000.0;
|
| 46 |
+
}
|
| 47 |
+
#endif
|
| 48 |
+
|
| 49 |
+
static void* mmap_file(const char* filepath, size_t* size_out) {
|
| 50 |
+
#ifdef _WIN32
|
| 51 |
+
HANDLE hFile = CreateFileA(filepath, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
|
| 52 |
+
if (hFile == INVALID_HANDLE_VALUE) return nullptr;
|
| 53 |
+
LARGE_INTEGER liSize;
|
| 54 |
+
if (!GetFileSizeEx(hFile, &liSize)) { CloseHandle(hFile); return nullptr; }
|
| 55 |
+
*size_out = liSize.QuadPart;
|
| 56 |
+
HANDLE hMap = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
|
| 57 |
+
if (!hMap) { CloseHandle(hFile); return nullptr; }
|
| 58 |
+
void* ptr = MapViewOfFile(hMap, FILE_MAP_READ, 0, 0, 0);
|
| 59 |
+
CloseHandle(hMap);
|
| 60 |
+
CloseHandle(hFile);
|
| 61 |
+
return ptr;
|
| 62 |
+
#else
|
| 63 |
+
int fd = open(filepath, O_RDONLY);
|
| 64 |
+
if (fd < 0) return nullptr;
|
| 65 |
+
struct stat sb;
|
| 66 |
+
if (fstat(fd, &sb) < 0) { close(fd); return nullptr; }
|
| 67 |
+
*size_out = sb.st_size;
|
| 68 |
+
void* ptr = mmap(NULL, sb.st_size, PROT_READ, MAP_SHARED, fd, 0);
|
| 69 |
+
close(fd);
|
| 70 |
+
if (ptr == MAP_FAILED) return nullptr;
|
| 71 |
+
return ptr;
|
| 72 |
+
#endif
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
static thread_local uint64_t tl_rng_state = 0;
|
| 76 |
+
static inline float rng_float() {
|
| 77 |
+
if (!tl_rng_state) {
|
| 78 |
+
uint64_t tid = 0;
|
| 79 |
+
#ifdef _OPENMP
|
| 80 |
+
tid = (uint64_t)omp_get_thread_num();
|
| 81 |
+
#endif
|
| 82 |
+
tl_rng_state = (uint64_t)get_ms() ^ (uint64_t)(uintptr_t)&tl_rng_state ^ (tid * 6364136223846793005ULL + 1);
|
| 83 |
+
if (!tl_rng_state) tl_rng_state = 0xDEADBEEFCAFEBABEULL;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
}
|
| 85 |
+
tl_rng_state ^= tl_rng_state << 13;
|
| 86 |
+
tl_rng_state ^= tl_rng_state >> 7;
|
| 87 |
+
tl_rng_state ^= tl_rng_state << 17;
|
| 88 |
+
return (float)(tl_rng_state >> 11) * (1.0f / (float)(1ULL << 53));
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
struct Config {
|
| 92 |
+
int n_layer;
|
| 93 |
+
int n_head;
|
| 94 |
+
int n_embd;
|
| 95 |
+
int block_size;
|
| 96 |
+
int vocab_size;
|
| 97 |
+
};
|
| 98 |
+
|
| 99 |
+
struct Weights {
|
| 100 |
+
float *wte, *wpe;
|
| 101 |
+
float **ln1_w, **ln1_b, **c_attn_w, **c_attn_b;
|
| 102 |
+
float **c_proj_w, **c_proj_b, **ln2_w, **ln2_b;
|
| 103 |
+
float **fc_w, **fc_b, **mlp_proj_w, **mlp_proj_b;
|
| 104 |
+
float *ln_f_w, *ln_f_b, *lm_head_w;
|
| 105 |
+
};
|
| 106 |
+
|
| 107 |
+
static Config cfg;
|
| 108 |
+
static Weights W;
|
| 109 |
+
static float* g_data = nullptr;
|
| 110 |
+
|
| 111 |
+
struct Session {
|
| 112 |
+
float* k_cache = nullptr;
|
| 113 |
+
float* v_cache = nullptr;
|
| 114 |
+
int pos = 0;
|
| 115 |
+
double last_use = 0.0;
|
| 116 |
+
std::vector<int> history;
|
| 117 |
+
};
|
| 118 |
+
|
| 119 |
+
static const int MAX_SESSIONS = 16;
|
| 120 |
+
static std::unordered_map<std::string, Session> g_sessions;
|
| 121 |
+
|
| 122 |
+
/* Single-token generation buffers */
|
| 123 |
+
static float *g_x, *g_buf, *g_qkv, *g_attn_buf;
|
| 124 |
+
static float *g_ff, *g_logits, *g_tmp_out;
|
| 125 |
+
static std::pair<float,int>* g_topk_pairs = nullptr;
|
| 126 |
+
|
| 127 |
+
/* Batch-prefill workspace */
|
| 128 |
+
static float *g_px = nullptr;
|
| 129 |
+
static float *g_pbuf = nullptr;
|
| 130 |
+
static float *g_pqkv = nullptr;
|
| 131 |
+
|
| 132 |
+
static thread_local float* tl_tmp = nullptr;
|
| 133 |
+
static thread_local float* tl_ff = nullptr;
|
| 134 |
+
|
| 135 |
+
static void ensure_thread_locals() {
|
| 136 |
+
if (!tl_tmp) {
|
| 137 |
+
tl_tmp = (float*)malloc(cfg.n_embd * sizeof(float));
|
| 138 |
+
tl_ff = (float*)malloc(4 * cfg.n_embd * sizeof(float));
|
| 139 |
+
if (!tl_tmp || !tl_ff) {
|
| 140 |
+
fprintf(stderr, "FATAL: thread-local buffer OOM\n");
|
| 141 |
+
exit(1);
|
| 142 |
+
}
|
| 143 |
+
}
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
/* ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 147 |
+
Core AVX2 / FMA Math Primitives
|
| 148 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββ */
|
| 149 |
+
|
| 150 |
+
static void layer_norm(float* out, const float* x, const float* w, const float* b, int N) {
|
| 151 |
+
__m256 vsum = _mm256_setzero_ps();
|
| 152 |
+
for (int i = 0; i < N; i += 8)
|
| 153 |
+
vsum = _mm256_add_ps(vsum, _mm256_loadu_ps(x + i));
|
| 154 |
+
float tmp[8]; _mm256_storeu_ps(tmp, vsum);
|
| 155 |
+
float mean = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
|
| 156 |
+
|
| 157 |
+
__m256 vmean = _mm256_set1_ps(mean);
|
| 158 |
+
__m256 vvar = _mm256_setzero_ps();
|
| 159 |
+
for (int i = 0; i < N; i += 8) {
|
| 160 |
+
__m256 d = _mm256_sub_ps(_mm256_loadu_ps(x + i), vmean);
|
| 161 |
+
vvar = _mm256_fmadd_ps(d, d, vvar);
|
| 162 |
+
}
|
| 163 |
+
_mm256_storeu_ps(tmp, vvar);
|
| 164 |
+
float var = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
|
| 165 |
+
__m256 vsc = _mm256_set1_ps(1.f / sqrtf(var + 1e-5f));
|
| 166 |
+
|
| 167 |
+
for (int i = 0; i < N; i += 8) {
|
| 168 |
+
__m256 d = _mm256_sub_ps(_mm256_loadu_ps(x + i), vmean);
|
| 169 |
+
__m256 norm = _mm256_mul_ps(d, vsc);
|
| 170 |
+
__m256 result = _mm256_fmadd_ps(norm, _mm256_loadu_ps(w + i), _mm256_loadu_ps(b + i));
|
| 171 |
+
_mm256_storeu_ps(out + i, result);
|
| 172 |
+
}
|
| 173 |
+
float sc = 1.f / sqrtf(var + 1e-5f);
|
| 174 |
+
for (int i = (N & ~7); i < N; i++)
|
| 175 |
+
out[i] = (x[i] - mean) * sc * w[i] + b[i];
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
static void matmul_vec(float* __restrict__ out, const float* __restrict__ mat,
|
| 179 |
+
const float* __restrict__ x, int M, int K) {
|
| 180 |
+
#pragma omp parallel for schedule(static) if(M >= 64)
|
| 181 |
+
for (int i = 0; i < M; i++) {
|
| 182 |
+
const float* row = mat + (long long)i * K;
|
| 183 |
+
__m256 acc = _mm256_setzero_ps();
|
| 184 |
+
int j = 0;
|
| 185 |
+
for (; j <= K-8; j += 8)
|
| 186 |
+
acc = _mm256_fmadd_ps(_mm256_loadu_ps(row+j), _mm256_loadu_ps(x+j), acc);
|
| 187 |
+
float t[8]; _mm256_storeu_ps(t, acc);
|
| 188 |
+
float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
|
| 189 |
+
for (; j < K; j++) s += row[j] * x[j];
|
| 190 |
+
out[i] = s;
|
| 191 |
+
}
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
static inline void matmul_vec_serial(float* __restrict__ out,
|
| 195 |
+
const float* __restrict__ mat,
|
| 196 |
+
const float* __restrict__ x,
|
| 197 |
+
int M, int K) {
|
| 198 |
+
for (int i = 0; i < M; i++) {
|
| 199 |
+
const float* row = mat + (long long)i * K;
|
| 200 |
+
__m256 acc = _mm256_setzero_ps();
|
| 201 |
+
int j = 0;
|
| 202 |
+
for (; j <= K-8; j += 8)
|
| 203 |
+
acc = _mm256_fmadd_ps(_mm256_loadu_ps(row+j), _mm256_loadu_ps(x+j), acc);
|
| 204 |
+
float t[8]; _mm256_storeu_ps(t, acc);
|
| 205 |
+
float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
|
| 206 |
+
for (; j < K; j++) s += row[j] * x[j];
|
| 207 |
+
out[i] = s;
|
| 208 |
+
}
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
static inline float dot_avx2(const float* __restrict__ a, const float* __restrict__ b, int n) {
|
| 212 |
+
__m256 acc = _mm256_setzero_ps();
|
| 213 |
+
int i = 0;
|
| 214 |
+
for (; i <= n-8; i += 8)
|
| 215 |
+
acc = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc);
|
| 216 |
+
float tmp[8]; _mm256_storeu_ps(tmp, acc);
|
| 217 |
+
float s = tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+t[6]+tmp[7];
|
| 218 |
+
for (; i < n; i++) s += a[i]*b[i];
|
| 219 |
+
return s;
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
static inline float dot_avx2_safe(const float* __restrict__ a, const float* __restrict__ b, int n) {
|
| 223 |
+
__m256 acc = _mm256_setzero_ps();
|
| 224 |
+
int i = 0;
|
| 225 |
+
for (; i <= n-8; i += 8)
|
| 226 |
+
acc = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc);
|
| 227 |
+
float tmp[8]; _mm256_storeu_ps(tmp, acc);
|
| 228 |
+
float s = tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7];
|
| 229 |
+
for (; i < n; i++) s += a[i]*b[i];
|
| 230 |
+
return s;
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
static inline void weighted_acc_avx2(float* __restrict__ out, const float* __restrict__ v, float w, int n) {
|
| 234 |
+
__m256 wv = _mm256_set1_ps(w);
|
| 235 |
+
int i = 0;
|
| 236 |
+
for (; i <= n-8; i += 8)
|
| 237 |
+
_mm256_storeu_ps(out+i, _mm256_fmadd_ps(wv, _mm256_loadu_ps(v+i), _mm256_loadu_ps(out+i)));
|
| 238 |
+
for (; i < n; i++) out[i] += w * v[i];
|
| 239 |
+
}
|
| 240 |
+
|
| 241 |
+
static inline void add_bias(float* x, const float* b, int N) {
|
| 242 |
+
for (int i = 0; i < N; i++) x[i] += b[i];
|
| 243 |
+
}
|
| 244 |
+
|
| 245 |
+
static inline void residual_add(float* x, const float* y, int N) {
|
| 246 |
+
for (int i = 0; i < N; i++) x[i] += y[i];
|
| 247 |
+
}
|
| 248 |
+
|
| 249 |
+
static void gelu_inplace(float* x, int N) {
|
| 250 |
+
for (int i = 0; i < N; i++) {
|
| 251 |
+
float v = x[i];
|
| 252 |
+
x[i] = 0.5f * v * (1.0f + tanhf(0.7978845608f * (v + 0.044715f * v * v * v)));
|
| 253 |
+
}
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
static void softmax_inplace(float* x, int N) {
|
| 257 |
+
float mx = x[0];
|
| 258 |
+
for (int i = 1; i < N; i++) if (x[i] > mx) mx = x[i];
|
| 259 |
+
float s = 0.f;
|
| 260 |
+
for (int i = 0; i < N; i++) { x[i] = expf(x[i]-mx); s += x[i]; }
|
| 261 |
+
for (int i = 0; i < N; i++) x[i] /= s;
|
| 262 |
+
}
|
| 263 |
+
|
| 264 |
+
/* ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 265 |
+
Forward Passes (Single-Token & Batch Prefill)
|
| 266 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββ */
|
| 267 |
+
static void forward(int token_id, int pos, float* k_cache, float* v_cache) {
|
| 268 |
+
const int C = cfg.n_embd, H = cfg.n_head, hs = C / H;
|
| 269 |
+
float* te = W.wte + (long long)token_id * C;
|
| 270 |
+
float* pe = W.wpe + (long long)pos * C;
|
| 271 |
+
#pragma omp parallel for
|
| 272 |
+
for (int i = 0; i < C; i++) g_x[i] = te[i] + pe[i];
|
| 273 |
+
|
| 274 |
+
for (int l = 0; l < cfg.n_layer; l++) {
|
| 275 |
+
layer_norm(g_buf, g_x, W.ln1_w[l], W.ln1_b[l], C);
|
| 276 |
+
matmul_vec(g_qkv, W.c_attn_w[l], g_buf, 3*C, C);
|
| 277 |
+
add_bias(g_qkv, W.c_attn_b[l], 3*C);
|
| 278 |
+
|
| 279 |
+
float* q = g_qkv;
|
| 280 |
+
float* k = g_qkv + C;
|
| 281 |
+
float* v = g_qkv + 2*C;
|
| 282 |
+
float* kc = k_cache + (long long)l * cfg.block_size * C;
|
| 283 |
+
float* vc = v_cache + (long long)l * cfg.block_size * C;
|
| 284 |
+
memcpy(kc + (long long)pos*C, k, C*sizeof(float));
|
| 285 |
+
memcpy(vc + (long long)pos*C, v, C*sizeof(float));
|
| 286 |
+
|
| 287 |
+
#pragma omp parallel for schedule(static)
|
| 288 |
+
for (int h = 0; h < H; h++) {
|
| 289 |
+
float* qh = q + h*hs;
|
| 290 |
+
float scale = 1.f / sqrtf((float)hs);
|
| 291 |
+
float* attn = g_attn_buf + h*cfg.block_size;
|
| 292 |
+
for (int t = 0; t <= pos; t++) {
|
| 293 |
+
float* kh = kc + (long long)t*C + h*hs;
|
| 294 |
+
attn[t] = dot_avx2_safe(qh, kh, hs) * scale;
|
| 295 |
+
}
|
| 296 |
+
softmax_inplace(attn, pos+1);
|
| 297 |
+
float* oh = g_buf + h*hs;
|
| 298 |
+
memset(oh, 0, hs*sizeof(float));
|
| 299 |
+
for (int t = 0; t <= pos; t++) {
|
| 300 |
+
float* vh = vc + (long long)t*C + h*hs;
|
| 301 |
+
weighted_acc_avx2(oh, vh, attn[t], hs);
|
| 302 |
+
}
|
| 303 |
+
}
|
| 304 |
+
matmul_vec(g_tmp_out, W.c_proj_w[l], g_buf, C, C);
|
| 305 |
+
add_bias(g_tmp_out, W.c_proj_b[l], C);
|
| 306 |
+
residual_add(g_x, g_tmp_out, C);
|
| 307 |
+
|
| 308 |
+
layer_norm(g_buf, g_x, W.ln2_w[l], W.ln2_b[l], C);
|
| 309 |
+
matmul_vec(g_ff, W.fc_w[l], g_buf, 4*C, C);
|
| 310 |
+
add_bias(g_ff, W.fc_b[l], 4*C);
|
| 311 |
+
gelu_inplace(g_ff, 4*C);
|
| 312 |
+
matmul_vec(g_tmp_out, W.mlp_proj_w[l], g_ff, C, 4*C);
|
| 313 |
+
add_bias(g_tmp_out, W.mlp_proj_b[l], C);
|
| 314 |
+
residual_add(g_x, g_tmp_out, C);
|
| 315 |
+
}
|
| 316 |
+
layer_norm(g_buf, g_x, W.ln_f_w, W.ln_f_b, C);
|
| 317 |
+
matmul_vec(g_logits, W.lm_head_w, g_buf, cfg.vocab_size, C);
|
| 318 |
+
}
|
| 319 |
+
|
| 320 |
+
static void forward_prefill_batch(const int* tokens, int T, int start_pos,
|
| 321 |
+
float* k_cache, float* v_cache) {
|
| 322 |
+
const int C = cfg.n_embd, H = cfg.n_head, hs = C / H;
|
| 323 |
+
|
| 324 |
+
for (int t = 0; t < T; t++) {
|
| 325 |
+
float* dst = g_px + (long long)t * C;
|
| 326 |
+
float* te = W.wte + (long long)tokens[t] * C;
|
| 327 |
+
float* pe = W.wpe + (long long)(start_pos + t) * C;
|
| 328 |
+
for (int i = 0; i < C; i++) dst[i] = te[i] + pe[i];
|
| 329 |
}
|
| 330 |
|
| 331 |
+
for (int l = 0; l < cfg.n_layer; l++) {
|
| 332 |
+
float* kc = k_cache + (long long)l * cfg.block_size * C;
|
| 333 |
+
float* vc = v_cache + (long long)l * cfg.block_size * C;
|
| 334 |
+
|
| 335 |
+
for (int t = 0; t < T; t++)
|
| 336 |
+
layer_norm(g_pbuf + (long long)t*C, g_px + (long long)t*C, W.ln1_w[l], W.ln1_b[l], C);
|
| 337 |
+
|
| 338 |
+
#pragma omp parallel for schedule(static)
|
| 339 |
+
for (int t = 0; t < T; t++) {
|
| 340 |
+
float* qkv = g_pqkv + (long long)t * 3 * C;
|
| 341 |
+
matmul_vec_serial(qkv, W.c_attn_w[l], g_pbuf + (long long)t*C, 3*C, C);
|
| 342 |
+
const float* bias = W.c_attn_b[l];
|
| 343 |
+
for (int i = 0; i < 3*C; i++) qkv[i] += bias[i];
|
| 344 |
+
memcpy(kc + (long long)(start_pos + t)*C, qkv + C, C*sizeof(float));
|
| 345 |
+
memcpy(vc + (long long)(start_pos + t)*C, qkv + 2*C, C*sizeof(float));
|
| 346 |
+
}
|
| 347 |
+
|
| 348 |
+
memset(g_pbuf, 0, (long long)T * C * sizeof(float));
|
| 349 |
+
const float scale = 1.f / sqrtf((float)hs);
|
| 350 |
+
|
| 351 |
+
#pragma omp parallel for schedule(static)
|
| 352 |
+
for (int h = 0; h < H; h++) {
|
| 353 |
+
std::vector<float> attn_scores_vec(cfg.block_size);
|
| 354 |
+
float* attn_scores = attn_scores_vec.data();
|
| 355 |
+
for (int t = 0; t < T; t++) {
|
| 356 |
+
float* qh = g_pqkv + (long long)t*3*C + h*hs;
|
| 357 |
+
int seq_len = start_pos + t + 1;
|
| 358 |
+
for (int s = 0; s < seq_len; s++)
|
| 359 |
+
attn_scores[s] = dot_avx2_safe(qh, kc + (long long)s*C + h*hs, hs) * scale;
|
| 360 |
+
softmax_inplace(attn_scores, seq_len);
|
| 361 |
+
float* oh = g_pbuf + (long long)t*C + h*hs;
|
| 362 |
+
for (int s = 0; s < seq_len; s++)
|
| 363 |
+
weighted_acc_avx2(oh, vc + (long long)s*C + h*hs, attn_scores[s], hs);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 364 |
}
|
| 365 |
+
}
|
| 366 |
+
|
| 367 |
+
#pragma omp parallel for schedule(static)
|
| 368 |
+
for (int t = 0; t < T; t++) {
|
| 369 |
+
ensure_thread_locals();
|
| 370 |
+
float* xt = g_px + (long long)t * C;
|
| 371 |
+
float* bt = g_pbuf + (long long)t * C;
|
| 372 |
+
matmul_vec_serial(tl_tmp, W.c_proj_w[l], bt, C, C);
|
| 373 |
+
const float* bias = W.c_proj_b[l];
|
| 374 |
+
for (int i = 0; i < C; i++) xt[i] += tl_tmp[i] + bias[i];
|
| 375 |
+
}
|
| 376 |
+
|
| 377 |
+
for (int t = 0; t < T; t++)
|
| 378 |
+
layer_norm(g_pbuf + (long long)t*C, g_px + (long long)t*C, W.ln2_w[l], W.ln2_b[l], C);
|
| 379 |
+
|
| 380 |
+
#pragma omp parallel for schedule(static)
|
| 381 |
+
for (int t = 0; t < T; t++) {
|
| 382 |
+
ensure_thread_locals();
|
| 383 |
+
float* xt = g_px + (long long)t * C;
|
| 384 |
+
float* bt = g_pbuf + (long long)t * C;
|
| 385 |
+
matmul_vec_serial(tl_ff, W.fc_w[l], bt, 4*C, C);
|
| 386 |
+
const float* bias_fc = W.fc_b[l];
|
| 387 |
+
for (int i = 0; i < 4*C; i++) tl_ff[i] += bias_fc[i];
|
| 388 |
+
gelu_inplace(tl_ff, 4*C);
|
| 389 |
+
matmul_vec_serial(tl_tmp, W.mlp_proj_w[l], tl_ff, C, 4*C);
|
| 390 |
+
const float* bias_p = W.mlp_proj_b[l];
|
| 391 |
+
for (int i = 0; i < C; i++) xt[i] += tl_tmp[i] + bias_p[i];
|
| 392 |
+
}
|
| 393 |
+
}
|
| 394 |
+
|
| 395 |
+
layer_norm(g_buf, g_px + (long long)(T-1)*C, W.ln_f_w, W.ln_f_b, C);
|
| 396 |
+
matmul_vec(g_logits, W.lm_head_w, g_buf, cfg.vocab_size, C);
|
| 397 |
+
}
|
| 398 |
+
|
| 399 |
+
/* ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 400 |
+
Weight Mapping
|
| 401 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββ */
|
| 402 |
+
static void map_weights(float* data, long long wbytes) {
|
| 403 |
+
float* p = data;
|
| 404 |
+
float* end = data + wbytes / sizeof(float);
|
| 405 |
+
const int C = cfg.n_embd, L = cfg.n_layer;
|
| 406 |
+
|
| 407 |
+
W.wte = p; p += (long long)cfg.vocab_size * C;
|
| 408 |
+
W.wpe = p; p += (long long)cfg.block_size * C;
|
| 409 |
+
|
| 410 |
+
#define ARR(f) W.f = (float**)malloc(L*sizeof(float*))
|
| 411 |
+
ARR(ln1_w); ARR(ln1_b); ARR(c_attn_w); ARR(c_attn_b);
|
| 412 |
+
ARR(c_proj_w); ARR(c_proj_b); ARR(ln2_w); ARR(ln2_b);
|
| 413 |
+
ARR(fc_w); ARR(fc_b); ARR(mlp_proj_w); ARR(mlp_proj_b);
|
| 414 |
+
#undef ARR
|
| 415 |
+
|
| 416 |
+
for (int l = 0; l < L; l++) {
|
| 417 |
+
W.ln1_w[l] = p; p += C;
|
| 418 |
+
W.ln1_b[l] = p; p += C;
|
| 419 |
+
W.c_attn_w[l] = p; p += 3LL*C*C;
|
| 420 |
+
W.c_attn_b[l] = p; p += 3LL*C;
|
| 421 |
+
W.c_proj_w[l] = p; p += 1LL*C*C;
|
| 422 |
+
W.c_proj_b[l] = p; p += C;
|
| 423 |
+
W.ln2_w[l] = p; p += C;
|
| 424 |
+
W.ln2_b[l] = p; p += C;
|
| 425 |
+
W.fc_w[l] = p; p += 4LL*C*C;
|
| 426 |
+
W.fc_b[l] = p; p += 4LL*C;
|
| 427 |
+
W.mlp_proj_w[l] = p; p += 1LL*C*4*C;
|
| 428 |
+
W.mlp_proj_b[l] = p; p += C;
|
| 429 |
+
}
|
| 430 |
+
W.ln_f_w = p; p += C;
|
| 431 |
+
W.ln_f_b = p; p += C;
|
| 432 |
+
W.lm_head_w = p; p += (long long)cfg.vocab_size * C;
|
| 433 |
+
|
| 434 |
+
if (p > end) {
|
| 435 |
+
long long needed = (p - data) * (long long)sizeof(float);
|
| 436 |
+
long long got = (end - data) * (long long)sizeof(float);
|
| 437 |
+
fprintf(stderr, "FATAL: model.bin truncated β needed %lld bytes, got %lld.\n", needed, got);
|
| 438 |
+
printf("ERROR model_truncated\n"); fflush(stdout);
|
| 439 |
+
exit(1);
|
| 440 |
+
}
|
| 441 |
+
}
|
| 442 |
+
|
| 443 |
+
/* ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 444 |
+
Session Management
|
| 445 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββ */
|
| 446 |
+
static long long kv_bytes() {
|
| 447 |
+
return (long long)cfg.n_layer * cfg.block_size * cfg.n_embd * sizeof(float);
|
| 448 |
+
}
|
| 449 |
+
static void free_session(Session& s) {
|
| 450 |
+
free(s.k_cache); free(s.v_cache);
|
| 451 |
+
s.k_cache = nullptr; s.v_cache = nullptr; s.pos = 0;
|
| 452 |
+
s.history.clear();
|
| 453 |
+
}
|
| 454 |
+
static void evict_oldest() {
|
| 455 |
+
if (g_sessions.empty()) return;
|
| 456 |
+
std::string oid; double ot = 1e300;
|
| 457 |
+
for (auto& kv : g_sessions)
|
| 458 |
+
if (kv.second.last_use < ot) { ot = kv.second.last_use; oid = kv.first; }
|
| 459 |
+
free_session(g_sessions[oid]);
|
| 460 |
+
g_sessions.erase(oid);
|
| 461 |
+
}
|
| 462 |
+
static Session& get_or_create(const std::string& id) {
|
| 463 |
+
auto it = g_sessions.find(id);
|
| 464 |
+
if (it != g_sessions.end()) { it->second.last_use = get_ms(); return it->second; }
|
| 465 |
+
if ((int)g_sessions.size() >= MAX_SESSIONS) evict_oldest();
|
| 466 |
+
long long nb = kv_bytes();
|
| 467 |
+
float* kc = (float*)calloc(nb, 1);
|
| 468 |
+
float* vc = (float*)calloc(nb, 1);
|
| 469 |
+
if (!kc || !vc) {
|
| 470 |
+
free(kc); free(vc);
|
| 471 |
+
printf("ERROR oom_kv_cache\n"); fflush(stdout);
|
| 472 |
+
static Session dummy;
|
| 473 |
+
return dummy;
|
| 474 |
+
}
|
| 475 |
+
auto pair = g_sessions.emplace(id, Session{});
|
| 476 |
+
auto ins_it = pair.first;
|
| 477 |
+
ins_it->second.k_cache = kc;
|
| 478 |
+
ins_it->second.v_cache = vc;
|
| 479 |
+
ins_it->second.pos = 0;
|
| 480 |
+
ins_it->second.last_use = get_ms();
|
| 481 |
+
return ins_it->second;
|
| 482 |
+
}
|
| 483 |
+
|
| 484 |
+
/* ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 485 |
+
Sampling: Repetition Penalty + Temp + Top-K + Top-P
|
| 486 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββ */
|
| 487 |
+
static int sample_token(float temperature, int top_k, float top_p, float rep_penalty,
|
| 488 |
+
const std::vector<int>& history) {
|
| 489 |
+
// 1. Apply Repetition Penalty on recent 64 tokens
|
| 490 |
+
if (rep_penalty > 1.0f && !history.empty()) {
|
| 491 |
+
int start_idx = std::max(0, (int)history.size() - 64);
|
| 492 |
+
std::unordered_set<int> recent(history.begin() + start_idx, history.end());
|
| 493 |
+
for (int tok_id : recent) {
|
| 494 |
+
if (tok_id >= 0 && tok_id < cfg.vocab_size) {
|
| 495 |
+
if (g_logits[tok_id] > 0.0f) g_logits[tok_id] /= rep_penalty;
|
| 496 |
+
else g_logits[tok_id] *= rep_penalty;
|
| 497 |
}
|
| 498 |
+
}
|
| 499 |
+
}
|
| 500 |
+
|
| 501 |
+
// 2. Temperature scaling & Top-K partial sort
|
| 502 |
+
float inv_temp = 1.0f / std::max(temperature, 1e-4f);
|
| 503 |
+
for (int v = 0; v < cfg.vocab_size; v++) {
|
| 504 |
+
g_topk_pairs[v] = {g_logits[v] * inv_temp, v};
|
| 505 |
+
}
|
| 506 |
+
|
| 507 |
+
int K = std::min(std::max(top_k, 1), cfg.vocab_size);
|
| 508 |
+
std::partial_sort(g_topk_pairs, g_topk_pairs + K, g_topk_pairs + cfg.vocab_size,
|
| 509 |
+
[](const auto& a, const auto& b){ return a.first > b.first; });
|
| 510 |
|
| 511 |
+
// 3. Softmax over Top-K candidates
|
| 512 |
+
float max_logit = g_topk_pairs[0].first;
|
| 513 |
+
float sum = 0.f;
|
| 514 |
+
for (int j = 0; j < K; j++) {
|
| 515 |
+
g_topk_pairs[j].first = expf(g_topk_pairs[j].first - max_logit);
|
| 516 |
+
sum += g_topk_pairs[j].first;
|
| 517 |
+
}
|
| 518 |
+
for (int j = 0; j < K; j++) g_topk_pairs[j].first /= sum;
|
| 519 |
+
|
| 520 |
+
// 4. Top-P (Nucleus) cutoff inside Top-K
|
| 521 |
+
int active_k = K;
|
| 522 |
+
if (top_p > 0.0f && top_p < 1.0f) {
|
| 523 |
+
float cum_prob = 0.0f;
|
| 524 |
+
for (int j = 0; j < K; j++) {
|
| 525 |
+
cum_prob += g_topk_pairs[j].first;
|
| 526 |
+
if (cum_prob >= top_p) {
|
| 527 |
+
active_k = j + 1;
|
| 528 |
+
break;
|
| 529 |
}
|
|
|
|
| 530 |
}
|
| 531 |
+
float p_sum = 0.0f;
|
| 532 |
+
for (int j = 0; j < active_k; j++) p_sum += g_topk_pairs[j].first;
|
| 533 |
+
for (int j = 0; j < active_k; j++) g_topk_pairs[j].first /= p_sum;
|
|
|
|
|
|
|
|
|
|
| 534 |
}
|
| 535 |
|
| 536 |
+
// 5. Multinomial sample
|
| 537 |
+
float r = rng_float(), cum = 0.f;
|
| 538 |
+
int best = g_topk_pairs[0].second;
|
| 539 |
+
for (int j = 0; j < active_k; j++) {
|
| 540 |
+
cum += g_topk_pairs[j].first;
|
| 541 |
+
if (r < cum) { best = g_topk_pairs[j].second; break; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 542 |
}
|
| 543 |
+
return best;
|
| 544 |
+
}
|
| 545 |
+
|
| 546 |
+
/* ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 547 |
+
Request Protocol
|
| 548 |
+
Format: REQUEST|sess_id|tokens|max_new|temp|top_k|stop_ids|top_p|rep_penalty
|
| 549 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββ */
|
| 550 |
+
static std::vector<std::string> split(const std::string& s, char d) {
|
| 551 |
+
std::vector<std::string> out; std::string cur;
|
| 552 |
+
for (char c : s) { if (c==d){out.push_back(cur);cur.clear();}else cur+=c; }
|
| 553 |
+
out.push_back(cur); return out;
|
| 554 |
+
}
|
| 555 |
+
static std::vector<int> parse_ints(const std::string& s) {
|
| 556 |
+
std::vector<int> out;
|
| 557 |
+
for (auto& t : split(s,',')) if (!t.empty()) out.push_back(atoi(t.c_str()));
|
| 558 |
+
return out;
|
| 559 |
+
}
|
| 560 |
+
|
| 561 |
+
static void handle_request(const std::string& line) {
|
| 562 |
+
auto parts = split(line, '|');
|
| 563 |
+
if (parts.size() < 7) { printf("ERROR bad_request_format\n"); fflush(stdout); return; }
|
| 564 |
+
|
| 565 |
+
std::string sess_id = parts[1];
|
| 566 |
+
auto new_tokens = parse_ints(parts[2]);
|
| 567 |
+
int max_new = atoi(parts[3].c_str());
|
| 568 |
+
float temp = (float)atof(parts[4].c_str());
|
| 569 |
+
int top_k = atoi(parts[5].c_str());
|
| 570 |
+
auto stop_list = parse_ints(parts[6]);
|
| 571 |
+
float top_p = (parts.size() >= 8) ? (float)atof(parts[7].c_str()) : 0.85f;
|
| 572 |
+
float rep_penalty = (parts.size() >= 9) ? (float)atof(parts[8].c_str()) : 1.18f;
|
| 573 |
+
|
| 574 |
+
temp = std::max(temp, 0.01f);
|
| 575 |
+
top_k = std::min(std::max(top_k, 1), cfg.vocab_size);
|
| 576 |
+
max_new = std::max(max_new, 1);
|
| 577 |
+
top_p = std::min(std::max(top_p, 0.05f), 1.0f);
|
| 578 |
+
rep_penalty = std::max(rep_penalty, 1.0f);
|
| 579 |
+
|
| 580 |
+
std::unordered_set<int> stop_ids(stop_list.begin(), stop_list.end());
|
| 581 |
+
stop_ids.insert(50256);
|
| 582 |
+
|
| 583 |
+
Session& sess = get_or_create(sess_id);
|
| 584 |
+
if (!sess.k_cache) return;
|
| 585 |
+
if (new_tokens.empty()) { printf("ERROR no_tokens\n"); fflush(stdout); return; }
|
| 586 |
+
|
| 587 |
+
// If single prompt exceeds block_size, keep the most recent tokens
|
| 588 |
+
if ((int)new_tokens.size() >= cfg.block_size - 4) {
|
| 589 |
+
new_tokens.erase(new_tokens.begin(), new_tokens.end() - (cfg.block_size - max_new - 8));
|
| 590 |
+
}
|
| 591 |
+
|
| 592 |
+
// Auto-reset session KV position if context window fills up
|
| 593 |
+
if (sess.pos + (int)new_tokens.size() >= cfg.block_size) {
|
| 594 |
+
sess.pos = 0;
|
| 595 |
+
sess.history.clear();
|
| 596 |
+
}
|
| 597 |
+
|
| 598 |
+
for (int t : new_tokens) sess.history.push_back(t);
|
| 599 |
+
|
| 600 |
+
double t0 = get_ms();
|
| 601 |
+
forward_prefill_batch(new_tokens.data(), (int)new_tokens.size(),
|
| 602 |
+
sess.pos, sess.k_cache, sess.v_cache);
|
| 603 |
+
sess.pos += (int)new_tokens.size();
|
| 604 |
+
|
| 605 |
+
int gen = 0;
|
| 606 |
+
for (int i = 0; i < max_new; i++) {
|
| 607 |
+
if (sess.pos >= cfg.block_size) break;
|
| 608 |
+
int next = sample_token(temp, top_k, top_p, rep_penalty, sess.history);
|
| 609 |
+
sess.history.push_back(next);
|
| 610 |
+
|
| 611 |
+
printf("TOKEN %d %.2f %d\n", next, get_ms() - t0, sess.pos);
|
| 612 |
+
fflush(stdout);
|
| 613 |
+
gen++;
|
| 614 |
+
|
| 615 |
+
if (stop_ids.count(next)) break;
|
| 616 |
+
forward(next, sess.pos, sess.k_cache, sess.v_cache);
|
| 617 |
+
sess.pos++;
|
| 618 |
+
}
|
| 619 |
+
printf("DONE %d %.2f %d\n", gen, get_ms() - t0, sess.pos);
|
| 620 |
+
fflush(stdout);
|
| 621 |
+
}
|
| 622 |
+
|
| 623 |
+
static void handle_reset(const std::string& line) {
|
| 624 |
+
auto parts = split(line, '|');
|
| 625 |
+
if (parts.size() >= 2) {
|
| 626 |
+
auto it = g_sessions.find(parts[1]);
|
| 627 |
+
if (it != g_sessions.end()) { free_session(it->second); g_sessions.erase(it); }
|
| 628 |
+
}
|
| 629 |
+
printf("RESET_OK\n"); fflush(stdout);
|
| 630 |
+
}
|
| 631 |
+
|
| 632 |
+
int main() {
|
| 633 |
+
size_t mapped_size = 0;
|
| 634 |
+
void* file_data = mmap_file("model.bin", &mapped_size);
|
| 635 |
+
if (!file_data) { printf("ERROR model.bin_not_found\n"); fflush(stdout); return 1; }
|
| 636 |
+
|
| 637 |
+
/* FIX: Read all 5 int32 values directly into Config (n_layer, n_head, n_embd, block_size, vocab_size) */
|
| 638 |
+
constexpr size_t MODEL_HEADER_BYTES = 5 * sizeof(int);
|
| 639 |
+
if (mapped_size < MODEL_HEADER_BYTES) {
|
| 640 |
+
printf("ERROR model.bin_header_truncated\n"); fflush(stdout); return 1;
|
| 641 |
+
}
|
| 642 |
+
|
| 643 |
+
memcpy(&cfg, (char*)file_data, MODEL_HEADER_BYTES);
|
| 644 |
+
fprintf(stderr, "[OmegaMind C++] Loaded Header: layers=%d, heads=%d, embd=%d, block=%d, vocab=%d\n",
|
| 645 |
+
cfg.n_layer, cfg.n_head, cfg.n_embd, cfg.block_size, cfg.vocab_size);
|
| 646 |
+
|
| 647 |
+
long long wbytes = (long long)mapped_size - (long long)MODEL_HEADER_BYTES;
|
| 648 |
+
g_data = (float*)((char*)file_data + MODEL_HEADER_BYTES);
|
| 649 |
+
map_weights(g_data, wbytes);
|
| 650 |
+
|
| 651 |
+
const int C = cfg.n_embd;
|
| 652 |
+
g_x = (float*)malloc(C * sizeof(float));
|
| 653 |
+
g_buf = (float*)malloc(C * sizeof(float));
|
| 654 |
+
g_qkv = (float*)malloc(3*C * sizeof(float));
|
| 655 |
+
g_attn_buf = (float*)malloc((long long)cfg.n_head * cfg.block_size * sizeof(float));
|
| 656 |
+
g_ff = (float*)malloc(4*C * sizeof(float));
|
| 657 |
+
g_logits = (float*)malloc((long long)cfg.vocab_size * sizeof(float));
|
| 658 |
+
g_tmp_out = (float*)malloc(C * sizeof(float));
|
| 659 |
+
g_topk_pairs = (std::pair<float,int>*)malloc((long long)cfg.vocab_size * sizeof(std::pair<float,int>));
|
| 660 |
+
|
| 661 |
+
g_px = (float*)malloc((long long)cfg.block_size * C * sizeof(float));
|
| 662 |
+
g_pbuf = (float*)malloc((long long)cfg.block_size * C * sizeof(float));
|
| 663 |
+
g_pqkv = (float*)malloc((long long)cfg.block_size * 3 * C * sizeof(float));
|
| 664 |
+
|
| 665 |
+
if (!g_x || !g_buf || !g_qkv || !g_attn_buf || !g_ff || !g_logits ||
|
| 666 |
+
!g_tmp_out || !g_topk_pairs || !g_px || !g_pbuf || !g_pqkv) {
|
| 667 |
+
printf("ERROR oom_buffers\n"); fflush(stdout); return 1;
|
| 668 |
+
}
|
| 669 |
+
|
| 670 |
+
printf("READY\n"); fflush(stdout);
|
| 671 |
+
|
| 672 |
+
std::string line;
|
| 673 |
+
while (std::getline(std::cin, line)) {
|
| 674 |
+
if (!line.empty() && line.back()=='\r') line.pop_back();
|
| 675 |
+
if (line.empty()) continue;
|
| 676 |
+
if (line == "QUIT") break;
|
| 677 |
+
else if (line.rfind("RESET|", 0)==0) handle_reset(line);
|
| 678 |
+
else if (line.rfind("REQUEST|", 0)==0) handle_request(line);
|
| 679 |
+
else { printf("ERROR unknown_cmd\n"); fflush(stdout); }
|
| 680 |
+
}
|
| 681 |
+
|
| 682 |
+
for (auto& kv : g_sessions) free_session(kv.second);
|
| 683 |
+
free(g_x); free(g_buf); free(g_qkv); free(g_attn_buf);
|
| 684 |
+
free(g_ff); free(g_logits); free(g_tmp_out); free(g_topk_pairs);
|
| 685 |
+
free(g_px); free(g_pbuf); free(g_pqkv);
|
| 686 |
+
return 0;
|
| 687 |
+
}
|