Update inference.cpp
Browse files- inference.cpp +271 -523
inference.cpp
CHANGED
|
@@ -1,7 +1,13 @@
|
|
| 1 |
/*
|
| 2 |
* ============================================================
|
| 3 |
-
* NanoMind —
|
| 4 |
* ============================================================
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
*/
|
| 6 |
#include <iostream>
|
| 7 |
#include <cstdio>
|
|
@@ -12,609 +18,351 @@
|
|
| 12 |
#include <algorithm>
|
| 13 |
#include <string>
|
| 14 |
#include <vector>
|
| 15 |
-
#include <unordered_map>
|
| 16 |
#include <unordered_set>
|
| 17 |
-
#include <
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
#ifdef _OPENMP
|
| 19 |
#include <omp.h>
|
| 20 |
#endif
|
| 21 |
#ifdef _WIN32
|
| 22 |
#include <windows.h>
|
| 23 |
-
static double get_ms() {
|
| 24 |
-
LARGE_INTEGER f, c;
|
| 25 |
-
QueryPerformanceFrequency(&f); QueryPerformanceCounter(&c);
|
| 26 |
-
return (double)c.QuadPart / f.QuadPart * 1000.0;
|
| 27 |
-
}
|
| 28 |
#else
|
| 29 |
#include <sys/time.h>
|
| 30 |
-
static double get_ms() {
|
| 31 |
-
struct timeval tv; gettimeofday(&tv, NULL);
|
| 32 |
-
return tv.tv_sec * 1000.0 + tv.tv_usec / 1000.0;
|
| 33 |
-
}
|
| 34 |
#endif
|
| 35 |
|
|
|
|
| 36 |
static thread_local uint64_t tl_rng_state = 0;
|
| 37 |
static inline float rng_float() {
|
| 38 |
-
if (!tl_rng_state) {
|
| 39 |
-
|
| 40 |
-
if (!tl_rng_state) tl_rng_state = 0xDEADBEEFCAFEBABEULL;
|
| 41 |
-
}
|
| 42 |
-
tl_rng_state ^= tl_rng_state << 13;
|
| 43 |
-
tl_rng_state ^= tl_rng_state >> 7;
|
| 44 |
-
tl_rng_state ^= tl_rng_state << 17;
|
| 45 |
return (float)(tl_rng_state >> 11) * (1.0f / (float)(1ULL << 53));
|
| 46 |
}
|
| 47 |
|
| 48 |
-
struct Config {
|
| 49 |
-
int n_layer, n_head, n_embd, block_size, vocab_size;
|
| 50 |
-
};
|
| 51 |
struct Weights {
|
| 52 |
-
float *wte, *wpe;
|
| 53 |
-
float **
|
| 54 |
-
float **
|
| 55 |
-
float **c_proj_w, **c_proj_b;
|
| 56 |
-
float **ln2_w, **ln2_b;
|
| 57 |
-
float **fc_w, **fc_b;
|
| 58 |
-
float **mlp_proj_w, **mlp_proj_b;
|
| 59 |
-
float *ln_f_w, *ln_f_b;
|
| 60 |
-
float *lm_head_w;
|
| 61 |
};
|
| 62 |
-
static Config
|
| 63 |
static Weights W;
|
| 64 |
static float* g_data = nullptr;
|
| 65 |
|
| 66 |
-
struct Session {
|
| 67 |
-
float* k_cache = nullptr;
|
| 68 |
-
float* v_cache = nullptr;
|
| 69 |
-
int pos = 0;
|
| 70 |
-
double last_use = 0.0;
|
| 71 |
-
};
|
| 72 |
-
static const int MAX_SESSIONS = 50;
|
| 73 |
-
static std::unordered_map<std::string, Session> g_sessions;
|
| 74 |
-
|
| 75 |
-
static float *g_x, *g_buf, *g_qkv, *g_attn_buf;
|
| 76 |
-
static float *g_ff, *g_logits, *g_tmp_out;
|
| 77 |
-
static std::pair<float,int>* g_topk_pairs = nullptr;
|
| 78 |
-
|
| 79 |
-
// ── BATCH ARRAYS ──
|
| 80 |
const int MAX_BATCH = 16;
|
| 81 |
static float *g_x_b[MAX_BATCH], *g_buf_b[MAX_BATCH], *g_qkv_b[MAX_BATCH];
|
| 82 |
static float *g_attn_buf_b[MAX_BATCH], *g_ff_b[MAX_BATCH], *g_tmp_out_b[MAX_BATCH], *g_logits_b[MAX_BATCH];
|
|
|
|
| 83 |
|
| 84 |
-
// ──
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
for (int i = 0; i < N; i += 8) vsum = _mm256_add_ps(vsum, _mm256_loadu_ps(x + i));
|
| 88 |
-
float tmp[8]; _mm256_storeu_ps(tmp, vsum);
|
| 89 |
-
float mean = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
|
| 90 |
-
|
| 91 |
-
__m256 vmean = _mm256_set1_ps(mean);
|
| 92 |
-
__m256 vvar = _mm256_setzero_ps();
|
| 93 |
-
for (int i = 0; i < N; i += 8) {
|
| 94 |
-
__m256 d = _mm256_sub_ps(_mm256_loadu_ps(x + i), vmean);
|
| 95 |
-
vvar = _mm256_fmadd_ps(d, d, vvar);
|
| 96 |
-
}
|
| 97 |
-
_mm256_storeu_ps(tmp, vvar);
|
| 98 |
-
float var = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
|
| 99 |
-
__m256 vsc = _mm256_set1_ps(1.f / sqrtf(var + 1e-5f));
|
| 100 |
-
|
| 101 |
-
for (int i = 0; i < N; i += 8) {
|
| 102 |
-
__m256 d = _mm256_sub_ps(_mm256_loadu_ps(x + i), vmean);
|
| 103 |
-
__m256 norm = _mm256_mul_ps(d, vsc);
|
| 104 |
-
__m256 result = _mm256_fmadd_ps(norm, _mm256_loadu_ps(w + i), _mm256_loadu_ps(b + i));
|
| 105 |
-
_mm256_storeu_ps(out + i, result);
|
| 106 |
-
}
|
| 107 |
-
float sc = 1.f / sqrtf(var + 1e-5f);
|
| 108 |
-
for (int i = (N & ~7); i < N; i++) out[i] = (x[i] - mean) * sc * w[i] + b[i];
|
| 109 |
-
}
|
| 110 |
|
| 111 |
-
static
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
for (int i = 0; i < M; i++) {
|
| 115 |
-
const float* row = mat + (long long)i * K;
|
| 116 |
-
__m256 acc = _mm256_setzero_ps();
|
| 117 |
-
int j = 0;
|
| 118 |
-
for (; j <= K-8; j += 8) acc = _mm256_fmadd_ps(_mm256_loadu_ps(row+j), _mm256_loadu_ps(x+j), acc);
|
| 119 |
-
float t[8]; _mm256_storeu_ps(t, acc);
|
| 120 |
-
float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
|
| 121 |
-
for (; j < K; j++) s += row[j] * x[j];
|
| 122 |
-
out[i] = s;
|
| 123 |
-
}
|
| 124 |
-
return;
|
| 125 |
-
}
|
| 126 |
-
#pragma omp parallel for schedule(static)
|
| 127 |
-
for (int i = 0; i < M; i++) {
|
| 128 |
-
const float* row = mat + (long long)i * K;
|
| 129 |
-
__m256 acc = _mm256_setzero_ps();
|
| 130 |
-
int j = 0;
|
| 131 |
-
for (; j <= K-8; j += 8) acc = _mm256_fmadd_ps(_mm256_loadu_ps(row+j), _mm256_loadu_ps(x+j), acc);
|
| 132 |
-
float t[8]; _mm256_storeu_ps(t, acc);
|
| 133 |
-
float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
|
| 134 |
-
for (; j < K; j++) s += row[j] * x[j];
|
| 135 |
-
out[i] = s;
|
| 136 |
-
}
|
| 137 |
-
}
|
| 138 |
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
float s = tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7];
|
| 145 |
-
for (; i < n; i++) s += a[i]*b[i];
|
| 146 |
-
return s;
|
| 147 |
}
|
| 148 |
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
int
|
| 152 |
-
|
| 153 |
-
_mm256_storeu_ps(out+i, _mm256_fmadd_ps(wv, _mm256_loadu_ps(v+i), _mm256_loadu_ps(out+i)));
|
| 154 |
-
for (; i < n; i++) out[i] += w * v[i];
|
| 155 |
}
|
| 156 |
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
}
|
| 161 |
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
}
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
const __m256 vlo = _mm256_set1_ps(-88.0f);
|
| 174 |
-
const __m256 vhi = _mm256_set1_ps(88.0f);
|
| 175 |
-
int i = 0;
|
| 176 |
-
for (; i <= N-8; i += 8) {
|
| 177 |
-
__m256 v = _mm256_loadu_ps(x + i);
|
| 178 |
-
__m256 t = _mm256_mul_ps(neg_c, v);
|
| 179 |
-
t = _mm256_max_ps(t, vlo); t = _mm256_min_ps(t, vhi);
|
| 180 |
-
__m256i ti = _mm256_cvttps_epi32(_mm256_fmadd_ps(t, scale, vbias));
|
| 181 |
-
__m256 et = _mm256_castsi256_ps(ti);
|
| 182 |
-
__m256 denom = _mm256_add_ps(vone, et);
|
| 183 |
-
__m256 r = _mm256_rcp_ps(denom);
|
| 184 |
-
r = _mm256_mul_ps(r, _mm256_fnmadd_ps(denom, r, vtwo));
|
| 185 |
-
_mm256_storeu_ps(x + i, _mm256_mul_ps(v, r));
|
| 186 |
}
|
| 187 |
-
for (; i < N; i++) x[i] = x[i] / (1.0f + expf(-1.702f * x[i]));
|
| 188 |
}
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
for (int i = 0; i < N; i++) { x[i] = expf(x[i]-mx); s += x[i]; }
|
| 195 |
-
for (int i = 0; i < N; i++) x[i] /= s;
|
| 196 |
}
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
const int hs = C / H;
|
| 202 |
-
|
| 203 |
-
float* te = W.wte + (long long)token_id * C;
|
| 204 |
-
float* pe = W.wpe + (long long)pos * C;
|
| 205 |
-
#pragma omp parallel for
|
| 206 |
-
for (int i = 0; i < C; i++) g_x[i] = te[i] + pe[i];
|
| 207 |
-
|
| 208 |
-
for (int l = 0; l < cfg.n_layer; l++) {
|
| 209 |
-
layer_norm(g_buf, g_x, W.ln1_w[l], W.ln1_b[l], C);
|
| 210 |
-
matmul_vec(g_qkv, W.c_attn_w[l], g_buf, 3*C, C);
|
| 211 |
-
add_bias(g_qkv, W.c_attn_b[l], 3*C);
|
| 212 |
-
|
| 213 |
-
float* q = g_qkv;
|
| 214 |
-
float* k = g_qkv + C;
|
| 215 |
-
float* v = g_qkv + 2*C;
|
| 216 |
-
float* kc = k_cache + (long long)l * cfg.block_size * C;
|
| 217 |
-
float* vc = v_cache + (long long)l * cfg.block_size * C;
|
| 218 |
-
memcpy(kc + (long long)pos*C, k, C*sizeof(float));
|
| 219 |
-
memcpy(vc + (long long)pos*C, v, C*sizeof(float));
|
| 220 |
-
|
| 221 |
-
#pragma omp parallel for schedule(static)
|
| 222 |
-
for (int h = 0; h < H; h++) {
|
| 223 |
-
float* qh = q + h*hs;
|
| 224 |
-
float scale = 1.f / sqrtf((float)hs);
|
| 225 |
-
float* attn = g_attn_buf + h*cfg.block_size;
|
| 226 |
-
for (int t = 0; t <= pos; t++) {
|
| 227 |
-
float* kh = kc + (long long)t*C + h*hs;
|
| 228 |
-
attn[t] = dot_avx2(qh, kh, hs) * scale;
|
| 229 |
-
}
|
| 230 |
-
softmax_inplace(attn, pos+1);
|
| 231 |
-
float* oh = g_buf + h*hs;
|
| 232 |
-
memset(oh, 0, hs*sizeof(float));
|
| 233 |
-
for (int t = 0; t <= pos; t++) {
|
| 234 |
-
float* vh = vc + (long long)t*C + h*hs;
|
| 235 |
-
weighted_acc_avx2(oh, vh, attn[t], hs);
|
| 236 |
-
}
|
| 237 |
-
}
|
| 238 |
-
|
| 239 |
-
matmul_vec(g_tmp_out, W.c_proj_w[l], g_buf, C, C);
|
| 240 |
-
add_bias(g_tmp_out, W.c_proj_b[l], C);
|
| 241 |
-
residual_add(g_x, g_tmp_out, C);
|
| 242 |
-
|
| 243 |
-
layer_norm(g_buf, g_x, W.ln2_w[l], W.ln2_b[l], C);
|
| 244 |
-
matmul_vec(g_ff, W.fc_w[l], g_buf, 4*C, C);
|
| 245 |
-
add_bias(g_ff, W.fc_b[l], 4*C);
|
| 246 |
-
gelu_inplace(g_ff, 4*C);
|
| 247 |
-
matmul_vec(g_tmp_out, W.mlp_proj_w[l], g_ff, C, 4*C);
|
| 248 |
-
add_bias(g_tmp_out, W.mlp_proj_b[l], C);
|
| 249 |
-
residual_add(g_x, g_tmp_out, C);
|
| 250 |
-
}
|
| 251 |
-
|
| 252 |
-
layer_norm(g_buf, g_x, W.ln_f_w, W.ln_f_b, C);
|
| 253 |
-
matmul_vec(g_logits, W.lm_head_w, g_buf, cfg.vocab_size, C);
|
| 254 |
}
|
|
|
|
|
|
|
| 255 |
|
| 256 |
-
static void
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
ARR(c_proj_w); ARR(c_proj_b); ARR(ln2_w); ARR(ln2_b);
|
| 265 |
-
ARR(fc_w); ARR(fc_b); ARR(mlp_proj_w); ARR(mlp_proj_b);
|
| 266 |
-
#undef ARR
|
| 267 |
-
for (int l = 0; l < L; l++) {
|
| 268 |
-
W.ln1_w[l] = p; p += C;
|
| 269 |
-
W.ln1_b[l] = p; p += C;
|
| 270 |
-
W.c_attn_w[l] = p; p += 3LL*C*C;
|
| 271 |
-
W.c_attn_b[l] = p; p += 3LL*C;
|
| 272 |
-
W.c_proj_w[l] = p; p += 1LL*C*C;
|
| 273 |
-
W.c_proj_b[l] = p; p += C;
|
| 274 |
-
W.ln2_w[l] = p; p += C;
|
| 275 |
-
W.ln2_b[l] = p; p += C;
|
| 276 |
-
W.fc_w[l] = p; p += 4LL*C*C;
|
| 277 |
-
W.fc_b[l] = p; p += 4LL*C;
|
| 278 |
-
W.mlp_proj_w[l] = p; p += 1LL*C*4*C;
|
| 279 |
-
W.mlp_proj_b[l] = p; p += C;
|
| 280 |
-
}
|
| 281 |
-
W.ln_f_w = p; p += C;
|
| 282 |
-
W.ln_f_b = p; p += C;
|
| 283 |
-
W.lm_head_w = p; p += (long long)cfg.vocab_size * C;
|
| 284 |
-
|
| 285 |
-
if (p > end) {
|
| 286 |
-
printf("ERROR model_truncated\n"); fflush(stdout);
|
| 287 |
-
exit(1);
|
| 288 |
}
|
|
|
|
| 289 |
}
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
if (g_sessions.empty()) return;
|
| 295 |
-
std::string oid; double ot = 1e300;
|
| 296 |
-
for (auto& kv : g_sessions) if (kv.second.last_use < ot) { ot = kv.second.last_use; oid = kv.first; }
|
| 297 |
-
free_session(g_sessions[oid]); g_sessions.erase(oid);
|
| 298 |
-
}
|
| 299 |
-
static Session& get_or_create(const std::string& id) {
|
| 300 |
-
auto it = g_sessions.find(id);
|
| 301 |
-
if (it != g_sessions.end()) { it->second.last_use = get_ms(); return it->second; }
|
| 302 |
-
if ((int)g_sessions.size() >= MAX_SESSIONS) evict_oldest();
|
| 303 |
-
Session s;
|
| 304 |
-
long long nb = kv_bytes();
|
| 305 |
-
s.k_cache = (float*)calloc(nb, 1);
|
| 306 |
-
s.v_cache = (float*)calloc(nb, 1);
|
| 307 |
-
s.pos = 0; s.last_use = get_ms();
|
| 308 |
-
g_sessions[id] = s;
|
| 309 |
-
return g_sessions[id];
|
| 310 |
-
}
|
| 311 |
-
|
| 312 |
-
static int sample_topk(float temperature, int top_k) {
|
| 313 |
-
for (int v = 0; v < cfg.vocab_size; v++) g_logits[v] /= temperature;
|
| 314 |
-
int K = std::min(top_k, cfg.vocab_size);
|
| 315 |
-
for (int v = 0; v < cfg.vocab_size; v++) g_topk_pairs[v] = {g_logits[v], v};
|
| 316 |
-
std::partial_sort(g_topk_pairs, g_topk_pairs + K, g_topk_pairs + cfg.vocab_size,
|
| 317 |
-
[](const auto& a, const auto& b){ return a.first > b.first; });
|
| 318 |
-
float sum = 0.f;
|
| 319 |
-
for (int j = 0; j < K; j++) {
|
| 320 |
-
g_topk_pairs[j].first = expf(g_topk_pairs[j].first); sum += g_topk_pairs[j].first;
|
| 321 |
-
}
|
| 322 |
-
for (int j = 0; j < K; j++) g_topk_pairs[j].first /= sum;
|
| 323 |
-
float r = rng_float(), cum = 0.f;
|
| 324 |
-
int best = g_topk_pairs[0].second;
|
| 325 |
-
for (int j = 0; j < K; j++) {
|
| 326 |
-
cum += g_topk_pairs[j].first;
|
| 327 |
-
if (r < cum) { best = g_topk_pairs[j].second; break; }
|
| 328 |
-
}
|
| 329 |
-
return best;
|
| 330 |
}
|
| 331 |
|
| 332 |
static int sample_topk_batch(float* logits, float temperature, int top_k) {
|
| 333 |
-
for
|
| 334 |
int K = std::min(top_k, cfg.vocab_size);
|
| 335 |
-
for
|
| 336 |
-
std::partial_sort(g_topk_pairs, g_topk_pairs
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
}
|
| 342 |
-
for (int j = 0; j < K; j++) g_topk_pairs[j].first /= sum;
|
| 343 |
-
float r = rng_float(), cum = 0.f;
|
| 344 |
-
int best = g_topk_pairs[0].second;
|
| 345 |
-
for (int j = 0; j < K; j++) {
|
| 346 |
-
cum += g_topk_pairs[j].first;
|
| 347 |
-
if (r < cum) { best = g_topk_pairs[j].second; break; }
|
| 348 |
-
}
|
| 349 |
return best;
|
| 350 |
}
|
| 351 |
|
| 352 |
-
static
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 361 |
}
|
| 362 |
|
| 363 |
-
static
|
| 364 |
-
|
| 365 |
-
if (parts.size() < 7) { printf("ERROR bad_request_format\n"); fflush(stdout); return; }
|
| 366 |
-
std::string sess_id = parts[1];
|
| 367 |
-
auto new_tokens = parse_ints(parts[2]);
|
| 368 |
-
int max_new = atoi(parts[3].c_str());
|
| 369 |
-
float temp = (float)atof(parts[4].c_str());
|
| 370 |
-
int top_k = atoi(parts[5].c_str());
|
| 371 |
-
auto stop_list = parse_ints(parts[6]);
|
| 372 |
-
|
| 373 |
-
temp = std::max(temp, 0.01f);
|
| 374 |
-
top_k = std::clamp(top_k, 1, cfg.vocab_size);
|
| 375 |
-
max_new = std::max(max_new, 1);
|
| 376 |
-
|
| 377 |
-
std::unordered_set<int> stop_ids(stop_list.begin(), stop_list.end());
|
| 378 |
-
stop_ids.insert(50256);
|
| 379 |
-
|
| 380 |
-
Session& sess = get_or_create(sess_id);
|
| 381 |
-
|
| 382 |
-
for (int tok : new_tokens) {
|
| 383 |
-
if (sess.pos >= cfg.block_size) { printf("ERROR context_window_full\n"); fflush(stdout); return; }
|
| 384 |
-
forward(tok, sess.pos, sess.k_cache, sess.v_cache);
|
| 385 |
-
sess.pos++;
|
| 386 |
-
}
|
| 387 |
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
gen++;
|
| 396 |
-
if (stop_ids.count(next)) break;
|
| 397 |
-
forward(next, sess.pos, sess.k_cache, sess.v_cache);
|
| 398 |
-
sess.pos++;
|
| 399 |
-
}
|
| 400 |
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
|
|
|
| 404 |
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
|
|
|
|
|
|
| 416 |
}
|
| 417 |
-
printf("PREFILL_OK\n"); fflush(stdout);
|
| 418 |
}
|
| 419 |
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
|
|
|
|
|
|
| 423 |
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
|
|
|
| 429 |
}
|
|
|
|
| 430 |
|
| 431 |
-
|
| 432 |
-
|
| 433 |
|
| 434 |
-
|
| 435 |
-
std::vector<int> tokens(B);
|
| 436 |
-
std::vector<int> positions(B);
|
| 437 |
-
std::vector<Session*> sessions(B);
|
| 438 |
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 446 |
|
| 447 |
-
|
| 448 |
-
|
| 449 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 450 |
|
| 451 |
-
|
| 452 |
-
float* te = W.wte + (long long)tokens[b] * C;
|
| 453 |
-
float* pe = W.wpe + (long long)positions[b] * C;
|
| 454 |
-
for (int i = 0; i < C; i++) g_x_b[b][i] = te[i] + pe[i];
|
| 455 |
-
}
|
| 456 |
|
| 457 |
-
|
| 458 |
-
|
| 459 |
|
| 460 |
-
//
|
| 461 |
for (int b = 0; b < B; b++) {
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
|
|
|
|
|
|
|
|
|
|
| 466 |
}
|
| 467 |
-
}
|
| 468 |
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 474 |
}
|
| 475 |
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
for (int
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 491 |
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 497 |
}
|
| 498 |
}
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
for (int b = 0; b < B; b++) {
|
| 503 |
-
#pragma omp parallel for schedule(static)
|
| 504 |
-
for (int i = 0; i < C; i++) {
|
| 505 |
-
const float* row = W.c_proj_w[l] + (long long)i * C;
|
| 506 |
-
g_tmp_out_b[b][i] = dot_avx2(row, g_buf_b[b], C) + W.c_proj_b[l][i];
|
| 507 |
}
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
// Cache optimized FC
|
| 514 |
-
for (int b = 0; b < B; b++) {
|
| 515 |
-
#pragma omp parallel for schedule(static)
|
| 516 |
-
for (int i = 0; i < 4 * C; i++) {
|
| 517 |
-
const float* row = W.fc_w[l] + (long long)i * C;
|
| 518 |
-
g_ff_b[b][i] = dot_avx2(row, g_buf_b[b], C) + W.fc_b[l][i];
|
| 519 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 520 |
}
|
| 521 |
-
for (int b = 0; b < B; b++)
|
| 522 |
|
| 523 |
-
//
|
|
|
|
| 524 |
for (int b = 0; b < B; b++) {
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 529 |
}
|
|
|
|
| 530 |
}
|
| 531 |
-
|
| 532 |
-
}
|
| 533 |
-
|
| 534 |
-
for (int b = 0; b < B; b++) layer_norm(g_buf_b[b], g_x_b[b], W.ln_f_w, W.ln_f_b, C);
|
| 535 |
-
|
| 536 |
-
std::string response = "TOKENS";
|
| 537 |
-
for (int b = 0; b < B; b++) {
|
| 538 |
-
#pragma omp parallel for schedule(static)
|
| 539 |
-
for (int i = 0; i < cfg.vocab_size; i++) {
|
| 540 |
-
const float* row = W.lm_head_w + (long long)i * C;
|
| 541 |
-
g_logits_b[b][i] = dot_avx2(row, g_buf_b[b], C);
|
| 542 |
-
}
|
| 543 |
-
int next_tok = sample_topk_batch(g_logits_b[b], temp, top_k);
|
| 544 |
-
response += " " + session_ids[b] + ":" + std::to_string(next_tok);
|
| 545 |
-
}
|
| 546 |
-
printf("%s\n", response.c_str());
|
| 547 |
-
fflush(stdout);
|
| 548 |
-
}
|
| 549 |
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
auto it = g_sessions.find(parts[1]);
|
| 554 |
-
if (it != g_sessions.end()) {
|
| 555 |
-
free_session(it->second); g_sessions.erase(it);
|
| 556 |
}
|
| 557 |
}
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
int main() {
|
| 562 |
-
FILE* f = fopen("model.bin", "rb");
|
| 563 |
-
if (!f) { printf("ERROR model.bin_not_found\n"); fflush(stdout); return 1; }
|
| 564 |
-
|
| 565 |
-
fread(&cfg, sizeof(int), 5, f);
|
| 566 |
-
fseek(f, 0, SEEK_END); long fsize = ftell(f);
|
| 567 |
-
fseek(f, 5*(long)sizeof(int), SEEK_SET);
|
| 568 |
-
long wbytes = fsize - 5*(long)sizeof(int);
|
| 569 |
-
|
| 570 |
-
g_data = (float*)malloc(wbytes);
|
| 571 |
-
if (!g_data) { printf("ERROR oom\n"); fflush(stdout); return 1; }
|
| 572 |
-
fread(g_data, 1, wbytes, f);
|
| 573 |
-
fclose(f);
|
| 574 |
-
|
| 575 |
-
map_weights(g_data, wbytes);
|
| 576 |
-
|
| 577 |
-
const int C = cfg.n_embd;
|
| 578 |
-
g_x = (float*)malloc(C * sizeof(float));
|
| 579 |
-
g_buf = (float*)malloc(C * sizeof(float));
|
| 580 |
-
g_qkv = (float*)malloc(3*C * sizeof(float));
|
| 581 |
-
g_attn_buf = (float*)malloc((long long)cfg.n_head * cfg.block_size * sizeof(float));
|
| 582 |
-
g_ff = (float*)malloc(4*C * sizeof(float));
|
| 583 |
-
g_logits = (float*)malloc((long long)cfg.vocab_size * sizeof(float));
|
| 584 |
-
g_tmp_out = (float*)malloc(C * sizeof(float));
|
| 585 |
-
g_topk_pairs = (std::pair<float,int>*)malloc((long long)cfg.vocab_size * sizeof(std::pair<float,int>));
|
| 586 |
-
|
| 587 |
-
for (int b = 0; b < MAX_BATCH; b++) {
|
| 588 |
-
g_x_b[b] = (float*)malloc(C * sizeof(float));
|
| 589 |
-
g_buf_b[b] = (float*)malloc(C * sizeof(float));
|
| 590 |
-
g_qkv_b[b] = (float*)malloc(3*C * sizeof(float));
|
| 591 |
-
g_attn_buf_b[b] = (float*)malloc((long long)cfg.n_head * cfg.block_size * sizeof(float));
|
| 592 |
-
g_ff_b[b] = (float*)malloc(4*C * sizeof(float));
|
| 593 |
-
g_tmp_out_b[b] = (float*)malloc(C * sizeof(float));
|
| 594 |
-
g_logits_b[b] = (float*)malloc((long long)cfg.vocab_size * sizeof(float));
|
| 595 |
-
}
|
| 596 |
-
|
| 597 |
-
printf("READY\n"); fflush(stdout);
|
| 598 |
-
|
| 599 |
-
std::string line;
|
| 600 |
-
while (std::getline(std::cin, line)) {
|
| 601 |
-
if (!line.empty() && line.back()=='\r') line.pop_back();
|
| 602 |
-
if (line.empty()) continue;
|
| 603 |
-
if (line == "QUIT") break;
|
| 604 |
-
else if (line.rfind("RESET|", 0)==0) handle_reset(line);
|
| 605 |
-
else if (line.rfind("REQUEST|", 0)==0) handle_request(line);
|
| 606 |
-
else if (line.rfind("PREFILL|", 0)==0) handle_prefill(line);
|
| 607 |
-
else if (line.rfind("BATCH_STEP|",0)==0) handle_batch_step(line);
|
| 608 |
-
else { printf("ERROR unknown_cmd\n"); fflush(stdout); }
|
| 609 |
-
}
|
| 610 |
-
|
| 611 |
-
for (auto& kv : g_sessions) free_session(kv.second);
|
| 612 |
-
free(g_data);
|
| 613 |
-
free(g_x); free(g_buf); free(g_qkv); free(g_attn_buf);
|
| 614 |
-
free(g_ff); free(g_logits); free(g_tmp_out); free(g_topk_pairs);
|
| 615 |
-
for (int b = 0; b < MAX_BATCH; b++) {
|
| 616 |
-
free(g_x_b[b]); free(g_buf_b[b]); free(g_qkv_b[b]);
|
| 617 |
-
free(g_attn_buf_b[b]); free(g_ff_b[b]); free(g_tmp_out_b[b]); free(g_logits_b[b]);
|
| 618 |
-
}
|
| 619 |
return 0;
|
| 620 |
}
|
|
|
|
| 1 |
/*
|
| 2 |
* ============================================================
|
| 3 |
+
* NanoMind — v7.0 (vLLM-Style Paged KV & Continuous Batching)
|
| 4 |
* ============================================================
|
| 5 |
+
* Features:
|
| 6 |
+
* - Paged KV Cache (Global Memory Pool, No Fragmentation)
|
| 7 |
+
* - Block Allocator (O(1) allocation/free)
|
| 8 |
+
* - Unified Forward Pass (Prefill and Decode run in parallel)
|
| 9 |
+
* - Continuous Slot Refill (Zero batch starvation)
|
| 10 |
+
* - Thread-safe IO & Async Orchestration
|
| 11 |
*/
|
| 12 |
#include <iostream>
|
| 13 |
#include <cstdio>
|
|
|
|
| 18 |
#include <algorithm>
|
| 19 |
#include <string>
|
| 20 |
#include <vector>
|
|
|
|
| 21 |
#include <unordered_set>
|
| 22 |
+
#include <queue>
|
| 23 |
+
#include <thread>
|
| 24 |
+
#include <mutex>
|
| 25 |
+
#include <atomic>
|
| 26 |
+
#include <immintrin.h>
|
| 27 |
+
|
| 28 |
#ifdef _OPENMP
|
| 29 |
#include <omp.h>
|
| 30 |
#endif
|
| 31 |
#ifdef _WIN32
|
| 32 |
#include <windows.h>
|
| 33 |
+
static double get_ms() { LARGE_INTEGER f,c; QueryPerformanceFrequency(&f); QueryPerformanceCounter(&c); return (double)c.QuadPart/f.QuadPart*1000.0; }
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
#else
|
| 35 |
#include <sys/time.h>
|
| 36 |
+
static double get_ms() { struct timeval tv; gettimeofday(&tv, NULL); return tv.tv_sec*1000.0 + tv.tv_usec/1000.0; }
|
|
|
|
|
|
|
|
|
|
| 37 |
#endif
|
| 38 |
|
| 39 |
+
// ── Globals & RNG ──
|
| 40 |
static thread_local uint64_t tl_rng_state = 0;
|
| 41 |
static inline float rng_float() {
|
| 42 |
+
if (!tl_rng_state) { tl_rng_state = (uint64_t)get_ms() ^ (uint64_t)(uintptr_t)&tl_rng_state; if (!tl_rng_state) tl_rng_state = 0xDEADBEEFCAFEBABEULL; }
|
| 43 |
+
tl_rng_state ^= tl_rng_state << 13; tl_rng_state ^= tl_rng_state >> 7; tl_rng_state ^= tl_rng_state << 17;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
return (float)(tl_rng_state >> 11) * (1.0f / (float)(1ULL << 53));
|
| 45 |
}
|
| 46 |
|
| 47 |
+
struct Config { int n_layer, n_head, n_embd, block_size, vocab_size; };
|
|
|
|
|
|
|
| 48 |
struct Weights {
|
| 49 |
+
float *wte, *wpe, **ln1_w, **ln1_b, **c_attn_w, **c_attn_b;
|
| 50 |
+
float **c_proj_w, **c_proj_b, **ln2_w, **ln2_b, **fc_w, **fc_b;
|
| 51 |
+
float **mlp_proj_w, **mlp_proj_b, *ln_f_w, *ln_f_b, *lm_head_w;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
};
|
| 53 |
+
static Config cfg;
|
| 54 |
static Weights W;
|
| 55 |
static float* g_data = nullptr;
|
| 56 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
const int MAX_BATCH = 16;
|
| 58 |
static float *g_x_b[MAX_BATCH], *g_buf_b[MAX_BATCH], *g_qkv_b[MAX_BATCH];
|
| 59 |
static float *g_attn_buf_b[MAX_BATCH], *g_ff_b[MAX_BATCH], *g_tmp_out_b[MAX_BATCH], *g_logits_b[MAX_BATCH];
|
| 60 |
+
static std::pair<float,int>* g_topk_pairs = nullptr;
|
| 61 |
|
| 62 |
+
// ── Paged KV Cache (The Core Breakthrough) ──
|
| 63 |
+
const int PAGE_SIZE = 16;
|
| 64 |
+
const int MAX_PAGES = 4096; // ~6.4 GB total memory footprint for KV Pool
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
|
| 66 |
+
static float* g_k_pool = nullptr;
|
| 67 |
+
static float* g_v_pool = nullptr;
|
| 68 |
+
static std::queue<int> g_free_pages;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
|
| 70 |
+
void init_kv_pool() {
|
| 71 |
+
long long page_bytes = (long long)cfg.n_layer * PAGE_SIZE * cfg.n_embd * sizeof(float);
|
| 72 |
+
g_k_pool = (float*)malloc(MAX_PAGES * page_bytes);
|
| 73 |
+
g_v_pool = (float*)malloc(MAX_PAGES * page_bytes);
|
| 74 |
+
for(int i = 0; i < MAX_PAGES; i++) g_free_pages.push(i);
|
|
|
|
|
|
|
|
|
|
| 75 |
}
|
| 76 |
|
| 77 |
+
int alloc_page() {
|
| 78 |
+
if (g_free_pages.empty()) return -1; // OOM
|
| 79 |
+
int p = g_free_pages.front(); g_free_pages.pop();
|
| 80 |
+
return p;
|
|
|
|
|
|
|
| 81 |
}
|
| 82 |
|
| 83 |
+
void free_pages(std::vector<int>& pages) {
|
| 84 |
+
for (int p : pages) g_free_pages.push(p);
|
| 85 |
+
pages.clear();
|
| 86 |
}
|
| 87 |
|
| 88 |
+
// ── AVX Math Kernels ──
|
| 89 |
+
static void layer_norm(float* out, const float* x, const float* w, const float* b, int N) {
|
| 90 |
+
__m256 vsum = _mm256_setzero_ps();
|
| 91 |
+
for (int i=0; i<N; i+=8) vsum = _mm256_add_ps(vsum, _mm256_loadu_ps(x+i));
|
| 92 |
+
float tmp[8]; _mm256_storeu_ps(tmp, vsum); float mean = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
|
| 93 |
+
__m256 vmean = _mm256_set1_ps(mean), vvar = _mm256_setzero_ps();
|
| 94 |
+
for (int i=0; i<N; i+=8) { __m256 d = _mm256_sub_ps(_mm256_loadu_ps(x+i), vmean); vvar = _mm256_fmadd_ps(d, d, vvar); }
|
| 95 |
+
_mm256_storeu_ps(tmp, vvar); float var = (tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7]) / (float)N;
|
| 96 |
+
__m256 vsc = _mm256_set1_ps(1.f / sqrtf(var + 1e-5f));
|
| 97 |
+
for (int i=0; i<N; i+=8) _mm256_storeu_ps(out+i, _mm256_fmadd_ps(_mm256_mul_ps(_mm256_sub_ps(_mm256_loadu_ps(x+i), vmean), vsc), _mm256_loadu_ps(w+i), _mm256_loadu_ps(b+i)));
|
| 98 |
}
|
| 99 |
+
static void matmul_vec(float* out, const float* mat, const float* x, int M, int K) {
|
| 100 |
+
#pragma omp parallel for schedule(static) if(M>=64)
|
| 101 |
+
for (int i=0; i<M; i++) {
|
| 102 |
+
const float* row = mat + (long long)i*K; __m256 acc = _mm256_setzero_ps(); int j=0;
|
| 103 |
+
for (; j<=K-8; j+=8) acc = _mm256_fmadd_ps(_mm256_loadu_ps(row+j), _mm256_loadu_ps(x+j), acc);
|
| 104 |
+
float t[8]; _mm256_storeu_ps(t, acc); float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
|
| 105 |
+
for (; j<K; j++) s += row[j]*x[j]; out[i] = s;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
}
|
|
|
|
| 107 |
}
|
| 108 |
+
static inline float dot_avx2(const float* a, const float* b, int n) {
|
| 109 |
+
__m256 acc = _mm256_setzero_ps(); int i=0;
|
| 110 |
+
for (; i<=n-8; i+=8) acc = _mm256_fmadd_ps(_mm256_loadu_ps(a+i), _mm256_loadu_ps(b+i), acc);
|
| 111 |
+
float t[8]; _mm256_storeu_ps(t, acc); float s = t[0]+t[1]+t[2]+t[3]+t[4]+t[5]+t[6]+t[7];
|
| 112 |
+
for (; i<n; i++) s += a[i]*b[i]; return s;
|
|
|
|
|
|
|
| 113 |
}
|
| 114 |
+
static inline void weighted_acc_avx2(float* out, const float* v, float w, int n) {
|
| 115 |
+
__m256 wv = _mm256_set1_ps(w); int i=0;
|
| 116 |
+
for (; i<=n-8; i+=8) _mm256_storeu_ps(out+i, _mm256_fmadd_ps(wv, _mm256_loadu_ps(v+i), _mm256_loadu_ps(out+i)));
|
| 117 |
+
for (; i<n; i++) out[i] += w * v[i];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 118 |
}
|
| 119 |
+
static inline void residual_add(float* x, const float* y, int N) { #pragma omp parallel for
|
| 120 |
+
for (int i=0; i<N; i++) x[i] += y[i]; }
|
| 121 |
|
| 122 |
+
static void gelu_inplace(float* x, int N) {
|
| 123 |
+
const __m256 scale=_mm256_set1_ps(12102203.0f), vbias=_mm256_set1_ps(1064807168.0f), neg_c=_mm256_set1_ps(-1.702f), vone=_mm256_set1_ps(1.0f), vtwo=_mm256_set1_ps(2.0f), vlo=_mm256_set1_ps(-88.0f), vhi=_mm256_set1_ps(88.0f);
|
| 124 |
+
int i=0;
|
| 125 |
+
for (; i<=N-8; i+=8) {
|
| 126 |
+
__m256 v = _mm256_loadu_ps(x+i), t = _mm256_min_ps(_mm256_max_ps(_mm256_mul_ps(neg_c, v), vlo), vhi);
|
| 127 |
+
__m256i ti = _mm256_cvttps_epi32(_mm256_fmadd_ps(t, scale, vbias)); __m256 et = _mm256_castsi256_ps(ti);
|
| 128 |
+
__m256 denom = _mm256_add_ps(vone, et), r = _mm256_rcp_ps(denom);
|
| 129 |
+
_mm256_storeu_ps(x+i, _mm256_mul_ps(v, _mm256_mul_ps(r, _mm256_fnmadd_ps(denom, r, vtwo))));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
}
|
| 131 |
+
for (; i<N; i++) x[i] = x[i] / (1.0f + expf(-1.702f * x[i]));
|
| 132 |
}
|
| 133 |
+
static void softmax_inplace(float* x, int N) {
|
| 134 |
+
float mx = x[0]; for (int i=1; i<N; i++) if(x[i]>mx) mx=x[i];
|
| 135 |
+
float s = 0.f; for (int i=0; i<N; i++) { x[i]=expf(x[i]-mx); s+=x[i]; }
|
| 136 |
+
for (int i=0; i<N; i++) x[i]/=s;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
}
|
| 138 |
|
| 139 |
static int sample_topk_batch(float* logits, float temperature, int top_k) {
|
| 140 |
+
for(int v=0; v<cfg.vocab_size; v++) logits[v] /= temperature;
|
| 141 |
int K = std::min(top_k, cfg.vocab_size);
|
| 142 |
+
for(int v=0; v<cfg.vocab_size; v++) g_topk_pairs[v] = {logits[v], v};
|
| 143 |
+
std::partial_sort(g_topk_pairs, g_topk_pairs+K, g_topk_pairs+cfg.vocab_size, [](const auto& a, const auto& b){ return a.first > b.first; });
|
| 144 |
+
float sum=0.f; for(int j=0; j<K; j++) { g_topk_pairs[j].first=expf(g_topk_pairs[j].first); sum+=g_topk_pairs[j].first; }
|
| 145 |
+
for(int j=0; j<K; j++) g_topk_pairs[j].first/=sum;
|
| 146 |
+
float r = rng_float(), cum = 0.f; int best = g_topk_pairs[0].second;
|
| 147 |
+
for(int j=0; j<K; j++) { cum+=g_topk_pairs[j].first; if(r<cum) { best=g_topk_pairs[j].second; break; } }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
return best;
|
| 149 |
}
|
| 150 |
|
| 151 |
+
static void map_weights(float* data, long wbytes) {
|
| 152 |
+
float* p=data; const int C=cfg.n_embd, L=cfg.n_layer;
|
| 153 |
+
W.wte=p; p+=cfg.vocab_size*C; W.wpe=p; p+=cfg.block_size*C;
|
| 154 |
+
#define ARR(f) W.f=(float**)malloc(L*sizeof(float*))
|
| 155 |
+
ARR(ln1_w); ARR(ln1_b); ARR(c_attn_w); ARR(c_attn_b); ARR(c_proj_w); ARR(c_proj_b); ARR(ln2_w); ARR(ln2_b); ARR(fc_w); ARR(fc_b); ARR(mlp_proj_w); ARR(mlp_proj_b);
|
| 156 |
+
#undef ARR
|
| 157 |
+
for(int l=0; l<L; l++) {
|
| 158 |
+
W.ln1_w[l]=p; p+=C; W.ln1_b[l]=p; p+=C; W.c_attn_w[l]=p; p+=3LL*C*C; W.c_attn_b[l]=p; p+=3LL*C;
|
| 159 |
+
W.c_proj_w[l]=p; p+=1LL*C*C; W.c_proj_b[l]=p; p+=C; W.ln2_w[l]=p; p+=C; W.ln2_b[l]=p; p+=C;
|
| 160 |
+
W.fc_w[l]=p; p+=4LL*C*C; W.fc_b[l]=p; p+=4LL*C; W.mlp_proj_w[l]=p; p+=1LL*C*4*C; W.mlp_proj_b[l]=p; p+=C;
|
| 161 |
+
}
|
| 162 |
+
W.ln_f_w=p; p+=C; W.ln_f_b=p; p+=C; W.lm_head_w=p; p+=cfg.vocab_size*C;
|
| 163 |
+
if(p > data + wbytes/sizeof(float)) { printf("ERROR|SYSTEM|model_truncated\n"); exit(1); }
|
| 164 |
}
|
| 165 |
|
| 166 |
+
static std::vector<std::string> split(const std::string& s, char d) { std::vector<std::string> out; std::string cur; for(char c:s){if(c==d){out.push_back(cur);cur.clear();}else cur+=c;} out.push_back(cur); return out; }
|
| 167 |
+
static std::vector<int> parse_ints(const std::string& s) { std::vector<int> out; for(auto& t:split(s,',')) if(!t.empty()) out.push_back(atoi(t.c_str())); return out; }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
|
| 169 |
+
// ── Multi-Threading & Scheduler State ──
|
| 170 |
+
struct Request { std::string id; std::vector<int> tokens; int max_new; float temp; int top_k; };
|
| 171 |
+
struct ActiveSeq {
|
| 172 |
+
std::string id; std::vector<int> prompt; int max_new; float temp; int top_k;
|
| 173 |
+
int generated; double t0; int pos; int prefill_cursor;
|
| 174 |
+
std::vector<int> page_table; bool is_prefilling;
|
| 175 |
+
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
|
| 177 |
+
std::mutex g_mtx;
|
| 178 |
+
std::vector<Request> g_pending;
|
| 179 |
+
std::unordered_set<std::string> g_cancels;
|
| 180 |
+
std::atomic<bool> g_running{true};
|
| 181 |
|
| 182 |
+
void io_thread_func() {
|
| 183 |
+
std::string line;
|
| 184 |
+
while (std::getline(std::cin, line)) {
|
| 185 |
+
if (line.empty()) continue;
|
| 186 |
+
if (line == "QUIT") { g_running = false; break; }
|
| 187 |
+
auto parts = split(line, '|');
|
| 188 |
+
if (parts[0] == "ENQUEUE" && parts.size() >= 6) {
|
| 189 |
+
Request req; req.id = parts[1]; req.tokens = parse_ints(parts[2]); req.max_new = atoi(parts[3].c_str());
|
| 190 |
+
req.temp = std::max(0.01f, (float)atof(parts[4].c_str())); req.top_k = std::clamp(atoi(parts[5].c_str()), 1, cfg.vocab_size);
|
| 191 |
+
std::lock_guard<std::mutex> lock(g_mtx); g_pending.push_back(req);
|
| 192 |
+
} else if (parts[0] == "CANCEL" && parts.size() >= 2) {
|
| 193 |
+
std::lock_guard<std::mutex> lock(g_mtx); g_cancels.insert(parts[1]);
|
| 194 |
+
}
|
| 195 |
}
|
|
|
|
| 196 |
}
|
| 197 |
|
| 198 |
+
int main() {
|
| 199 |
+
FILE* f = fopen("model.bin", "rb"); if(!f) return 1;
|
| 200 |
+
fread(&cfg, sizeof(int), 5, f); fseek(f, 0, SEEK_END); long fsize = ftell(f); fseek(f, 5*(long)sizeof(int), SEEK_SET);
|
| 201 |
+
long wbytes = fsize - 5*(long)sizeof(int); g_data = (float*)malloc(wbytes); fread(g_data, 1, wbytes, f); fclose(f);
|
| 202 |
+
map_weights(g_data, wbytes);
|
| 203 |
|
| 204 |
+
const int C = cfg.n_embd;
|
| 205 |
+
g_topk_pairs = (std::pair<float,int>*)malloc(cfg.vocab_size*sizeof(std::pair<float,int>));
|
| 206 |
+
for (int b=0; b<MAX_BATCH; b++) {
|
| 207 |
+
g_x_b[b] = (float*)malloc(C*sizeof(float)); g_buf_b[b] = (float*)malloc(C*sizeof(float)); g_qkv_b[b] = (float*)malloc(3*C*sizeof(float));
|
| 208 |
+
g_attn_buf_b[b] = (float*)malloc(cfg.n_head*cfg.block_size*sizeof(float)); g_ff_b[b] = (float*)malloc(4*C*sizeof(float));
|
| 209 |
+
g_tmp_out_b[b] = (float*)malloc(C*sizeof(float)); g_logits_b[b] = (float*)malloc(cfg.vocab_size*sizeof(float));
|
| 210 |
}
|
| 211 |
+
init_kv_pool();
|
| 212 |
|
| 213 |
+
std::thread io_thread(io_thread_func);
|
| 214 |
+
std::vector<ActiveSeq> active;
|
| 215 |
|
| 216 |
+
printf("READY\n"); fflush(stdout);
|
|
|
|
|
|
|
|
|
|
| 217 |
|
| 218 |
+
while(g_running) {
|
| 219 |
+
// 1. Safe Teardown & Admissions
|
| 220 |
+
{
|
| 221 |
+
std::lock_guard<std::mutex> lock(g_mtx);
|
| 222 |
+
for(const auto& cid : g_cancels) {
|
| 223 |
+
for (auto it = active.begin(); it != active.end(); ) {
|
| 224 |
+
if (it->id == cid) { free_pages(it->page_table); it = active.erase(it); }
|
| 225 |
+
else ++it;
|
| 226 |
+
}
|
| 227 |
+
g_pending.erase(std::remove_if(g_pending.begin(), g_pending.end(), [&](auto& req){ return req.id == cid; }), g_pending.end());
|
| 228 |
+
}
|
| 229 |
+
g_cancels.clear();
|
| 230 |
|
| 231 |
+
// Continuous Slot Refill
|
| 232 |
+
while(active.size() < MAX_BATCH && !g_pending.empty()) {
|
| 233 |
+
auto req = g_pending.front(); g_pending.erase(g_pending.begin());
|
| 234 |
+
ActiveSeq s; s.id = req.id; s.prompt = req.tokens; s.max_new = req.max_new; s.temp = req.temp; s.top_k = req.top_k;
|
| 235 |
+
s.generated = 0; s.pos = 0; s.prefill_cursor = 0; s.t0 = get_ms(); s.is_prefilling = true;
|
| 236 |
+
|
| 237 |
+
int p = alloc_page();
|
| 238 |
+
if (p == -1) { printf("ERROR|%s|OOM\n", s.id.c_str()); fflush(stdout); continue; }
|
| 239 |
+
s.page_table.push_back(p);
|
| 240 |
+
active.push_back(s);
|
| 241 |
+
}
|
| 242 |
+
}
|
| 243 |
|
| 244 |
+
if (active.empty()) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); continue; }
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
|
| 246 |
+
int B = active.size();
|
| 247 |
+
std::vector<int> current_tokens(B);
|
| 248 |
|
| 249 |
+
// 2. Prepare Unified Batch Inputs (No Sequential Prefill Blocks)
|
| 250 |
for (int b = 0; b < B; b++) {
|
| 251 |
+
auto& seq = active[b];
|
| 252 |
+
|
| 253 |
+
// Check page boundaries before step
|
| 254 |
+
if (seq.pos > 0 && seq.pos % PAGE_SIZE == 0) {
|
| 255 |
+
int p = alloc_page();
|
| 256 |
+
if (p == -1) { printf("ERROR|%s|OOM\n", seq.id.c_str()); fflush(stdout); seq.max_new = -1; continue; }
|
| 257 |
+
seq.page_table.push_back(p);
|
| 258 |
}
|
|
|
|
| 259 |
|
| 260 |
+
int tok;
|
| 261 |
+
if (seq.is_prefilling) {
|
| 262 |
+
tok = seq.prompt[seq.prefill_cursor];
|
| 263 |
+
} else {
|
| 264 |
+
tok = seq.prompt.back(); // Latest generated token appended here later
|
| 265 |
+
}
|
| 266 |
+
current_tokens[b] = tok;
|
| 267 |
+
|
| 268 |
+
float* te = W.wte + (long long)tok * C; float* pe = W.wpe + (long long)seq.pos * C;
|
| 269 |
+
for (int i = 0; i < C; i++) g_x_b[b][i] = te[i] + pe[i];
|
| 270 |
}
|
| 271 |
|
| 272 |
+
// 3. Batched Transformer Execution with Paged Attention
|
| 273 |
+
const int H = cfg.n_head; const int hs = C / H;
|
| 274 |
+
long long page_stride = (long long)cfg.n_layer * PAGE_SIZE * C;
|
| 275 |
+
long long layer_stride = PAGE_SIZE * C;
|
| 276 |
+
long long tok_stride = C;
|
| 277 |
+
|
| 278 |
+
for (int l = 0; l < cfg.n_layer; l++) {
|
| 279 |
+
for (int b = 0; b < B; b++) layer_norm(g_buf_b[b], g_x_b[b], W.ln1_w[l], W.ln1_b[l], C);
|
| 280 |
+
for (int b = 0; b < B; b++) {
|
| 281 |
+
#pragma omp parallel for schedule(static)
|
| 282 |
+
for (int i = 0; i < 3*C; i++) g_qkv_b[b][i] = dot_avx2(W.c_attn_w[l] + (long long)i*C, g_buf_b[b], C) + W.c_attn_b[l][i];
|
| 283 |
+
}
|
| 284 |
+
|
| 285 |
+
// Write to Paged KV & Attention
|
| 286 |
+
for (int b = 0; b < B; b++) {
|
| 287 |
+
auto& seq = active[b];
|
| 288 |
+
if (seq.max_new == -1) continue; // OOMed
|
| 289 |
+
|
| 290 |
+
int phys_p = seq.page_table[seq.pos / PAGE_SIZE];
|
| 291 |
+
int offset = seq.pos % PAGE_SIZE;
|
| 292 |
+
float* kc = g_k_pool + phys_p * page_stride + l * layer_stride + offset * tok_stride;
|
| 293 |
+
float* vc = g_v_pool + phys_p * page_stride + l * layer_stride + offset * tok_stride;
|
| 294 |
+
memcpy(kc, g_qkv_b[b]+C, C*sizeof(float)); memcpy(vc, g_qkv_b[b]+2*C, C*sizeof(float));
|
| 295 |
|
| 296 |
+
#pragma omp parallel for schedule(static)
|
| 297 |
+
for (int h = 0; h < H; h++) {
|
| 298 |
+
float scale = 1.f / sqrtf((float)hs), *attn = g_attn_buf_b[b] + h*cfg.block_size;
|
| 299 |
+
for (int t = 0; t <= seq.pos; t++) {
|
| 300 |
+
int read_p = seq.page_table[t / PAGE_SIZE];
|
| 301 |
+
int read_off = t % PAGE_SIZE;
|
| 302 |
+
float* kh = g_k_pool + read_p * page_stride + l * layer_stride + read_off * tok_stride + h*hs;
|
| 303 |
+
attn[t] = dot_avx2(g_qkv_b[b]+h*hs, kh, hs) * scale;
|
| 304 |
+
}
|
| 305 |
+
softmax_inplace(attn, seq.pos+1);
|
| 306 |
+
memset(g_buf_b[b]+h*hs, 0, hs*sizeof(float));
|
| 307 |
+
for (int t = 0; t <= seq.pos; t++) {
|
| 308 |
+
int read_p = seq.page_table[t / PAGE_SIZE];
|
| 309 |
+
int read_off = t % PAGE_SIZE;
|
| 310 |
+
float* vh = g_v_pool + read_p * page_stride + l * layer_stride + read_off * tok_stride + h*hs;
|
| 311 |
+
weighted_acc_avx2(g_buf_b[b]+h*hs, vh, attn[t], hs);
|
| 312 |
+
}
|
| 313 |
}
|
| 314 |
}
|
| 315 |
+
for (int b = 0; b < B; b++) {
|
| 316 |
+
#pragma omp parallel for schedule(static)
|
| 317 |
+
for (int i = 0; i < C; i++) g_tmp_out_b[b][i] = dot_avx2(W.c_proj_w[l] + (long long)i*C, g_buf_b[b], C) + W.c_proj_b[l][i];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 318 |
}
|
| 319 |
+
for (int b = 0; b < B; b++) residual_add(g_x_b[b], g_tmp_out_b[b], C);
|
| 320 |
+
for (int b = 0; b < B; b++) layer_norm(g_buf_b[b], g_x_b[b], W.ln2_w[l], W.ln2_b[l], C);
|
| 321 |
+
for (int b = 0; b < B; b++) {
|
| 322 |
+
#pragma omp parallel for schedule(static)
|
| 323 |
+
for (int i = 0; i < 4*C; i++) g_ff_b[b][i] = dot_avx2(W.fc_w[l] + (long long)i*C, g_buf_b[b], C) + W.fc_b[l][i];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 324 |
}
|
| 325 |
+
for (int b = 0; b < B; b++) gelu_inplace(g_ff_b[b], 4*C);
|
| 326 |
+
for (int b = 0; b < B; b++) {
|
| 327 |
+
#pragma omp parallel for schedule(static)
|
| 328 |
+
for (int i = 0; i < C; i++) g_tmp_out_b[b][i] = dot_avx2(W.mlp_proj_w[l] + (long long)i*4*C, g_ff_b[b], 4*C) + W.mlp_proj_b[l][i];
|
| 329 |
+
}
|
| 330 |
+
for (int b = 0; b < B; b++) residual_add(g_x_b[b], g_tmp_out_b[b], C);
|
| 331 |
}
|
| 332 |
+
for (int b = 0; b < B; b++) layer_norm(g_buf_b[b], g_x_b[b], W.ln_f_w, W.ln_f_b, C);
|
| 333 |
|
| 334 |
+
// 4. Update State, Output & Teardown
|
| 335 |
+
std::vector<int> to_remove;
|
| 336 |
for (int b = 0; b < B; b++) {
|
| 337 |
+
auto& seq = active[b];
|
| 338 |
+
if (seq.max_new == -1) { to_remove.push_back(b); continue; }
|
| 339 |
+
|
| 340 |
+
if (seq.is_prefilling) {
|
| 341 |
+
seq.prefill_cursor++;
|
| 342 |
+
if (seq.prefill_cursor >= seq.prompt.size()) seq.is_prefilling = false;
|
| 343 |
+
} else {
|
| 344 |
+
#pragma omp parallel for schedule(static)
|
| 345 |
+
for (int i = 0; i < cfg.vocab_size; i++) g_logits_b[b][i] = dot_avx2(W.lm_head_w + (long long)i*C, g_buf_b[b], C);
|
| 346 |
+
int tok = sample_topk_batch(g_logits_b[b], seq.temp, seq.top_k);
|
| 347 |
+
seq.prompt.push_back(tok);
|
| 348 |
+
seq.generated++;
|
| 349 |
+
printf("TOKEN|%s|%d\n", seq.id.c_str(), tok);
|
| 350 |
+
|
| 351 |
+
if (tok == 50256 || seq.generated >= seq.max_new || seq.pos >= cfg.block_size - 2) {
|
| 352 |
+
printf("DONE|%s|%d|%.2f\n", seq.id.c_str(), seq.generated, get_ms() - seq.t0);
|
| 353 |
+
to_remove.push_back(b);
|
| 354 |
+
}
|
| 355 |
}
|
| 356 |
+
seq.pos++;
|
| 357 |
}
|
| 358 |
+
fflush(stdout);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
|
| 360 |
+
for (int i = to_remove.size() - 1; i >= 0; i--) {
|
| 361 |
+
free_pages(active[to_remove[i]].page_table);
|
| 362 |
+
active.erase(active.begin() + to_remove[i]);
|
|
|
|
|
|
|
|
|
|
| 363 |
}
|
| 364 |
}
|
| 365 |
+
|
| 366 |
+
g_running = false; io_thread.join();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 367 |
return 0;
|
| 368 |
}
|