Spaces:

FINAL-Bench
/

POCKET-26B-CPU

Running on CPU Upgrade

File size: 17,312 Bytes

<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>POCKET-26B · CPU chat & A/B vs Bonsai (Gemma4-based)</title>
<style>
  :root{ --gold:#f0992a; --teal:#0f9d6e; --ink:#12203a; --muted:#5b6b8f;
         --line:#e7ecf5; --bg:#f7f9fc; --card:#ffffff; --bonsai:#7c8aa5; }
  *{ box-sizing:border-box; }
  body{ margin:0; font-family:'Segoe UI',Helvetica,Arial,sans-serif; color:var(--ink);
        background:var(--bg); line-height:1.55; }
  .wrap{ max-width:960px; margin:0 auto; padding:24px 18px 60px; }
  header{ text-align:center; padding:22px 0 8px; }
  .brand{ font-size:40px; font-weight:800; letter-spacing:1px;
          background:linear-gradient(90deg,#2ec4a0,#0f9d6e); -webkit-background-clip:text;
          background-clip:text; color:transparent; }
  .tag{ font-size:18px; font-weight:600; margin:6px 0 2px; }
  .sub{ color:var(--muted); font-size:14px; }
  .chips{ display:flex; gap:8px; justify-content:center; flex-wrap:wrap; margin:14px 0 6px; }
  .chip{ background:#fff; border:1px solid var(--line); border-radius:20px; padding:6px 14px;
         font-size:13px; font-weight:600; color:var(--ink); }
  .chip b{ color:var(--teal); }
  .chip.g{ background:#eef7f3; border-color:#cfe9df; }
  .chip.gpu{ background:#fef3f2; border-color:#fdd; color:#b42318; }
  .tabs{ display:flex; gap:8px; justify-content:center; margin:16px 0 4px; }
  .tab{ background:#fff; border:1px solid var(--line); border-radius:12px; padding:9px 18px;
        font-size:14.5px; font-weight:700; color:var(--muted); cursor:pointer; }
  .tab.active{ color:#08321f; background:linear-gradient(90deg,#d6f3e8,#a9e6d1); border-color:#0f9d6e; }
  .status{ text-align:center; font-size:13px; color:var(--muted); margin:10px 0; }
  .dot{ display:inline-block; width:8px; height:8px; border-radius:50%; background:#c9c8c2;
        margin-right:6px; vertical-align:1px; }
  .dot.ready{ background:var(--teal); } .dot.loading{ background:var(--gold); }
  .card{ background:var(--card); border:1px solid var(--line); border-radius:16px;
         box-shadow:0 1px 3px rgba(20,32,58,.04); }
  .chat{ padding:16px; min-height:320px; max-height:52vh; overflow-y:auto; }
  .msg{ margin:10px 0; display:flex; }
  .msg.u{ justify-content:flex-end; }
  .bub{ max-width:82%; padding:10px 14px; border-radius:14px; font-size:15px; white-space:pre-wrap; }
  .u .bub{ background:#eaf4ff; color:#12325a; border-bottom-right-radius:4px; }
  .a .bub{ background:#f4f6fb; color:var(--ink); border-bottom-left-radius:4px; }
  .meta{ font-size:12.5px; color:var(--muted); margin:-2px 6px 12px; font-weight:600; }
  .meta b{ color:var(--teal); font-size:13.5px; }
  .empty{ color:var(--muted); text-align:center; padding:40px 10px; font-size:14px; }
  .bar{ display:flex; gap:8px; padding:12px; border-top:1px solid var(--line); }
  #q,#abq{ flex:1; border:1px solid var(--line); border-radius:12px; padding:12px 14px; font-size:15px;
      font-family:inherit; outline:none; }
  #q:focus,#abq:focus{ border-color:var(--teal); }
  #send,#absend{ background:var(--teal); color:#fff; border:none; border-radius:12px; padding:0 20px;
         font-weight:700; font-size:15px; cursor:pointer; }
  #send:disabled,#absend:disabled{ background:#e7ecf5; color:#9aa6bd; cursor:not-allowed; }
  .about{ margin-top:24px; padding:20px 22px; font-size:14.5px; color:#2b3a56; }
  .about h3{ margin:0 0 8px; font-size:16px; }
  .about table{ width:100%; border-collapse:collapse; margin:10px 0; font-size:13.5px; }
  .about td,.about th{ border-bottom:1px solid var(--line); padding:7px 6px; text-align:left; }
  .links{ display:flex; gap:10px; flex-wrap:wrap; margin-top:12px; }
  .links a{ text-decoration:none; font-size:13px; font-weight:600; color:#08321f;
            background:#eef7f3; border:1px solid #cfe9df; border-radius:10px; padding:7px 12px; }
  .note{ color:var(--muted); font-size:12.5px; margin-top:14px; }
  .ex{ display:flex; gap:8px; flex-wrap:wrap; padding:0 12px 12px; }
  .ex button{ background:#fff; border:1px solid var(--line); border-radius:10px; padding:7px 12px;
              font-size:13px; color:var(--muted); cursor:pointer; }
  /* A/B */
  .abhead{ display:grid; grid-template-columns:1fr 1fr; gap:12px; padding:14px 14px 4px; }
  .abcol{ font-size:15px; font-weight:800; }
  .abcol span{ display:block; font-size:12px; font-weight:600; color:var(--muted); margin-top:2px; }
  .abcol.b{ color:var(--bonsai); } .abcol.p{ color:var(--teal); }
  .abgrid{ display:grid; grid-template-columns:1fr 1fr; gap:12px; padding:6px 14px 4px; }
  .abpane{ border:1px solid var(--line); border-radius:12px; padding:12px; min-height:200px;
           max-height:42vh; overflow-y:auto; background:#fbfcfe; }
  .abpane .txt{ font-size:14px; white-space:pre-wrap; color:var(--ink); }
  .abpane .m{ font-size:12.5px; font-weight:700; color:var(--muted); margin-top:8px; }
  .abpane .m b{ font-size:14px; }
  .abpane.b .m b{ color:var(--bonsai); } .abpane.p .m b{ color:var(--teal); }
  .abresult{ text-align:center; font-size:17px; font-weight:800; color:var(--ink);
             padding:12px; min-height:24px; }
  .abresult b{ color:var(--teal); }
  @media (max-width:640px){ .abhead,.abgrid{ grid-template-columns:1fr; } }
</style>
</head>
<body>
<div class="wrap">
  <header>
    <div class="brand">POCKET-26B</div>
    <div class="tag">A 26B model that out-runs the top 1-bit 27B — on a CPU, no GPU.</div>
    <div class="sub">Gemma4-based sparse Mixture-of-Experts · ~4B active per token · same stock llama.cpp, side by side</div>
    <div class="chips">
      <span class="chip g">Gemma4-26B-A4B based</span>
      <span class="chip"><b>25.2B</b> total</span>
      <span class="chip"><b>~4B</b> active/token</span>
      <span class="chip gpu"><b>0</b> GPU</span>
      <span class="chip">Q4_K_M · 17&nbsp;GB</span>
    </div>
  </header>

  <div class="tabs">
    <button class="tab active" data-tab="ab">⚔️ POCKET vs Bonsai</button>
    <button class="tab" data-tab="chat">💬 Chat</button>
  </div>

  <div class="status"><span id="dot" class="dot loading"></span><span id="st">starting…</span></div>

  <!-- ===== Chat (secondary tab) ===== -->
  <div id="panel-chat" style="display:none">
    <div class="card">
      <div class="chat" id="chat"><div class="empty" id="empty">Ask anything. This runs on CPU only — first reply may take a moment.</div></div>
      <div class="ex" id="ex">
        <button>Explain mixture-of-experts in one line.</button>
        <button>Write a haiku about running AI with no GPU.</button>
        <button>Answer in Korean: introduce yourself in three sentences.</button>
      </div>
      <div class="bar">
        <input id="q" placeholder="Type a message…" autocomplete="off"/>
        <button id="send" disabled>Send</button>
      </div>
    </div>

    <div class="card about">
      <h3>What is POCKET-26B?</h3>
      <p>POCKET-26B is the <b>Gemma4</b> sibling of the POCKET on-device family — built from Google's
      <a href="https://huggingface.co/google/gemma-4-26B-A4B-it">Gemma4-26B-A4B</a> (Apache-2.0). It is a sparse
      Mixture-of-Experts model (25.2B total, ~4B active per token), so it runs on a phone and on a PC with
      <b>no GPU</b>, in the apps you already use. This demo serves the <b>Q4_K_M</b> build (17&nbsp;GB) on a
      plain CPU. It is <b>unpruned</b> and keeps <b>GPQA-Diamond 67%</b> — on par with the full Gemma4-26B —
      via Korean-imatrix calibration + mixed-precision quantization.</p>
      <table>
        <tr><th>Build</th><th>Size</th><th>Runs on</th><th>Best for</th></tr>
        <tr><td>POCKET-26B <b>Q4_K_M</b> ⭐ (this demo)</td><td>17 GB</td><td>PC 32 GB</td><td>top quality</td></tr>
        <tr><td>POCKET-26B Q2_K</td><td>11 GB</td><td>12 GB phone / mini-PC, no GPU</td><td>smallest footprint</td></tr>
        <tr><td>POCKET-KR / EN (Qwen, 5 GB)</td><td>~5 GB</td><td>iPhone / Android</td><td>smallest footprint</td></tr>
      </table>
      <div class="links">
        <a href="https://huggingface.co/collections/FINAL-Bench/pocket-models-6a618ee5d23eafb7e185a5c6">📚 POCKET collection</a>
        <a href="https://huggingface.co/FINAL-Bench/POCKET-26B-GGUF">POCKET-26B-GGUF</a>
        <a href="https://huggingface.co/google/gemma-4-26B-A4B-it">🧠 Gemma4-26B-A4B (base)</a>
        <a href="https://huggingface.co/FINAL-Bench/POCKET-35B-GGUF">POCKET-35B (Qwen sibling)</a>
      </div>
      <div class="note">This Space runs the Q4_K_M build on an upgraded CPU (no GPU); a smaller Q2_K (11 GB) is also available to download. Speed here reflects a shared CPU box — a dedicated mini-PC is faster. Apache-2.0 · a VIDRAFT model family.</div>
    </div>
  </div>

  <!-- ===== POCKET vs Bonsai (A/B) — primary/landing tab ===== -->
  <div id="panel-ab">
    <div class="card">
      <div class="abhead">
        <div class="abcol b">🌳 Bonsai-27B <span>1-bit dense · runs first</span></div>
        <div class="abcol p">📦 POCKET-26B <span>Gemma4 sparse MoE · runs second</span></div>
      </div>
      <div class="abgrid">
        <div class="abpane b"><div class="txt" id="ab-b-txt"></div><div class="m" id="ab-b-m"></div></div>
        <div class="abpane p"><div class="txt" id="ab-p-txt"></div><div class="m" id="ab-p-m"></div></div>
      </div>
      <div class="abresult" id="ab-result">Type a prompt — Bonsai answers first, then POCKET-26B on the same CPU.</div>
      <div class="ex" id="abex">
        <button>In one sentence, why is the sky blue?</button>
        <button>Introduce yourself in one short sentence.</button>
        <button>Give me a one-sentence tip for better sleep.</button>
      </div>
      <div class="bar">
        <input id="abq" placeholder="Ask something short — both models answer…" autocomplete="off"/>
        <button id="absend" disabled>Race</button>
      </div>
    </div>

    <div class="card about">
      <h3>What am I looking at?</h3>
      <p>The exact same prompt is sent to <b>Bonsai-27B</b> (the most-downloaded on-device model, a 1-bit dense build)
      and to <b>POCKET-26B</b> (our Gemma4-based sparse Mixture-of-Experts, Q4_K_M) — on the
      <b>same CPU-only box, the same stock <code>llama.cpp</code></b>. Bonsai runs first, then POCKET. Watch the
      <b>tok/s</b> on each side.</p>
      <p>Why POCKET wins on generation: a dense 27B reads all of its weights every token; POCKET's MoE wakes only
      ~4B parameters. Fewer bytes moved per token → faster on a CPU. The weaker the hardware, the wider the gap.</p>
      <div class="note"><b>Honest notes:</b> sequential race, each model gets the full CPU. tok/s is llama.cpp's measured generation speed. On a shared box a dedicated machine is faster. Prompt <i>processing</i> (not shown) is one place Bonsai can lead on very long inputs. Bonsai is a trademark of its owners; shown here for factual comparison only.</div>
      <div class="links">
        <a href="https://huggingface.co/FINAL-Bench/POCKET-26B-GGUF">📦 POCKET-26B-GGUF</a>
        <a href="https://huggingface.co/collections/FINAL-Bench/pocket-models-6a618ee5d23eafb7e185a5c6">📚 POCKET collection</a>
        <a href="https://huggingface.co/google/gemma-4-26B-A4B-it">🧠 Gemma4-26B-A4B (base)</a>
      </div>
    </div>
  </div>
</div>

<script>
const dot=document.getElementById('dot'), st=document.getElementById('st');
let ready={pocket:false,bonsai:false};

/* ---- tabs ---- */
document.querySelector('.tabs').addEventListener('click',e=>{
  if(e.target.tagName!=='BUTTON') return;
  document.querySelectorAll('.tab').forEach(t=>t.classList.remove('active'));
  e.target.classList.add('active');
  const t=e.target.dataset.tab;
  document.getElementById('panel-chat').style.display = t==='chat'?'':'none';
  document.getElementById('panel-ab').style.display   = t==='ab'?'':'none';
});

/* ---- status poll ---- */
async function poll(){
  try{ const r=await fetch('/api/status'); const s=await r.json();
    st.textContent=s.detail; dot.className='dot '+(s.state==='ready'?'ready':'loading');
    ready.pocket=!!s.pocket; ready.bonsai=!!s.bonsai;
    send.disabled = !ready.pocket || busy;
    absend.disabled = !(ready.pocket&&ready.bonsai) || abBusy;
    if(!(ready.pocket&&ready.bonsai)) setTimeout(poll,2500);
  }catch(e){ setTimeout(poll,3000); }
}

/* ---- Chat tab ---- */
const chat=document.getElementById('chat'), empty=document.getElementById('empty');
const q=document.getElementById('q'), send=document.getElementById('send');
let history=[], busy=false;
function add(role,text){
  if(empty&&empty.parentNode) empty.remove();
  const m=document.createElement('div'); m.className='msg '+(role==='user'?'u':'a');
  const b=document.createElement('div'); b.className='bub'; b.textContent=text; m.appendChild(b);
  chat.appendChild(m); chat.scrollTop=chat.scrollHeight; return b;
}
async function ask(text){
  if(busy||!text.trim()) return; busy=true; send.disabled=true; q.value='';
  add('user',text); history.push({role:'user',content:text});
  const bub=add('assistant','');
  const meta=document.createElement('div'); meta.className='meta'; meta.textContent='⏳ generating on CPU…'; chat.appendChild(meta);
  let acc='', nTok=0, t0=0;
  try{
    const r=await fetch('/api/chat',{method:'POST',headers:{'Content-Type':'application/json'},
      body:JSON.stringify({messages:history,max_tokens:512})});
    const rd=r.body.getReader(); const dec=new TextDecoder(); let buf='';
    while(true){ const {done,value}=await rd.read(); if(done) break;
      buf+=dec.decode(value,{stream:true}); const parts=buf.split('\n\n'); buf=parts.pop();
      for(const p of parts){ if(!p.startsWith('data:')) continue;
        const j=JSON.parse(p.slice(5));
        if(j.t){ if(!t0) t0=performance.now(); nTok++; acc+=j.t; bub.textContent=acc;
          if(nTok>1){ const s=nTok/((performance.now()-t0)/1000); meta.innerHTML='⚡ <b>'+s.toFixed(1)+'</b> tok/s'; }
          chat.scrollTop=chat.scrollHeight; }
        if(j.done){ const sp=j.tok_s || (t0? nTok/((performance.now()-t0)/1000) : 0);
          meta.innerHTML='⚡ <b>'+(sp?sp.toFixed(1):'—')+'</b> tok/s · '+(j.n||nTok)+' tokens · on CPU, no GPU'; }
        if(j.error){ bub.textContent='⚠️ '+j.error; meta.remove(); }
      }
    }
    history.push({role:'assistant',content:acc});
  }catch(e){ bub.textContent='⚠️ '+e; }
  busy=false; send.disabled=false; q.focus();
}
send.onclick=()=>ask(q.value);
q.addEventListener('keydown',e=>{ if(e.key==='Enter') ask(q.value); });
document.getElementById('ex').addEventListener('click',e=>{ if(e.target.tagName==='BUTTON') ask(e.target.textContent); });

/* ---- A/B race ---- */
const absend=document.getElementById('absend'), abq=document.getElementById('abq');
const abTxt={bonsai:document.getElementById('ab-b-txt'),pocket:document.getElementById('ab-p-txt')};
const abM={bonsai:document.getElementById('ab-b-m'),pocket:document.getElementById('ab-p-m')};
const abResult=document.getElementById('ab-result');
let abBusy=false;
async function race(text){
  if(abBusy||!text.trim()) return; abBusy=true; absend.disabled=true; abq.value='';
  abTxt.bonsai.textContent=''; abTxt.pocket.textContent='';
  abM.bonsai.innerHTML='⏳ Bonsai generating…'; abM.pocket.innerHTML='⏳ waiting for Bonsai to finish…';
  abResult.textContent='Racing on one CPU…';
  const acc={bonsai:'',pocket:''}, cnt={bonsai:0,pocket:0}, t0={bonsai:0,pocket:0}, spd={bonsai:0,pocket:0};
  try{
    const r=await fetch('/api/chat_ab',{method:'POST',headers:{'Content-Type':'application/json'},
      body:JSON.stringify({messages:[{role:'user',content:text}],max_tokens:512})});
    const rd=r.body.getReader(); const dec=new TextDecoder(); let buf='';
    while(true){ const {done,value}=await rd.read(); if(done) break;
      buf+=dec.decode(value,{stream:true}); const parts=buf.split('\n\n'); buf=parts.pop();
      for(const p of parts){ if(!p.startsWith('data:')) continue;
        const j=JSON.parse(p.slice(5)); const m=j.m;
        if(j.t && m){ if(!t0[m]) t0[m]=performance.now(); cnt[m]++; acc[m]+=j.t; abTxt[m].textContent=acc[m];
          if(cnt[m]>1){ const s=cnt[m]/((performance.now()-t0[m])/1000); abM[m].innerHTML='⚡ <b>'+s.toFixed(1)+'</b> tok/s'; }
          abTxt[m].parentNode.scrollTop=abTxt[m].parentNode.scrollHeight; }
        if(j.done && m){ spd[m]=j.tok_s||0;
          abM[m].innerHTML='⚡ <b>'+(spd[m]?spd[m].toFixed(1):'—')+'</b> tok/s · '+(j.n||cnt[m])+' tokens';
          if(m==='bonsai') abM.pocket.innerHTML='⏳ POCKET-26B generating…'; }
        if(j.error && m){ abTxt[m].textContent='⚠️ '+j.error; }
        if(j.ab_done){
          if(spd.bonsai>0 && spd.pocket>0){ const x=spd.pocket/spd.bonsai;
            abResult.innerHTML='⚡ POCKET-26B was <b>'+x.toFixed(1)+'× faster</b> than Bonsai — same CPU, no GPU';
          } else { abResult.textContent='Done.'; } }
      }
    }
  }catch(e){ abResult.textContent='⚠️ '+e; }
  abBusy=false; absend.disabled=false; abq.focus();
}
absend.onclick=()=>race(abq.value);
abq.addEventListener('keydown',e=>{ if(e.key==='Enter') race(abq.value); });
document.getElementById('abex').addEventListener('click',e=>{ if(e.target.tagName==='BUTTON') race(e.target.textContent); });

poll();
</script>
</body>
</html>