Upload 6 files

app.py

@@ -0,0 +1,724 @@
+import gradio as gr
+import threading, time, json
+from datetime import datetime, UTC
+from collections import deque
+from pathlib import Path
+
+from neural_network import LivingNetwork, CATEGORIES as TEXT_CATS, KNOWLEDGE_FILE
+from data_fetcher import DataFetcher
+from image_fetcher import ImageFetcher, IMAGE_DIR, CATEGORIES as IMG_CATS
+from image_model import LivingImageNetwork
+from chatbot import RAGChatbot
+
+# ── globals ──────────────────────────────────────────────────────────────────
+network     = LivingNetwork()
+fetcher     = DataFetcher()
+img_fetcher = ImageFetcher()
+img_network = LivingImageNetwork()
+chatbot     = RAGChatbot()
+
+app_log   = deque(maxlen=300)
+is_alive  = False
+img_alive = False
+
+def log(msg):
+    app_log.appendleft(f"[{datetime.now(UTC).strftime('%H:%M:%S')}] {msg}")
+
+log("🧠 Text network ready")
+log("👁️  Image CNN ready")
+kf = network.get_knowledge_file_stats()
+if kf['exists']:   log(f"📚 Restored {kf['lines']} articles from knowledge.jsonl")
+if network.epoch:  log(f"✅ Text model: epoch {network.epoch:,}")
+if img_network.epoch: log(f"✅ Image CNN: epoch {img_network.epoch:,}")
+
+# ── background loops ─────────────────────────────────────────────────────────
+def text_loop():
+    global is_alive
+    lf = ls = 0
+    log("🚀 Text network LIVE")
+    while is_alive:
+        now = time.time()
+        if now - lf >= 40:
+            try:
+                items = fetcher.fetch_round()
+                for it in items:
+                    network.ingest(it['text'], it['category'], source=it.get('source','web'))
+                kf2 = network.get_knowledge_file_stats()
+                log(f"📡 +{len(items)} articles | knowledge: {kf2['lines']}")
+                lf = now
+            except Exception as e:
+                log(f"⚠ fetch: {str(e)[:50]}")
+        if network.stats['buffer_size'] >= 32 and network.vocab.is_built:
+            r = network.train_n_steps(80)
+            if r['steps']:
+                s = network.stats
+                log(f"📝 Epoch {s['epoch']:,} loss {s['loss']} acc {s['accuracy']}%")
+        else:
+            log(f"⏳ need {max(0,32-network.stats['buffer_size'])} more items")
+        if now - ls >= 180:
+            network.save_checkpoint(); log("💾 saved"); ls = now
+        time.sleep(2)
+    log("⏹ text stopped")
+
+def image_loop():
+    global img_alive
+    lf = ls = 0
+    log("🚀 Image CNN LIVE")
+    while img_alive:
+        now = time.time()
+        if now - lf >= 60:
+            try:
+                r = img_fetcher.fetch_round()
+                log(f"🖼️  +{r['downloaded']} images | total {r['total']}")
+                lf = now
+            except Exception as e:
+                log(f"⚠ img fetch: {str(e)[:50]}")
+        if img_fetcher.get_stats()['total'] >= 16:
+            r = img_network.train_n_steps(IMAGE_DIR, 20)
+            if r['steps']:
+                s = img_network.stats
+                log(f"👁️  CNN epoch {s['epoch']:,} loss {s['loss']} acc {s['accuracy']}%")
+        else:
+            log(f"⏳ img CNN needs {max(0,16-img_fetcher.get_stats()['total'])} more images")
+        if now - ls >= 180:
+            img_network._save_checkpoint(); log("💾 img saved"); ls = now
+        time.sleep(3)
+    log("⏹ image stopped")
+
+# ── controls ──────────────────────────────────────────────────────────────────
+def start_text():
+    global is_alive
+    if is_alive: return "⚠ already running"
+    is_alive = True
+    threading.Thread(target=text_loop, daemon=True).start()
+    return "🟢 text network LIVE"
+
+def stop_text():
+    global is_alive; is_alive = False
+    network.save_checkpoint(); return "🔴 stopped + saved"
+
+def start_img():
+    global img_alive
+    if img_alive: return "⚠ already running"
+    img_alive = True
+    threading.Thread(target=image_loop, daemon=True).start()
+    return "🟢 image CNN LIVE"
+
+def stop_img():
+    global img_alive; img_alive = False
+    img_network._save_checkpoint(); return "🔴 stopped + saved"
+
+def do_fetch_text():
+    items = fetcher.fetch_round()
+    for it in items: network.ingest(it['text'], it['category'], source=it.get('source','web'))
+    return f"✅ +{len(items)} articles"
+
+def do_fetch_img():
+    r = img_fetcher.fetch_round()
+    return f"✅ +{r['downloaded']} images"
+
+def do_train_text(n):
+    r = network.train_n_steps(int(n))
+    return f"✅ {r['steps']} steps · loss {r['avg_loss']}" if r['steps'] else "⚠ need more data"
+
+def do_train_img(n):
+    r = img_network.train_n_steps(IMAGE_DIR, int(n))
+    return f"✅ {r['steps']} steps · loss {r['avg_loss']}" if r['steps'] else "⚠ need more images"
+
+# ── stats fns ─────────────────────────────────────────────────────────────────
+def status_bar():
+    s = network.stats; si = img_network.stats
+    kf2 = network.get_knowledge_file_stats(); ig = img_fetcher.get_stats()
+    return (f"{'🟢' if is_alive else '🔴'} TEXT ep{s['epoch']:,} loss{s['loss']} acc{s['accuracy']}%  |  "
+            f"{'🟢' if img_alive else '🔴'} CNN ep{si['epoch']:,} loss{si['loss']} acc{si['accuracy']}%  |  "
+            f"📚{kf2['lines']} articles  🖼️{ig['total']} images")
+
+def text_stats():
+    s=network.stats; fs=fetcher.get_stats(); cc=network.category_counts
+    hist=network.loss_history[-30:]
+    spark=''.join(' ▁▂▃▄▅▆▇█'[min(8,int(((v-min(hist))/(max(hist)-min(hist)+1e-9))*8))] for v in hist) if len(hist)>=2 else '…'
+    rows='\n'.join(f"|{c}|{cc.get(c,0)}|" for c in TEXT_CATS)
+    return f"""### 📝 Text Network
+|Metric|Value|
+|---|---|
+|Epoch|{s['epoch']:,}|Loss|{s['loss']}|
+|Accuracy|{s['accuracy']}%|Samples|{s['total_samples']:,}|
+|Knowledge|{s['knowledge_count']} articles|Vocab|{s['vocab_size']:,}|
+|LR|{s['lr']}|Buffer|{s['buffer_size']}|
+
+Loss `{spark}`
+
+### Sources — {fs['total_fetched']} total
+RSS·{fs['sources']['rss']} Reddit·{fs['sources']['reddit']} Wiki·{fs['sources']['wikipedia']} HN·{fs['sources']['hackernews']}
+
+### Categories
+|Cat|Count|
+|---|---|
+{rows}
+> _{s['last_text']}_"""
+
+def img_stats():
+    s=img_network.stats; ifs=img_fetcher.get_stats()
+    hist=img_network.loss_history[-30:]
+    spark=''.join(' ▁▂▃▄▅▆▇█'[min(8,int(((v-min(hist))/(max(hist)-min(hist)+1e-9))*8))] for v in hist) if len(hist)>=2 else '…'
+    rows='\n'.join(f"|{c}|{ifs['by_category'].get(c,0)}|" for c in IMG_CATS)
+    return f"""### 👁️ Image CNN
+|Metric|Value|
+|---|---|
+|Epoch|{s['epoch']:,}|Loss|{s['loss']}|
+|Accuracy|{s['accuracy']}%|Images|{s['total_images']:,}|
+|On Disk|{ifs['total']}|Disk|{ifs['disk_mb']} MB|
+
+Loss `{spark}`
+
+### By Category
+|Cat|Count|
+|---|---|
+{rows}
+
+**What each block learns:**
+Block 1 → edges & colours · Block 2 → shapes & textures · Block 3 → objects & parts"""
+
+def get_log():
+    lines = list(app_log)[:80]
+    def color(line):
+        if '✅' in line or '🟢' in line: c = '#4ade80'
+        elif '⚠' in line or '🔴' in line or 'error' in line.lower(): c = '#f87171'
+        elif '🚀' in line or '🤖' in line: c = '#818cf8'
+        elif '📰' in line or '🦆' in line or '🟠' in line: c = '#38bdf8'
+        elif '💻' in line or '📖' in line: c = '#fb923c'
+        elif '🔥' in line or 'loss' in line.lower(): c = '#facc15'
+        else: c = '#94a3b8'
+        return f'<div style="color:{c};font-family:monospace;font-size:12px;padding:1px 0">{line}</div>'
+    rows = ''.join(color(l) for l in lines) or '<div style="color:#64748b;font-style:italic">No logs yet...</div>'
+    return f'<div style="background:#0a0f1e;padding:12px;border-radius:8px;max-height:420px;overflow-y:auto;border:1px solid #1e293b">{rows}</div>' 
+def get_feed():
+    items=fetcher.get_recent_items(10)
+    if not items: return "_No data yet_"
+    return '\n\n---\n\n'.join(f"**[{i['source'].upper()}]** `{i['category'].upper()}`\n{i['text'][:130]}…" for i in items)
+
+def get_knowledge():
+    kf2=network.get_knowledge_file_stats(); items=network.get_recent_knowledge(20)
+    if not kf2['exists'] or not items: return "### 📭 Empty — start text network"
+    rows=[]
+    for it in items:
+        rows.append(f"**`{it.get('category','?').upper()}`** `{it.get('source','?')}` `{it.get('timestamp','')[:10]}`\n> {it.get('text','')[:130]}…")
+    return f"### 📚 {kf2['lines']} articles · {kf2['size_kb']} KB\n\n---\n\n"+'\n\n---\n\n'.join(rows)
+
+def predict_text(txt):
+    if not txt.strip(): return "Enter text."
+    r=network.predict(txt)
+    if 'error' in r: return f"⚠ {r['error']}"
+    bars=''.join(f"`{c:<15}` {'█'*int(p/5)}{'░'*(20-int(p/5))} **{p:.1f}%**\n\n" for c,p in sorted(r['all_probs'].items(),key=lambda x:-x[1]))
+    return f"## → {r['prediction'].upper()}\n**{r['confidence']}% confidence**\n\n{bars}"
+
+def predict_img(path):
+    if not path: return "Upload image."
+    r=img_network.predict_image(path)
+    if 'error' in r: return f"⚠ {r['error']}"
+    bars=''.join(f"`{c:<15}` {'█'*int(p/5)}{'░'*(20-int(p/5))} **{p:.1f}%**\n\n" for c,p in sorted(r['all_probs'].items(),key=lambda x:-x[1]))
+    return f"## → {r['prediction'].upper()}\n**{r['confidence']}% confidence**\n\n{bars}"
+
+def chat_fn(msg, history):
+    new_history = chatbot.chat(msg, history or [])
+    return new_history, ""  # also clear the input box
+
+# ── 3D VIZ — fully animated with signal pulses ────────────────────────────────
+def build_viz(state_json: str) -> str:
+    return """<!DOCTYPE html><html><head><meta charset="UTF-8">
+<style>
+*{margin:0;padding:0;box-sizing:border-box}
+body{background:#030610;overflow:hidden;font-family:monospace}
+#c{width:100vw;height:100vh;display:block}
+#hud{position:fixed;top:12px;left:14px;color:#00f5c4;font-size:11px;line-height:2;pointer-events:none;text-shadow:0 0 8px #00f5c466}
+#tip{position:fixed;bottom:12px;left:50%;transform:translateX(-50%);font-size:10px;color:#2a3a5a}
+</style></head><body>
+<canvas id="c"></canvas>
+<div id="hud">
+  <div id="h1">■ LOADING...</div>
+  <div id="h2"></div><div id="h3"></div><div id="h4"></div>
+</div>
+<div id="tip">drag to rotate · scroll to zoom · signals travel left→right every 800ms</div>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
+<script>
+// ── initial state from Python ──
+const INIT = """ + state_json + """;
+
+// ── renderer ──
+const canvas = document.getElementById('c');
+const renderer = new THREE.WebGLRenderer({canvas, antialias:true, alpha:false});
+renderer.setPixelRatio(Math.min(devicePixelRatio,2));
+renderer.setSize(innerWidth, innerHeight);
+renderer.shadowMap.enabled = true;
+
+const scene  = new THREE.Scene();
+scene.background = new THREE.Color(0x030610);
+scene.fog = new THREE.FogExp2(0x030610, 0.010);
+
+const camera = new THREE.PerspectiveCamera(52, innerWidth/innerHeight, 0.1, 600);
+camera.position.set(0,1,26);
+
+// ── lights ──
+scene.add(new THREE.AmbientLight(0x0a1020, 4));
+const kl = new THREE.PointLight(0x00f5c4, 8, 80); kl.position.set(0,10,8); scene.add(kl);
+const bl = new THREE.PointLight(0x5b6fff, 4, 50); bl.position.set(-12,-6,4); scene.add(bl);
+const rl = new THREE.PointLight(0xff6b35, 2, 40); rl.position.set(12,-4,-2); scene.add(rl);
+
+// ── orbit ──
+let rX=0.18, rY=0, zoom=26, drag=false, last={x:0,y:0};
+canvas.addEventListener('mousedown', e=>{drag=true;last={x:e.clientX,y:e.clientY}});
+window.addEventListener('mouseup', ()=>drag=false);
+window.addEventListener('mousemove', e=>{
+  if(!drag)return;
+  rY += (e.clientX-last.x)*0.011;
+  rX += (e.clientY-last.y)*0.007;
+  rX = Math.max(-1.1, Math.min(1.1, rX));
+  last = {x:e.clientX, y:e.clientY};
+});
+canvas.addEventListener('wheel', e=>{zoom=Math.max(7,Math.min(45,zoom+e.deltaY*0.025));e.preventDefault();},{passive:false});
+let lt=null;
+canvas.addEventListener('touchstart',e=>{lt=e.touches[0];},{passive:true});
+canvas.addEventListener('touchmove',e=>{
+  if(!lt)return; const t=e.touches[0];
+  rY+=(t.clientX-lt.clientX)*0.011; rX+=(t.clientY-lt.clientY)*0.007;
+  lt=t; e.preventDefault();
+},{passive:false});
+
+// ── starfield ──
+const sg=new THREE.BufferGeometry();
+const sp=new Float32Array(600*3);
+for(let i=0;i<sp.length;i++) sp[i]=(Math.random()-0.5)*120;
+sg.setAttribute('position',new THREE.BufferAttribute(sp,3));
+scene.add(new THREE.Points(sg, new THREE.PointsMaterial({color:0x0d1a33,size:0.06})));
+
+// ── build network from state ──
+const group = new THREE.Group();
+scene.add(group);
+
+// ── DYNAMIC BRAIN SIZE — grows as the AI learns more ──────────────────────────
+const rawSizes = INIT.layer_sizes || [64,256,128,64,8];
+const kc = (INIT.stats && (INIT.stats.knowledge_count || INIT.stats.total_images)) || 0;
+// Scale: 0 articles = 1 neuron, 10=2, 30=3, 60=4, 100=5, 200=7, 500=10, 1000+=12
+const MAX_N  = kc===0 ? 1 : kc<5 ? 2 : kc<15 ? 3 : kc<40 ? 4 : kc<80 ? 5 : kc<150 ? 6 : kc<300 ? 8 : kc<600 ? 10 : 12;
+// Scale all layer sizes proportionally
+const scaleFactor = MAX_N / 12;
+const sizes  = rawSizes.map((s,i) => Math.max(1, Math.round(s * scaleFactor)));
+const nL     = sizes.length;
+const LGAP   = 5.8;
+const startX = -(nL-1)*LGAP/2;
+const acts   = INIT.activations || {};
+const aKeys  = Object.keys(acts);
+
+function getAct(li, ni){
+  const k=aKeys[li]; if(!k) return Math.random()*0.4+0.1;
+  const a=acts[k];
+  return (a&&a[ni]!=null) ? Math.max(0,Math.min(1,a[ni])) : Math.random()*0.3;
+}
+
+// Shared geometries
+const nGeo = new THREE.SphereGeometry(0.22, 18, 18);
+const gGeo = new THREE.SphereGeometry(0.34, 8, 8);
+
+const LAYER_NAMES = ['INPUT','HIDDEN-1','HIDDEN-2','HIDDEN-3','OUTPUT',
+                     'CONV-1','CONV-2','FC-1','FC-2'];
+
+// Store neuron meshes for animation
+const neurons = [];  // neurons[layerIdx] = [{mesh, glowMesh, baseAct, fireTimer, pos}]
+const connections = []; // {from, to, line, fromIdx, toIdx, lFromIdx, lToIdx}
+
+for(let li=0;li<nL;li++){
+  const show  = Math.min(sizes[li], MAX_N);
+  const yGap  = Math.min(1.9, 11/show);
+  const yStart= -(show-1)*yGap/2;
+  const layer = [];
+
+  for(let ni=0;ni<show;ni++){
+    const act = getAct(li,ni);
+    const x   = startX + li*LGAP;
+    const y   = yStart + ni*yGap;
+    const z   = (Math.random()-0.5)*0.5;
+
+    // Neuron colour based on activation
+    const r=Math.floor(act*240), g=Math.floor(act*200), b=Math.floor(60+act*100);
+    const col = new THREE.Color(`rgb(${r},${g},${b})`);
+
+    const mat = new THREE.MeshStandardMaterial({
+      color: col, emissive: col,
+      emissiveIntensity: 0.3+act*1.8,
+      roughness:0.2, metalness:0.9,
+    });
+    const mesh = new THREE.Mesh(nGeo, mat);
+    mesh.position.set(x,y,z);
+    group.add(mesh);
+
+    // Glow halo
+    const gMat = new THREE.MeshBasicMaterial({
+      color:0x00f5c4, transparent:true,
+      opacity: act>0.4?(act-0.4)*0.5:0,
+      wireframe:true
+    });
+    const gMesh = new THREE.Mesh(gGeo, gMat);
+    gMesh.position.set(x,y,z);
+    group.add(gMesh);
+
+    layer.push({mesh, gMesh, baseAct:act, fireTimer:0, pos:new THREE.Vector3(x,y,z)});
+  }
+
+  // Layer label
+  const lc=document.createElement('canvas');
+  lc.width=256; lc.height=40;
+  const lx=lc.getContext('2d');
+  lx.fillStyle='rgba(0,245,196,0.5)';
+  lx.font='bold 18px monospace'; lx.textAlign='center';
+  lx.fillText(LAYER_NAMES[li]||`L${li}`, 128, 28);
+  const lbl = new THREE.Mesh(
+    new THREE.PlaneGeometry(2.6,0.45),
+    new THREE.MeshBasicMaterial({map:new THREE.CanvasTexture(lc),transparent:true,side:THREE.DoubleSide})
+  );
+  lbl.position.set(startX+li*LGAP, yStart-2.0, 0);
+  group.add(lbl);
+  neurons.push(layer);
+}
+
+// Build connections between adjacent layers
+let connDrawn = 0;
+const MAX_CONN = 160;
+for(let li=0;li<nL-1&&connDrawn<MAX_CONN;li++){
+  const fromL=neurons[li], toL=neurons[li+1];
+  for(let fi=0;fi<fromL.length&&connDrawn<MAX_CONN;fi++){
+    for(let ti=0;ti<toL.length&&connDrawn<MAX_CONN;ti++){
+      const sig=(fromL[fi].baseAct+toL[ti].baseAct)/2;
+      if(sig<0.05&&Math.random()>0.3) continue;
+      const positive=Math.random()>0.35;
+      const col=positive?0x00f5c4:0xff6b35;
+      const pts=[fromL[fi].pos.clone(), toL[ti].pos.clone()];
+      const geo=new THREE.BufferGeometry().setFromPoints(pts);
+      const mat=new THREE.LineBasicMaterial({color:col,transparent:true,opacity:0.04+sig*0.35});
+      const line=new THREE.Line(geo,mat);
+      group.add(line);
+      connections.push({from:fromL[fi],to:toL[ti],line,mat,lFromIdx:li,lToIdx:li+1,fi,ti});
+      connDrawn++;
+    }
+  }
+}
+
+// ── SIGNAL PARTICLE SYSTEM ────────────────────────────────────────────────────
+// This is what makes the network look ALIVE — glowing orbs traveling between neurons
+const signals = [];
+const sigGeo  = new THREE.SphereGeometry(0.12, 8, 8);
+
+function spawnSignal(fromNeuron, toNeuron, color=0x00f5c4){
+  const mat  = new THREE.MeshBasicMaterial({color, transparent:true, opacity:0.95});
+  const mesh = new THREE.Mesh(sigGeo, mat);
+  mesh.position.copy(fromNeuron.pos);
+  scene.add(mesh);  // add to scene not group so it stays in world space
+
+  // Trail
+  const trailMat = new THREE.MeshBasicMaterial({color, transparent:true, opacity:0.4});
+  const trail    = new THREE.Mesh(new THREE.SphereGeometry(0.07,6,6), trailMat);
+  scene.add(trail);
+
+  signals.push({
+    mesh, trail, mat, trailMat,
+    from: fromNeuron.pos.clone(),
+    to:   toNeuron.pos.clone(),
+    toNeuron,
+    t: 0,
+    speed: 0.028 + Math.random()*0.015,
+    color,
+  });
+}
+
+function fireNeuron(layerIdx, neuronIdx, cascade=true){
+  if(layerIdx>=neurons.length || neuronIdx>=neurons[layerIdx].length) return;
+  const n = neurons[layerIdx][neuronIdx];
+  n.fireTimer = 1.0;
+
+  // Spawn signals to next layer
+  if(cascade && layerIdx < nL-1){
+    const nextL = neurons[layerIdx+1];
+    const count = Math.min(nextL.length, 2+Math.floor(Math.random()*3));
+    const shuffled = [...nextL].sort(()=>Math.random()-0.5).slice(0, count);
+    shuffled.forEach(target => {
+      spawnSignal(n, target, layerIdx===0?0x00f5c4:0x5b9fff);
+    });
+  }
+}
+
+// Every 800ms: fire random input neurons → cascade through the network
+let fireTimer = 0;
+function triggerFiring(){
+  if(neurons.length===0) return;
+  const inputLayer = neurons[0];
+  const count = 1+Math.floor(Math.random()*3);
+  for(let i=0;i<count;i++){
+    const ni = Math.floor(Math.random()*inputLayer.length);
+    fireNeuron(0, ni, true);
+  }
+}
+
+// ── LOSS HISTORY GRAPH ───────────────────────────────────────────────────────
+(function(){
+  const hist = INIT.loss_history||[];
+  if(hist.length<3) return;
+  const gc=document.createElement('canvas');
+  gc.width=200; gc.height=50;
+  gc.style.cssText='position:fixed;bottom:14px;right:14px;border:1px solid #1a2240;border-radius:6px;background:rgba(11,15,30,0.8)';
+  const ctx=gc.getContext('2d');
+  const mn=Math.min(...hist), mx=Math.max(...hist), rng=mx-mn||1;
+  // fill
+  ctx.beginPath();
+  hist.forEach((v,i)=>{const x=(i/(hist.length-1))*200,y=50-4-((v-mn)/rng)*42; i===0?ctx.moveTo(x,50):ctx.lineTo(x,y)});
+  ctx.lineTo(200,50); ctx.closePath();
+  ctx.fillStyle='rgba(0,245,196,0.08)'; ctx.fill();
+  // line
+  ctx.beginPath();
+  hist.forEach((v,i)=>{const x=(i/(hist.length-1))*200,y=50-4-((v-mn)/rng)*42; i===0?ctx.moveTo(x,y):ctx.lineTo(x,y)});
+  ctx.strokeStyle='#00f5c4'; ctx.lineWidth=1.5; ctx.stroke();
+  document.body.appendChild(gc);
+  const lb=document.createElement('div');
+  lb.style.cssText='position:fixed;bottom:68px;right:14px;font-size:9px;color:#2a3a5a;font-family:monospace';
+  lb.textContent='LOSS ↓'; document.body.appendChild(lb);
+})();
+
+// ── HUD ──────────────────────────────────────────────────────────────────────
+const st = INIT.stats||{};
+const isImg = INIT.type==='cnn';
+const kCount = st.knowledge_count || st.total_images || 0;
+document.getElementById('h1').textContent = (isImg?'👁️ IMAGE CNN':'📝 TEXT NETWORK')+' — LIVE';
+document.getElementById('h2').textContent = `EPOCH  ${(st.epoch||0).toLocaleString()}`;
+document.getElementById('h3').textContent = `LOSS   ${st.loss||'—'}  ACC ${st.accuracy||'—'}%`;
+document.getElementById('h4').textContent = `BRAIN  ${kCount} ${isImg?'images':'articles'} · ${sizes.map((s,i)=>Math.min(s,MAX_N)).join('→')} neurons`;
+
+// ── ANIMATE ──────────────────────────────────────────────────────────────────
+let frame = 0;
+const clock = new THREE.Clock();
+
+function animate(){
+  requestAnimationFrame(animate);
+  const dt    = clock.getDelta();
+  const elaps = clock.getElapsedTime();
+  frame++;
+
+  // ① Auto-fire every 800ms
+  fireTimer += dt;
+  if(fireTimer > 0.8){
+    triggerFiring();
+    fireTimer = 0;
+  }
+
+  // ② Update neurons — pulse and fire glow
+  neurons.forEach((layer, li)=>{
+    layer.forEach((n, ni)=>{
+      // Base sinusoidal pulse — each neuron breathes at its own rate
+      const phase   = li*0.7 + ni*0.3;
+      const pulse   = Math.sin(elaps*2.1+phase)*0.15 + Math.sin(elaps*0.9+phase*2)*0.08;
+      const act     = Math.max(0, Math.min(1, n.baseAct + pulse + n.fireTimer*0.6));
+
+      const r=Math.floor(act*240), g=Math.floor(act*200), b=Math.floor(60+act*100);
+      n.mesh.material.emissive.setRGB(r/255, g/255, b/255);
+      n.mesh.material.emissiveIntensity = 0.3 + act*2.2;
+      n.mesh.scale.setScalar(1 + n.fireTimer*0.6 + pulse*0.05);
+
+      // Glow halo
+      n.gMesh.material.opacity = n.fireTimer>0 ? n.fireTimer*0.7 : Math.max(0,(act-0.45)*0.4);
+      n.gMesh.scale.setScalar(1 + n.fireTimer*1.0);
+
+      // Decay fire
+      if(n.fireTimer > 0) n.fireTimer = Math.max(0, n.fireTimer - dt*1.8);
+    });
+  });
+
+  // ③ Update connection lines — flash when signals pass
+  connections.forEach(conn=>{
+    const sig=(conn.from.baseAct+conn.to.baseAct)/2;
+    const flash = conn.from.fireTimer*0.5;
+    conn.mat.opacity = Math.min(0.9, 0.04 + sig*0.3 + flash);
+  });
+
+  // ④ Move signal particles
+  for(let i=signals.length-1;i>=0;i--){
+    const s=signals[i];
+    s.t += s.speed;
+    if(s.t>=1){
+      scene.remove(s.mesh); scene.remove(s.trail);
+      signals.splice(i,1);
+      // Fire destination neuron (next layer cascade)
+      s.toNeuron.fireTimer = 0.8;
+      // If not at output, cascade further
+      const lIdx = neurons.findIndex(l=>l.includes(s.toNeuron));
+      if(lIdx>=0 && lIdx<nL-1){
+        const nextL=neurons[lIdx+1];
+        if(Math.random()>0.3){
+          const t2=nextL[Math.floor(Math.random()*nextL.length)];
+          spawnSignal(s.toNeuron, t2, 0x5b9fff);
+        }
+      }
+    } else {
+      s.mesh.position.lerpVectors(s.from, s.to, s.t);
+      s.trail.position.lerpVectors(s.from, s.to, Math.max(0,s.t-0.08));
+      // Fade out near end
+      s.mat.opacity = s.t<0.8 ? 0.95 : (1-s.t)*4.75;
+      s.trailMat.opacity = s.t<0.8 ? 0.35 : (1-s.t)*1.75;
+    }
+  }
+
+  // ⑤ Camera orbit + gentle network bob
+  const autoY = elaps * 0.12;
+  camera.position.x = Math.sin(rY + autoY)*zoom;
+  camera.position.y = Math.sin(rX)*zoom*0.45 + 1;
+  camera.position.z = Math.cos(rY + autoY)*zoom;
+  camera.lookAt(0, 0, 0);
+
+  kl.intensity = 7 + Math.sin(elaps*1.5)*2;
+  bl.intensity = 3 + Math.cos(elaps*2.3)*1;
+  kl.position.x = Math.sin(elaps*0.4)*6;
+  group.position.y = Math.sin(elaps*0.6)*0.15;
+
+  renderer.render(scene, camera);
+}
+animate();
+
+window.addEventListener('resize',()=>{
+  camera.aspect=innerWidth/innerHeight;
+  camera.updateProjectionMatrix();
+  renderer.setSize(innerWidth,innerHeight);
+});
+</script></body></html>"""
+
+def get_text_viz():
+    s = network.get_viz_state(); s['type']='text'
+    h = build_viz(json.dumps(s))
+    e = h.replace('"','&quot;').replace('\n','&#10;')
+    return f'<iframe srcdoc="{e}" style="width:100%;height:650px;border:none;border-radius:12px"></iframe>'
+
+def get_img_viz():
+    s = img_network.get_viz_state()
+    h = build_viz(json.dumps(s))
+    e = h.replace('"','&quot;').replace('\n','&#10;')
+    return f'<iframe srcdoc="{e}" style="width:100%;height:650px;border:none;border-radius:12px"></iframe>'
+
+# ── UI ────────────────────────────────────────────────────────────────────────
+with gr.Blocks(title="Living Neural Network") as demo:
+    gr.Markdown("# 🧠 Living Neural Network\n*Two AIs training on live internet data — one reads, one looks*")
+
+    sbar = gr.Textbox(label="", value=status_bar(), interactive=False, lines=2)
+
+    with gr.Tabs():
+
+        # CONTROL
+        with gr.Tab("🎛️ Control"):
+            gr.Markdown("### 📝 Text Network")
+            with gr.Row():
+                gr.Button("▶ Start Text", variant="primary").click(start_text, outputs=gr.Textbox(label="",lines=1,interactive=False))
+                gr.Button("⏹ Stop Text",  variant="stop"   ).click(stop_text,  outputs=gr.Textbox(label="",lines=1,interactive=False))
+                gr.Button("📡 Fetch Text"                  ).click(do_fetch_text, outputs=gr.Textbox(label="",lines=1,interactive=False))
+            with gr.Row():
+                ts=gr.Slider(10,500,100,step=10,label="Steps")
+                gr.Button("🔥 Train Text").click(do_train_text,inputs=ts,outputs=gr.Textbox(label="",lines=1,interactive=False))
+            gr.Markdown("---\n### 👁️ Image CNN")
+            with gr.Row():
+                gr.Button("▶ Start Images",variant="primary").click(start_img, outputs=gr.Textbox(label="",lines=1,interactive=False))
+                gr.Button("⏹ Stop Images",variant="stop"   ).click(stop_img,  outputs=gr.Textbox(label="",lines=1,interactive=False))
+                gr.Button("🖼️ Fetch Images"                ).click(do_fetch_img, outputs=gr.Textbox(label="",lines=1,interactive=False))
+            with gr.Row():
+                is_=gr.Slider(5,100,20,step=5,label="Steps")
+                gr.Button("🔥 Train CNN").click(do_train_img,inputs=is_,outputs=gr.Textbox(label="",lines=1,interactive=False))
+
+        # STATS
+        with gr.Tab("📊 Stats"):
+            with gr.Row():
+                tmd=gr.Markdown(value=text_stats())
+                imd=gr.Markdown(value=img_stats())
+            lbox=gr.HTML(value=get_log(),label="Live Logs")
+
+        # 3D TEXT VIZ
+        with gr.Tab("🔮 Text Network 3D"):
+            gr.Markdown("**Neurons fire & signals travel left→right in real time.** Drag=rotate Scroll=zoom")
+            tvb=gr.Button("🔄 Refresh",variant="primary")
+            tvo=gr.HTML('<div style="height:650px;background:#030610;border-radius:12px;display:flex;align-items:center;justify-content:center;color:#00f5c4;font-family:monospace;font-size:14px">Click 🔄 Refresh to load the 3D network</div>')
+            tvb.click(get_text_viz,outputs=tvo)
+
+        # 3D IMAGE VIZ
+        with gr.Tab("👁️ Image CNN 3D"):
+            gr.Markdown("**Conv layers visualised in 3D.** Each block learns progressively deeper features.")
+            ivb=gr.Button("🔄 Refresh",variant="primary")
+            ivo=gr.HTML('<div style="height:650px;background:#030610;border-radius:12px;display:flex;align-items:center;justify-content:center;color:#00f5c4;font-family:monospace;font-size:14px">Click 🔄 Refresh to load CNN visualization</div>')
+            ivb.click(get_img_viz,outputs=ivo)
+
+        # CHAT
+        with gr.Tab("💬 Chat with my AI"):
+            gr.Markdown(
+                "### Talk to your AI — it answers from what it has actually learned\n"
+                "The more articles it collects, the smarter the answers.\n"
+                "Type `stats` to see what it knows. Type `help` for tips."
+            )
+            chatbox = gr.Chatbot(label="", height=480)
+            with gr.Row():
+                msg_in = gr.Textbox(label="", placeholder="Ask anything — e.g. 'What's happening in AI?'", scale=5)
+                send_b = gr.Button("Send", variant="primary", scale=1)
+            send_b.click(chat_fn, inputs=[msg_in, chatbox], outputs=[chatbox, msg_in])
+            msg_in.submit(chat_fn, inputs=[msg_in, chatbox], outputs=[chatbox, msg_in])
+            gr.Examples(
+                [["What's happening in AI and technology?"],
+                 ["Tell me about recent science discoveries"],
+                 ["What's in the news about sports?"],
+                 ["stats"],["help"]],
+                inputs=msg_in
+            )
+
+        # KNOWLEDGE
+        with gr.Tab("📚 Knowledge Base"):
+            gr.Markdown("Everything the AI has read — saved to `knowledge.jsonl` instantly")
+            kmd=gr.Markdown(value=get_knowledge())
+
+        # DATA FEED
+        with gr.Tab("📰 Data Feed"):
+            fmd=gr.Markdown(value=get_feed())
+
+        # PREDICT TEXT
+        with gr.Tab("🔍 Classify Text"):
+            gr.Markdown("Test your trained text model")
+            pt=gr.Textbox(label="Text",lines=3,placeholder="Scientists discover...")
+            pb=gr.Button("Classify",variant="primary")
+            po=gr.Markdown()
+            pb.click(predict_text,inputs=pt,outputs=po)
+
+        # PREDICT IMAGE
+        with gr.Tab("🖼️ Classify Image"):
+            gr.Markdown("Test your trained image CNN")
+            pi=gr.Image(label="Upload image",type="filepath")
+            pib=gr.Button("Classify",variant="primary")
+            pio=gr.Markdown()
+            pib.click(predict_img,inputs=pi,outputs=pio)
+
+    # timers
+    t3=gr.Timer(3); t5=gr.Timer(5); t8=gr.Timer(8); t12=gr.Timer(12)
+    t3.tick(status_bar,outputs=sbar)
+    t3.tick(get_log,outputs=lbox)
+    t5.tick(text_stats,outputs=tmd)
+    t5.tick(img_stats,outputs=imd)
+    t8.tick(get_knowledge,outputs=kmd)
+    t8.tick(get_feed,outputs=fmd)
+    t12.tick(get_text_viz,outputs=tvo)
+    t12.tick(get_img_viz,outputs=ivo)
+
+# ── AUTO-START: both networks begin training immediately on startup ──────────
+def _auto_start():
+    import time as _t
+    _t.sleep(3)  # give Gradio time to finish starting
+    try:
+        start_text()
+        log("🤖 Auto-started TEXT network")
+    except Exception as e:
+        log(f"⚠ Auto-start text error: {e}")
+    try:
+        start_img()
+        log("🤖 Auto-started IMAGE network")
+    except Exception as e:
+        log(f"⚠ Auto-start image error: {e}")
+
+threading.Thread(target=_auto_start, daemon=True).start()
+
+if __name__=="__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7860, share=True, ssr_mode=False)

chatbot.py

@@ -0,0 +1,160 @@
+import json, re, math, os, random
+from collections import Counter, defaultdict
+
+KNOWLEDGE_FILE = 'knowledge.jsonl'
+STOPWORDS = {
+    'a','an','the','is','it','in','on','at','to','for','of','and','or','but',
+    'was','are','were','be','been','have','has','had','do','does','did','will',
+    'would','could','should','may','might','that','this','these','those','with',
+    'from','by','as','not','also','than','then','so','if','when','what','how',
+    'who','which','its','their','our','your','my','his','her','we','they','he',
+    'she','you','i','me','him','us','them','said','says',
+}
+
+def tokenize(text):
+    text = re.sub(r'[^\w\s]', ' ', text.lower())
+    return [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
+
+class KnowledgeBase:
+    def __init__(self):
+        self.docs = []; self.doc_tokens = []; self.idf = {}; self.last_count = 0
+        self._load()
+
+    def _load(self):
+        if not os.path.exists(KNOWLEDGE_FILE): return
+        new_docs = []
+        try:
+            with open(KNOWLEDGE_FILE, 'r', encoding='utf-8') as f:
+                for line in f:
+                    line = line.strip()
+                    if line:
+                        try: new_docs.append(json.loads(line))
+                        except: pass
+        except: return
+        if len(new_docs) == self.last_count: return
+        self.docs = new_docs; self.last_count = len(new_docs)
+        self.doc_tokens = [tokenize(d.get('text','')) for d in self.docs]
+        N = len(self.docs)
+        df = defaultdict(int)
+        for tokens in self.doc_tokens:
+            for w in set(tokens): df[w] += 1
+        self.idf = {w: math.log((N+1)/(c+1))+1 for w,c in df.items()}
+
+    def search(self, query, top_k=5):
+        self._load()
+        if not self.docs: return []
+        q = tokenize(query)
+        if not q: return []
+        def score(dt):
+            tf = Counter(dt); n = len(dt)
+            return sum((tf[w]/n)*self.idf.get(w,0) for w in q if w in tf)
+        ranked = sorted(range(len(self.docs)), key=lambda i: -score(self.doc_tokens[i]))
+        return [(self.docs[i], score(self.doc_tokens[i])) for i in ranked[:top_k] if score(self.doc_tokens[i]) > 0]
+
+    def get_stats(self):
+        self._load()
+        return {
+            'total':      len(self.docs),
+            'categories': dict(Counter(d.get('category','other') for d in self.docs)),
+            'sources':    dict(Counter(d.get('source','unknown') for d in self.docs)),
+        }
+
+class RAGChatbot:
+    def __init__(self):
+        self.kb = KnowledgeBase()
+
+    def _build_answer(self, query, results, stats):
+        total = stats['total']
+        if total == 0:
+            return ("My brain is empty right now 🧠 — I haven't read any articles yet.\n\n"
+                    "Go to **Control → ▶ Start Text** to feed me internet data!")
+
+        if not results:
+            return (f"I've read **{total} articles** so far, but I haven't learned anything about that topic yet.\n\n"
+                    f"My current knowledge covers: **{', '.join(list(stats['categories'].keys())[:5])}**.\n\n"
+                    "Keep training me and I'll learn more over time! 📚")
+
+        # Build answer from best matching sentences
+        q_tokens = set(tokenize(query))
+        seen, parts = set(), []
+
+        for doc, sc in results[:3]:
+            text = doc.get('text','')
+            cat  = doc.get('category','general').upper()
+            src  = doc.get('source','?').upper()
+            sents = [s.strip() for s in re.split(r'(?<=[.!?])\s+', text) if len(s.strip()) > 40]
+            scored = sorted(
+                [(len(q_tokens & set(tokenize(s))), s) for s in sents if s not in seen],
+                reverse=True
+            )[:2]
+            for _, s in scored: seen.add(s)
+            if scored:
+                snippet = ' '.join(s for _, s in scored if _>0) or scored[0][1]
+                parts.append(f"📰 **[{cat} / {src}]**\n{snippet}")
+            elif len(text) > 40:
+                parts.append(f"📰 **[{cat} / {src}]**\n{text[:250]}...")
+
+        if not parts:
+            return f"I found some related articles ({total} total) but couldn't extract a clear answer. Try rephrasing!"
+
+        intro = random.choice([
+            f"Based on what I've learned ({total} articles read):\n\n",
+            f"Here's what I know from my training data:\n\n",
+            f"From {total} articles I've read:\n\n",
+        ])
+        return intro + '\n\n'.join(parts)
+
+    def chat(self, user_message, history):
+        history = list(history or [])
+        msg = user_message.strip()
+        if not msg: return history
+
+        lower = msg.lower()
+        stats = self.kb.get_stats()
+
+        if lower in ('hi','hello','hey','sup','yo','hiya','howdy','helo'):
+            total = stats['total']
+            if total == 0:
+                bot_reply = ("Hey! 👋 I'm your AI — but I haven't learned anything yet.\n\n"
+                             "Go to **Control → ▶ Start Text** to feed me internet articles!")
+            else:
+                bot_reply = (f"Hey! 👋 I'm your AI — I've read **{total} articles** so far!\n\n"
+                             f"Ask me anything about what I've learned. My strongest topics: "
+                             f"**{', '.join(list(stats['categories'].keys())[:4])}**.")
+
+        elif any(p in lower for p in ('how are you','how r you',"what's up",'whats up')):
+            bot_reply = "I'm learning and growing! 🤖 The more articles I read, the smarter I get. Ask me something!"
+
+        elif any(p in lower for p in ('who are you','what are you','what can you do','tell me about yourself')):
+            bot_reply = (
+                "I'm a **Living Neural Network** — an AI that learns from real internet articles in real time! 🧠\n\n"
+                f"I've read **{stats['total']} articles** so far covering: "
+                f"{', '.join(list(stats['categories'].keys())[:5]) or 'nothing yet'}.\n\n"
+                "I only know what I've actually been trained on — no pre-built knowledge. "
+                "The more you train me, the smarter I become. Like a brain growing from scratch!"
+            )
+
+        elif lower in ('stats','status','what do you know','knowledge'):
+            lines = '\n'.join(f"  • **{k}**: {v}" for k,v in sorted(stats['categories'].items(), key=lambda x: -x[1]))
+            srcs  = ', '.join(f"{k}({v})" for k,v in stats['sources'].items())
+            bot_reply = (f"## 🧠 My Knowledge\n\n**Total articles:** {stats['total']}\n\n"
+                         f"**By topic:**\n{lines or '  • nothing yet'}\n\n"
+                         f"**Sources:** {srcs or 'none yet'}")
+
+        elif lower in ('help','?','commands'):
+            bot_reply = ("## 💡 How to use me\n\n"
+                         "Just ask me anything — I'll search what I've learned!\n\n"
+                         "**Try:** *What's happening in AI?* · *Tell me about climate change* · *Who is Elon Musk?*\n\n"
+                         "**Commands:** `stats` · `help`\n\n"
+                         "⚡ The more I train, the more I know!")
+
+        elif any(p in lower for p in ('thanks','thank you','thx','ty')):
+            bot_reply = "You're welcome! 😊 Keep training me!"
+
+        else:
+            results   = self.kb.search(msg, top_k=5)
+            bot_reply = self._build_answer(msg, results, stats)
+
+        history.append({"role": "user",      "content": user_message})
+        history.append({"role": "assistant", "content": bot_reply})
+        return history

data_fetcher.py

@@ -0,0 +1,317 @@
+"""
+data_fetcher.py — Fetches real live data from the internet.
+Sources: RSS, Reddit, Wikipedia, HackerNews, DuckDuckGo Search,
+         DEV.to, GitHub Trending, Google News RSS, Al Jazeera, Guardian, NPR, arXiv
+"""
+
+import requests, feedparser, random, re, time, json
+from datetime import datetime
+from collections import deque
+from urllib.parse import quote_plus
+
+HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (compatible; LivingNeuralNetwork/2.0; educational)',
+    'Accept-Language': 'en-US,en;q=0.9',
+}
+
+# ─── RSS FEEDS ────────────────────────────────────────────────────────────────
+RSS_FEEDS = {
+    'technology': [
+        'https://feeds.feedburner.com/TechCrunch',
+        'https://feeds.arstechnica.com/arstechnica/index',
+        'https://www.wired.com/feed/rss',
+        'https://hnrss.org/frontpage',
+        'https://www.theverge.com/rss/index.xml',
+        'https://feeds.feedburner.com/venturebeat/SZYF',
+        'https://dev.to/feed',
+        'https://thenextweb.com/feed/',
+    ],
+    'science': [
+        'https://www.sciencedaily.com/rss/all.xml',
+        'https://www.newscientist.com/feed/home/',
+        'https://rss.nytimes.com/services/xml/rss/nyt/Science.xml',
+        'https://www.nature.com/nature.rss',
+        'http://export.arxiv.org/rss/cs.AI',
+        'http://export.arxiv.org/rss/cs.LG',
+        'https://phys.org/rss-feed/breaking/',
+    ],
+    'world': [
+        'https://feeds.bbci.co.uk/news/world/rss.xml',
+        'https://rss.nytimes.com/services/xml/rss/nyt/World.xml',
+        'https://www.aljazeera.com/xml/rss/all.xml',
+        'https://feeds.npr.org/1004/rss.xml',
+        'https://www.theguardian.com/world/rss',
+        'https://feeds.reuters.com/reuters/worldNews',
+    ],
+    'sports': [
+        'https://feeds.bbci.co.uk/sport/rss.xml',
+        'https://rss.nytimes.com/services/xml/rss/nyt/Sports.xml',
+        'https://www.espn.com/espn/rss/news',
+    ],
+    'business': [
+        'https://feeds.bbci.co.uk/news/business/rss.xml',
+        'https://rss.nytimes.com/services/xml/rss/nyt/Business.xml',
+        'https://feeds.bloomberg.com/markets/news.rss',
+        'https://www.theguardian.com/business/rss',
+        'https://feeds.reuters.com/reuters/businessNews',
+    ],
+    'health': [
+        'https://feeds.bbci.co.uk/news/health/rss.xml',
+        'https://rss.nytimes.com/services/xml/rss/nyt/Health.xml',
+        'https://www.theguardian.com/society/health/rss',
+        'https://feeds.npr.org/1128/rss.xml',
+    ],
+    'entertainment': [
+        'https://feeds.bbci.co.uk/news/entertainment_and_arts/rss.xml',
+        'https://rss.nytimes.com/services/xml/rss/nyt/Arts.xml',
+        'https://www.theguardian.com/culture/rss',
+        'https://variety.com/feed/',
+    ],
+    'ai': [
+        'http://export.arxiv.org/rss/cs.AI',
+        'http://export.arxiv.org/rss/cs.LG',
+        'https://hnrss.org/frontpage',
+        'https://feeds.feedburner.com/TechCrunch',
+    ],
+}
+
+REDDIT_FEEDS = [
+    ('technology',    'https://www.reddit.com/r/technology/top.json?limit=25&t=day'),
+    ('science',       'https://www.reddit.com/r/science/top.json?limit=25&t=day'),
+    ('world',         'https://www.reddit.com/r/worldnews/top.json?limit=25&t=day'),
+    ('sports',        'https://www.reddit.com/r/sports/top.json?limit=25&t=day'),
+    ('business',      'https://www.reddit.com/r/business/top.json?limit=25&t=day'),
+    ('health',        'https://www.reddit.com/r/Health/top.json?limit=25&t=day'),
+    ('entertainment', 'https://www.reddit.com/r/movies/top.json?limit=25&t=day'),
+    ('ai',            'https://www.reddit.com/r/MachineLearning/top.json?limit=25&t=day'),
+    ('ai',            'https://www.reddit.com/r/artificial/top.json?limit=25&t=day'),
+    ('science',       'https://www.reddit.com/r/askscience/top.json?limit=25&t=day'),
+    ('technology',    'https://www.reddit.com/r/programming/top.json?limit=25&t=day'),
+    ('world',         'https://www.reddit.com/r/geopolitics/top.json?limit=25&t=day'),
+]
+
+# Topics to actively search on DuckDuckGo
+DDG_SEARCH_TOPICS = [
+    ('technology', 'latest AI artificial intelligence news'),
+    ('technology', 'tech industry news today'),
+    ('science',    'science discovery research breakthrough'),
+    ('world',      'world news today'),
+    ('business',   'business economy markets news'),
+    ('health',     'health medicine research news'),
+    ('ai',         'machine learning deep learning news'),
+    ('ai',         'ChatGPT OpenAI Anthropic Google AI'),
+    ('world',      'politics international relations'),
+    ('science',    'space NASA astronomy discovery'),
+    ('technology', 'cybersecurity data breach'),
+    ('entertainment', 'movies music entertainment news'),
+]
+
+WIKIPEDIA_API = 'https://en.wikipedia.org/api/rest_v1/page/random/summary'
+WIKI_SEARCH   = 'https://en.wikipedia.org/api/rest_v1/page/summary/{}'
+HN_TOP_API    = 'https://hacker-news.firebaseio.com/v0/topstories.json'
+HN_ITEM_API   = 'https://hacker-news.firebaseio.com/v0/item/{}.json'
+
+# ─── UTILS ────────────────────────────────────────────────────────────────────
+def clean(text: str, max_chars: int = 700) -> str:
+    if not text: return ''
+    text = re.sub(r'<[^>]+>', ' ', text)
+    text = re.sub(r'http\S+', '', text)
+    text = re.sub(r'[^\w\s.,!?;:\'\-–—]', ' ', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text[:max_chars]
+
+def make_item(text, category, source, extra=None):
+    text = clean(text)
+    if len(text) < 30: return None
+    return {
+        'text':      text,
+        'category':  category,
+        'source':    source,
+        'timestamp': datetime.utcnow().isoformat(),
+        **(extra or {}),
+    }
+
+# ─── FETCHERS ─────────────────────────────────────────────────────────────────
+def fetch_rss(category: str, url: str) -> list:
+    items = []
+    try:
+        feed = feedparser.parse(url)
+        for entry in feed.entries[:15]:
+            title   = entry.get('title', '')
+            summary = entry.get('summary', entry.get('description', ''))
+            text    = f"{title}. {summary}"
+            item = make_item(text, category, 'rss', {'feed': url.split('/')[2]})
+            if item: items.append(item)
+    except Exception: pass
+    return items
+
+def fetch_reddit(category: str, url: str) -> list:
+    items = []
+    try:
+        r = requests.get(url, headers=HEADERS, timeout=10)
+        r.raise_for_status()
+        posts = r.json().get('data', {}).get('children', [])
+        for post in posts:
+            d = post.get('data', {})
+            title    = d.get('title', '')
+            selftext = d.get('selftext', '')
+            text = f"{title}. {selftext}"
+            item = make_item(text, category, 'reddit',
+                           {'subreddit': d.get('subreddit',''), 'score': d.get('score',0)})
+            if item: items.append(item)
+    except Exception: pass
+    return items
+
+def fetch_duckduckgo(query: str, category: str) -> list:
+    """Search DuckDuckGo and extract text snippets — no API key needed."""
+    items = []
+    try:
+        url = f"https://html.duckduckgo.com/html/?q={quote_plus(query)}"
+        r = requests.get(url, headers={**HEADERS, 'Accept': 'text/html'}, timeout=12)
+        # Extract result snippets
+        snippets = re.findall(r'class="result__snippet"[^>]*>(.*?)</[as]>', r.text, re.DOTALL)
+        titles   = re.findall(r'class="result__a"[^>]*>(.*?)</a>', r.text, re.DOTALL)
+        for i, snippet in enumerate(snippets[:8]):
+            title = titles[i] if i < len(titles) else query
+            title = re.sub(r'<[^>]+>', '', title).strip()
+            snippet = re.sub(r'<[^>]+>', '', snippet).strip()
+            text = f"{title}. {snippet}"
+            item = make_item(text, category, 'duckduckgo', {'query': query})
+            if item: items.append(item)
+    except Exception: pass
+    return items
+
+def fetch_wikipedia_random() -> dict | None:
+    try:
+        r = requests.get(WIKIPEDIA_API, headers=HEADERS, timeout=10)
+        r.raise_for_status()
+        data  = r.json()
+        title = data.get('title', '')
+        text  = f"{title}. {data.get('extract','')}"
+        return make_item(text, 'other', 'wikipedia', {'title': title})
+    except Exception: return None
+
+def fetch_wikipedia_topic(topic: str, category: str) -> dict | None:
+    """Fetch Wikipedia article for a specific topic."""
+    try:
+        url = WIKI_SEARCH.format(quote_plus(topic))
+        r = requests.get(url, headers=HEADERS, timeout=10)
+        r.raise_for_status()
+        data  = r.json()
+        title = data.get('title', '')
+        text  = f"{title}. {data.get('extract','')}"
+        return make_item(text, category, 'wikipedia', {'title': title})
+    except Exception: return None
+
+def fetch_hackernews(n: int = 8) -> list:
+    items = []
+    try:
+        r = requests.get(HN_TOP_API, headers=HEADERS, timeout=10)
+        r.raise_for_status()
+        ids    = r.json()[:50]
+        chosen = random.sample(ids, min(n, len(ids)))
+        for sid in chosen:
+            try:
+                sr    = requests.get(HN_ITEM_API.format(sid), headers=HEADERS, timeout=6)
+                story = sr.json()
+                title = story.get('title', '')
+                body  = story.get('text', '')
+                item  = make_item(f"{title}. {body}", 'technology', 'hackernews',
+                                 {'score': story.get('score', 0)})
+                if item: items.append(item)
+                time.sleep(0.08)
+            except Exception: continue
+    except Exception: pass
+    return items
+
+# ─── MAIN FETCHER ─────────────────────────────────────────────────────────────
+class DataFetcher:
+    def __init__(self):
+        self.total_fetched = 0
+        self.source_counts = {
+            'rss': 0, 'reddit': 0, 'wikipedia': 0,
+            'hackernews': 0, 'duckduckgo': 0
+        }
+        self.recent_items = deque(maxlen=100)
+        self.log          = deque(maxlen=300)
+        self._ddg_idx     = 0   # rotate through DDG search topics
+
+    def _log(self, msg: str):
+        ts    = datetime.utcnow().strftime('%H:%M:%S')
+        entry = f"[{ts}] {msg}"
+        self.log.appendleft(entry)
+        return entry
+
+    def fetch_round(self) -> list:
+        """Fetch one full round from ALL sources."""
+        all_items = []
+
+        # 1. RSS — pick 2 random categories
+        for _ in range(2):
+            try:
+                cat = random.choice(list(RSS_FEEDS.keys()))
+                url = random.choice(RSS_FEEDS[cat])
+                items = fetch_rss(cat, url)
+                all_items.extend(items)
+                self.source_counts['rss'] += len(items)
+                self._log(f"📰 RSS [{cat.upper()}] +{len(items)} ← {url.split('/')[2]}")
+            except Exception as e:
+                self._log(f"⚠ RSS error: {e}")
+
+        # 2. Reddit
+        try:
+            cat, url = random.choice(REDDIT_FEEDS)
+            items = fetch_reddit(cat, url)
+            all_items.extend(items)
+            self.source_counts['reddit'] += len(items)
+            sub = url.split('/r/')[1].split('/')[0]
+            self._log(f"🟠 Reddit [r/{sub}] +{len(items)}")
+        except Exception as e:
+            self._log(f"⚠ Reddit error: {e}")
+
+        # 3. DuckDuckGo search (rotate through topics)
+        try:
+            cat, query = DDG_SEARCH_TOPICS[self._ddg_idx % len(DDG_SEARCH_TOPICS)]
+            self._ddg_idx += 1
+            items = fetch_duckduckgo(query, cat)
+            all_items.extend(items)
+            self.source_counts['duckduckgo'] += len(items)
+            self._log(f"🦆 DuckDuckGo [{cat.upper()}] \"{query[:40]}\" +{len(items)}")
+        except Exception as e:
+            self._log(f"⚠ DDG error: {e}")
+
+        # 4. Wikipedia (random + topic)
+        try:
+            item = fetch_wikipedia_random()
+            if item:
+                all_items.append(item)
+                self.source_counts['wikipedia'] += 1
+                self._log(f"📖 Wikipedia (random): {item['text'][:50]}…")
+        except Exception as e:
+            self._log(f"⚠ Wikipedia error: {e}")
+
+        # 5. HackerNews (every other round)
+        if random.random() < 0.5:
+            try:
+                items = fetch_hackernews(6)
+                all_items.extend(items)
+                self.source_counts['hackernews'] += len(items)
+                self._log(f"💻 HackerNews +{len(items)}")
+            except Exception as e:
+                self._log(f"⚠ HN error: {e}")
+
+        for item in all_items:
+            self.recent_items.appendleft(item)
+        self.total_fetched += len(all_items)
+        self._log(f"✅ Round complete — {len(all_items)} new items | total: {self.total_fetched}")
+        return all_items
+
+    def get_stats(self) -> dict:
+        return {
+            'total_fetched': self.total_fetched,
+            'sources':       dict(self.source_counts),
+            'recent_log':    list(self.log)[:30],
+        }
+
+    def get_recent_items(self, n: int = 20) -> list:
+        return list(self.recent_items)[:n]

image_model.py

@@ -0,0 +1,363 @@
+"""
+image_model.py — A real CNN (Convolutional Neural Network) built from scratch in PyTorch.
+
+Architecture:
+  Input (3 x 64 x 64 image)
+      → Conv2d(3, 32, 3)  + BatchNorm + ReLU + MaxPool    →  32 x 31 x 31
+      → Conv2d(32, 64, 3) + BatchNorm + ReLU + MaxPool    →  64 x 14 x 14
+      → Conv2d(64,128, 3) + BatchNorm + ReLU + MaxPool    → 128 x  6 x  6
+      → Flatten → Linear(128*6*6, 512) → ReLU → Dropout
+      → Linear(512, 128) → ReLU
+      → Linear(128, 8 categories)
+
+This CNN learns to LOOK at images the same way your text network learns to READ text.
+"""
+
+import torch
+import threading
+_CNN_LOCK = threading.Lock()
+import numpy as np
+import os
+import json
+from datetime import datetime, UTC
+from pathlib import Path
+from collections import defaultdict
+
+try:
+    from PIL import Image
+    PIL_AVAILABLE = True
+except ImportError:
+    PIL_AVAILABLE = False
+
+IMG_SIZE   = 64       # resize all images to 64x64 (small = faster on CPU)
+CATEGORIES = [
+    'nature', 'technology', 'science', 'people',
+    'animals', 'food', 'sports', 'architecture'
+]
+
+CNN_CHECKPOINT  = 'cnn_checkpoint.pt'
+CNN_STATS_FILE  = 'cnn_stats.json'
+
+
+# ── IMAGE PREPROCESSING ───────────────────────────────────────────────────────
+def load_image_tensor(path: str, size: int = IMG_SIZE):
+    """Load image from disk → normalised float tensor (3, size, size)."""
+    if not PIL_AVAILABLE:
+        raise RuntimeError("Pillow not installed. Add 'Pillow' to requirements.txt")
+    img = Image.open(path).convert('RGB')
+    img = img.resize((size, size), Image.BILINEAR)
+    arr = np.array(img, dtype=np.float32) / 255.0
+    # Normalize with ImageNet mean/std (works well even for non-ImageNet data)
+    mean = np.array([0.485, 0.456, 0.406])
+    std  = np.array([0.229, 0.224, 0.225])
+    arr  = (arr - mean) / std
+    return torch.tensor(arr).permute(2, 0, 1)   # HWC → CHW
+
+
+# ── CNN MODEL ─────────────────────────────────────────────────────────────────
+class ImageCNN(nn.Module):
+    """
+    A real Convolutional Neural Network.
+    Learns to detect edges → shapes → textures → objects, layer by layer.
+    Each Conv2d layer is looking for patterns the previous layer found.
+    """
+
+    def __init__(self, num_classes: int = 8):
+        super().__init__()
+        self.num_classes = num_classes
+
+        # Convolutional feature extractor
+        self.features = nn.Sequential(
+            # Block 1 — learns basic edges and colours
+            nn.Conv2d(3, 32, kernel_size=3, padding=1),
+            nn.BatchNorm2d(32),
+            nn.ReLU(),
+            nn.Conv2d(32, 32, kernel_size=3, padding=1),
+            nn.BatchNorm2d(32),
+            nn.ReLU(),
+            nn.MaxPool2d(2, 2),        # 64→32
+            nn.Dropout2d(0.1),
+
+            # Block 2 — learns corners, curves, textures
+            nn.Conv2d(32, 64, kernel_size=3, padding=1),
+            nn.BatchNorm2d(64),
+            nn.ReLU(),
+            nn.Conv2d(64, 64, kernel_size=3, padding=1),
+            nn.BatchNorm2d(64),
+            nn.ReLU(),
+            nn.MaxPool2d(2, 2),        # 32→16
+            nn.Dropout2d(0.15),
+
+            # Block 3 — learns complex shapes and object parts
+            nn.Conv2d(64, 128, kernel_size=3, padding=1),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            nn.Conv2d(128, 128, kernel_size=3, padding=1),
+            nn.BatchNorm2d(128),
+            nn.ReLU(),
+            nn.MaxPool2d(2, 2),        # 16→8
+            nn.Dropout2d(0.2),
+        )
+
+        # Classifier head
+        self.classifier = nn.Sequential(
+            nn.Flatten(),
+            nn.Linear(128 * 8 * 8, 512),
+            nn.ReLU(),
+            nn.Dropout(0.4),
+            nn.Linear(512, 128),
+            nn.ReLU(),
+            nn.Linear(128, num_classes),
+        )
+
+        # Activation tracking for visualization
+        self._activations = {}
+        self._register_hooks()
+        self._initialize_weights()
+
+    def _initialize_weights(self):
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
+            elif isinstance(m, nn.BatchNorm2d):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+            elif isinstance(m, nn.Linear):
+                nn.init.xavier_normal_(m.weight)
+                nn.init.constant_(m.bias, 0)
+
+    def _register_hooks(self):
+        def make_hook(name):
+            def hook(module, inp, out):
+                if isinstance(out, torch.Tensor):
+                    v = out.detach().float()
+                    if v.dim() > 1:
+                        v = v.mean(0)
+                    if v.dim() > 1:
+                        v = v.mean(-1).mean(-1)  # spatial mean for conv layers
+                    self._activations[name] = v[:16].tolist()
+            return hook
+        for i, layer in enumerate(self.features):
+            layer.register_forward_hook(make_hook(f'conv_{i}'))
+        for i, layer in enumerate(self.classifier):
+            layer.register_forward_hook(make_hook(f'fc_{i}'))
+
+    def forward(self, x):
+        x = self.features(x)
+        return self.classifier(x)
+
+    def get_activations(self):
+        return dict(self._activations)
+
+    def get_feature_maps(self, x):
+        """Return intermediate feature maps for visualization."""
+        maps = {}
+        for i, layer in enumerate(self.features):
+            x = layer(x)
+            if isinstance(layer, nn.ReLU):
+                maps[f'relu_{i}'] = x.detach()
+        return maps
+
+
+# ── LIVING IMAGE NETWORK ──────────────────────────────────────────────────────
+class LivingImageNetwork:
+    """
+    Wraps the CNN with training loop, data loading, and stats.
+    Trains on images downloaded by ImageFetcher.
+    """
+
+    def __init__(self):
+        self.model      = ImageCNN(num_classes=len(CATEGORIES))
+        self.optimizer  = optim.Adam(self.model.parameters(), lr=0.001, weight_decay=1e-4)
+        self.scheduler  = optim.lr_scheduler.StepLR(self.optimizer, step_size=50, gamma=0.8)
+        self.criterion  = nn.CrossEntropyLoss()
+
+        self.epoch           = 0
+        self.total_images    = 0
+        self.loss_history    = []
+        self.acc_history     = []
+        self.category_counts = defaultdict(int)
+
+        self.stats = {
+            'epoch':        0,
+            'loss':         '—',
+            'accuracy':     '—',
+            'total_images': 0,
+            'lr':           0.001,
+            'last_image':   '(none yet)',
+            'status':       'idle',
+        }
+
+        self._load_checkpoint()
+
+    # ── DATA LOADING ──────────────────────────────────────────────────────────
+    def _load_batch(self, image_dir: Path, batch_size: int = 16):
+        """
+        Load a random batch of images from image_data/ folder.
+        Returns (tensor_batch, label_batch) or None if not enough images.
+        """
+        if not PIL_AVAILABLE:
+            return None
+
+        all_paths = []
+        for cat_idx, cat in enumerate(CATEGORIES):
+            cat_dir = image_dir / cat
+            if cat_dir.exists():
+                for p in cat_dir.iterdir():
+                    if p.suffix.lower() in ('.jpg', '.jpeg', '.png', '.webp'):
+                        all_paths.append((str(p), cat_idx))
+
+        if len(all_paths) < batch_size:
+            return None
+
+        import random
+        batch_paths = random.sample(all_paths, batch_size)
+        tensors, labels = [], []
+
+        for path, label in batch_paths:
+            try:
+                t = load_image_tensor(path)
+                tensors.append(t)
+                labels.append(label)
+                self.category_counts[CATEGORIES[label]] += 1
+            except Exception:
+                continue
+
+        if not tensors:
+            return None
+
+        return torch.stack(tensors), torch.tensor(labels, dtype=torch.long)
+
+    # ── TRAINING ──────────────────────────────────────────────────────────────
+    def train_step(self, image_dir: Path, batch_size: int = 16):
+        """One training step — load images, forward pass, backprop."""
+        batch = self._load_batch(image_dir, batch_size)
+        if batch is None:
+            return None
+
+        x, y = batch
+        x = x.float()  # ensure float32
+        with _CNN_LOCK:
+            self.model.train()
+            self.model.zero_grad(set_to_none=True)
+            logits = self.model(x)
+            loss   = self.criterion(logits, y)
+            loss.backward()
+            for p in self.model.parameters():
+                if p.grad is not None:
+                    p.grad.data.clamp_(-1.0, 1.0)
+            self.optimizer.step()
+            loss_val = loss.detach().item()
+            acc      = (logits.detach().argmax(1) == y).float().mean().item()
+
+        self.epoch       += 1
+        self.total_images += len(x)
+        self.scheduler.step()
+
+        self.loss_history.append(round(loss_val, 5))
+        self.acc_history.append(round(acc, 4))
+        if len(self.loss_history) > 500:
+            self.loss_history = self.loss_history[-500:]
+            self.acc_history  = self.acc_history[-500:]
+
+        last_path = batch[0]  # just the paths string
+        self.stats.update({
+            'epoch':        self.epoch,
+            'loss':         round(loss_val, 4),
+            'accuracy':     round(acc * 100, 1),
+            'total_images': self.total_images,
+            'lr':           round(self.optimizer.param_groups[0]['lr'], 7),
+        })
+
+        if self.epoch % 20 == 0:
+            self._save_checkpoint()
+        self._write_stats()
+        return loss_val
+
+    def train_n_steps(self, image_dir: Path, n: int = 20):
+        losses = []
+        for _ in range(n):
+            l = self.train_step(image_dir)
+            if l is not None:
+                losses.append(l)
+        return {
+            'steps':    len(losses),
+            'avg_loss': round(sum(losses)/len(losses), 5) if losses else None,
+        }
+
+    # ── INFERENCE ─────────────────────────────────────────────────────────────
+    def predict_image(self, image_path: str) -> dict:
+        """Predict category of a single image."""
+        if not PIL_AVAILABLE:
+            return {'error': 'Pillow not installed'}
+        try:
+            t = load_image_tensor(image_path).unsqueeze(0)
+            self.model.eval()
+            with torch.no_grad():
+                logits = self.model(t)
+                probs  = torch.softmax(logits, dim=1)[0].tolist()
+                pred   = int(logits.argmax(1).item())
+            return {
+                'prediction': CATEGORIES[pred],
+                'confidence': round(probs[pred] * 100, 1),
+                'all_probs':  {c: round(p*100, 2) for c, p in zip(CATEGORIES, probs)},
+            }
+        except Exception as e:
+            return {'error': str(e)}
+
+    # ── VIZ STATE ─────────────────────────────────────────────────────────────
+    def get_viz_state(self) -> dict:
+        self.model.eval()
+        with torch.no_grad():
+            dummy = torch.zeros(1, 3, IMG_SIZE, IMG_SIZE)
+            self.model(dummy)
+        return {
+            'layer_sizes':  [3, 32, 64, 128, 512, 128, len(CATEGORIES)],
+            'activations':  self.model.get_activations(),
+            'loss_history': self.loss_history[-100:],
+            'acc_history':  self.acc_history[-100:],
+            'stats':        self.stats,
+            'type':         'cnn',
+        }
+
+    # ── PERSISTENCE ───────────────────────────────────────────────────────────
+    def _save_checkpoint(self):
+        try:
+            torch.save({
+                'model':           self.model.state_dict(),
+                'optimizer':       self.optimizer.state_dict(),
+                'epoch':           self.epoch,
+                'total_images':    self.total_images,
+                'loss_history':    self.loss_history,
+                'acc_history':     self.acc_history,
+                'category_counts': dict(self.category_counts),
+            }, CNN_CHECKPOINT)
+        except Exception:
+            pass
+
+    def _load_checkpoint(self):
+        if not os.path.exists(CNN_CHECKPOINT):
+            return
+        try:
+            ck = torch.load(CNN_CHECKPOINT, map_location='cpu')
+            self.model.load_state_dict(ck['model'])
+            self.optimizer.load_state_dict(ck['optimizer'])
+            self.epoch           = ck.get('epoch', 0)
+            self.total_images    = ck.get('total_images', 0)
+            self.loss_history    = ck.get('loss_history', [])
+            self.acc_history     = ck.get('acc_history', [])
+            self.category_counts = defaultdict(int, ck.get('category_counts', {}))
+            if self.epoch > 0:
+                self.stats.update({
+                    'epoch':        self.epoch,
+                    'total_images': self.total_images,
+                })
+        except Exception:
+            pass
+
+    def _write_stats(self):
+        try:
+            with open(CNN_STATS_FILE, 'w') as f:
+                json.dump(self.stats, f, indent=2)
+        except Exception:
+            pass

neural_network.py

@@ -0,0 +1,439 @@
+"""
+neural_network.py — Real PyTorch neural network built from scratch.
+
+KEY CHANGES:
+  - Every item ingested is IMMEDIATELY written to knowledge.jsonl
+  - On startup, knowledge.jsonl is read back → data_buffer is restored
+  - Model checkpoint auto-saves every 30 training epochs
+  - training_stats.json written after every training step (human-readable)
+"""
+
+import torch
+import threading
+_TEXT_LOCK = threading.Lock()
+import numpy as np
+import os
+import json
+import re
+from datetime import datetime, UTC
+from collections import Counter, defaultdict
+
+# ─── FILES ON DISK ────────────────────────────────────────────────────────────
+KNOWLEDGE_FILE   = 'knowledge.jsonl'      # Every article/text the AI has seen
+CHECKPOINT_FILE  = 'model_checkpoint.pt'  # PyTorch weights + optimizer state
+STATS_FILE       = 'training_stats.json'  # Human-readable live stats
+
+# ─── CATEGORIES ───────────────────────────────────────────────────────────────
+CATEGORIES = ['technology', 'science', 'world', 'sports',
+              'business', 'health', 'entertainment', 'other']
+
+# ─── VOCABULARY ───────────────────────────────────────────────────────────────
+STOPWORDS = {
+    'a','an','the','is','it','in','on','at','to','for','of','and','or','but',
+    'was','are','were','be','been','have','has','had','do','does','did','will',
+    'would','could','should','may','might','that','this','these','those','with',
+    'from','by','as','not','also','than','then','so','if','when','what','how',
+    'who','which','its','their','our','your','my','his','her','we','they','he',
+    'she','you','i','me','him','us','them','said','says','new','one','two',
+}
+
+class Vocabulary:
+    def __init__(self, max_size=10000):
+        self.word2idx   = {'<PAD>': 0, '<UNK>': 1}
+        self.idx2word   = {0: '<PAD>', 1: '<UNK>'}
+        self.word_counts = Counter()
+        self.max_size   = max_size
+        self.is_built   = False
+
+    def update(self, text: str):
+        self.word_counts.update(self._tokenize(text))
+
+    def build(self):
+        top = self.word_counts.most_common(self.max_size - 2)
+        self.word2idx = {'<PAD>': 0, '<UNK>': 1}
+        self.idx2word = {0: '<PAD>', 1: '<UNK>'}
+        for i, (word, _) in enumerate(top):
+            idx = i + 2
+            self.word2idx[word] = idx
+            self.idx2word[idx]  = word
+        self.is_built = True
+
+    def encode(self, text: str, max_len: int = 64) -> list:
+        words = self._tokenize(text)[:max_len]
+        ids   = [self.word2idx.get(w, 1) for w in words]
+        ids  += [0] * (max_len - len(ids))
+        return ids
+
+    def _tokenize(self, text: str) -> list:
+        text = text.lower()
+        text = re.sub(r'[^\w\s]', ' ', text)
+        return [w for w in text.split() if w not in STOPWORDS and len(w) > 2]
+
+    def __len__(self):
+        return len(self.word2idx)
+
+
+# ─── MODEL ────────────────────────────────────────────────────────────────────
+class TextClassifier(nn.Module):
+    def __init__(self, vocab_size=10002, embed_dim=64,
+                 hidden=[256, 128, 64], num_classes=8):
+        super().__init__()
+        self.embed_dim   = embed_dim
+        self.hidden_dims = hidden
+        self.num_classes = num_classes
+
+        self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0)
+        nn.init.normal_(self.embedding.weight, 0, 0.1)
+
+        layers = []
+        in_dim = embed_dim
+        for h in hidden:
+            layers += [nn.Linear(in_dim, h), nn.LayerNorm(h),
+                       nn.ReLU(), nn.Dropout(0.25)]
+            in_dim = h
+        layers.append(nn.Linear(in_dim, num_classes))
+        self.net = nn.Sequential(*layers)
+
+        self._activations = {}
+        self._register_hooks()
+
+    def _register_hooks(self):
+        def make_hook(name):
+            def hook(module, inp, out):
+                if isinstance(out, torch.Tensor):
+                    v = out.detach().float()
+                    if v.dim() > 1:
+                        v = v.mean(0)
+                    self._activations[name] = v[:32].tolist()
+            return hook
+        for i, layer in enumerate(self.net):
+            layer.register_forward_hook(make_hook(f'net.{i}'))
+
+    def forward(self, x):
+        emb    = self.embedding(x)
+        mask   = (x != 0).float().unsqueeze(-1)
+        pooled = (emb * mask).sum(1) / mask.sum(1).clamp(min=1)
+        return self.net(pooled)
+
+    def get_activations(self) -> dict:
+        return dict(self._activations)
+
+    def get_weight_info(self) -> dict:
+        info = {}
+        for name, param in self.named_parameters():
+            if 'weight' in name and param.dim() == 2:
+                w    = param.detach().float().numpy()
+                r, c = min(w.shape[0], 16), min(w.shape[1], 16)
+                info[name] = {
+                    'shape':    list(w.shape),
+                    'mean_abs': float(np.mean(np.abs(w))),
+                    'std':      float(np.std(w)),
+                    'sample':   w[:r, :c].tolist(),
+                }
+        return info
+
+
+# ─── LIVING NETWORK ───────────────────────────────────────────────────────────
+class LivingNetwork:
+    """
+    The brain. Wraps the PyTorch model with:
+      - knowledge.jsonl  → persistent record of everything it has read
+      - model_checkpoint.pt → saved weights (restored on restart)
+      - training_stats.json → live stats readable by the UI
+    """
+
+    AUTO_SAVE_EVERY = 30   # Save checkpoint every N training epochs
+
+    def __init__(self):
+        self.vocab      = Vocabulary()
+        self.model      = TextClassifier()
+        self.optimizer  = optim.Adam(self.model.parameters(), lr=0.001, weight_decay=1e-5)
+        self.scheduler  = optim.lr_scheduler.ReduceLROnPlateau(
+            self.optimizer, mode='min', patience=20, factor=0.5, min_lr=1e-5)
+        self.criterion  = nn.CrossEntropyLoss()
+
+        self.epoch           = 0
+        self.total_samples   = 0
+        self.data_buffer     = []   # (text, label_int) — in-memory training pool
+        self.loss_history    = []
+        self.acc_history     = []
+        self.category_counts = defaultdict(int)
+        self.knowledge_count = 0    # Total articles ever ingested
+
+        self.stats = {
+            'epoch':         0,
+            'loss':          '—',
+            'accuracy':      '—',
+            'total_samples': 0,
+            'lr':            0.001,
+            'buffer_size':   0,
+            'knowledge_count': 0,
+            'last_text':     '(nothing yet)',
+            'vocab_size':    2,
+            'status':        'idle',
+        }
+
+        # Load checkpoint first, then restore knowledge buffer
+        self._load_checkpoint()
+        self._load_knowledge()
+
+    # ── KNOWLEDGE FILE ────────────────────────────────────────────────────────
+    def _write_knowledge(self, text: str, category: str, source: str = 'unknown'):
+        """Append one learned item to knowledge.jsonl immediately."""
+        record = {
+            'text':      text,
+            'category':  category,
+            'source':    source,
+            'timestamp': datetime.now(UTC).isoformat(),
+            'epoch_at_ingestion': self.epoch,
+        }
+        try:
+            with open(KNOWLEDGE_FILE, 'a', encoding='utf-8') as f:
+                f.write(json.dumps(record, ensure_ascii=False) + '\n')
+            self.knowledge_count += 1
+        except Exception:
+            pass
+
+    def _load_knowledge(self):
+        """On startup: read knowledge.jsonl and rebuild data_buffer + vocab."""
+        if not os.path.exists(KNOWLEDGE_FILE):
+            return
+        loaded = 0
+        try:
+            with open(KNOWLEDGE_FILE, 'r', encoding='utf-8') as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        rec   = json.loads(line)
+                        text  = rec.get('text', '')
+                        cat   = rec.get('category', 'other')
+                        label = CATEGORIES.index(cat) if cat in CATEGORIES else 7
+                        if len(text) > 20:
+                            self.data_buffer.append((text, label))
+                            self.vocab.update(text)
+                            self.category_counts[cat] += 1
+                            loaded += 1
+                    except Exception:
+                        continue
+            self.knowledge_count = loaded
+            if loaded > 0:
+                self.vocab.build()
+            # Trim buffer if huge
+            if len(self.data_buffer) > 5000:
+                self.data_buffer = self.data_buffer[-4000:]
+        except Exception:
+            pass
+        self.stats['knowledge_count'] = self.knowledge_count
+        self.stats['buffer_size']     = len(self.data_buffer)
+        self.stats['vocab_size']      = len(self.vocab)
+
+    def get_knowledge_file_stats(self) -> dict:
+        """Return stats about the knowledge file for the UI."""
+        if not os.path.exists(KNOWLEDGE_FILE):
+            return {'exists': False, 'lines': 0, 'size_kb': 0}
+        size = os.path.getsize(KNOWLEDGE_FILE)
+        lines = 0
+        try:
+            with open(KNOWLEDGE_FILE, 'r', encoding='utf-8') as f:
+                lines = sum(1 for l in f if l.strip())
+        except Exception:
+            pass
+        return {'exists': True, 'lines': lines, 'size_kb': round(size / 1024, 1)}
+
+    def get_recent_knowledge(self, n: int = 20) -> list:
+        """Return last N items from knowledge.jsonl for display."""
+        if not os.path.exists(KNOWLEDGE_FILE):
+            return []
+        lines = []
+        try:
+            with open(KNOWLEDGE_FILE, 'r', encoding='utf-8') as f:
+                all_lines = [l.strip() for l in f if l.strip()]
+            for line in reversed(all_lines[-n:]):
+                try:
+                    lines.append(json.loads(line))
+                except Exception:
+                    pass
+        except Exception:
+            pass
+        return lines
+
+    # ── INGEST ────────────────────────────────────────────────────────────────
+    def ingest(self, text: str, category: str, source: str = 'unknown'):
+        """
+        Add text to training buffer AND write to knowledge.jsonl immediately.
+        This is how the AI 'remembers' what it has learned.
+        """
+        cleaned = text.strip()
+        if len(cleaned) < 20:
+            return
+        label = CATEGORIES.index(category) if category in CATEGORIES else 7
+
+        # ① Write to disk first — never lose this
+        self._write_knowledge(cleaned, category, source)
+
+        # ② Add to in-memory training buffer
+        self.data_buffer.append((cleaned, label))
+        self.vocab.update(cleaned)
+        self.category_counts[category] += 1
+
+        # Rebuild vocab every 25 items
+        if len(self.data_buffer) % 25 == 0:
+            self.vocab.build()
+
+        # Keep buffer bounded (disk has the full history)
+        if len(self.data_buffer) > 5000:
+            self.data_buffer = self.data_buffer[-4000:]
+
+        self.stats.update({
+            'buffer_size':     len(self.data_buffer),
+            'vocab_size':      len(self.vocab),
+            'knowledge_count': self.knowledge_count,
+        })
+
+    # ── TRAINING ──────────────────────────────────────────────────────────────
+    def train_step(self, batch_size: int = 32) -> float | None:
+        if len(self.data_buffer) < batch_size or not self.vocab.is_built:
+            return None
+        with _TEXT_LOCK:
+            self.model.train()
+            idx   = np.random.choice(len(self.data_buffer), batch_size, replace=False)
+            batch = [self.data_buffer[i] for i in idx]
+            texts, labels = zip(*batch)
+
+            x = torch.tensor([self.vocab.encode(t) for t in texts], dtype=torch.long)
+            y = torch.tensor(list(labels), dtype=torch.long)
+
+            self.model.zero_grad(set_to_none=True)
+            logits = self.model(x)
+            loss   = self.criterion(logits, y)
+            loss.backward()
+            for p in self.model.parameters():
+                if p.grad is not None:
+                    p.grad.data.clamp_(-1.0, 1.0)
+            self.optimizer.step()
+            loss_val = loss.detach().item()
+            acc      = (logits.detach().argmax(1) == y).float().mean().item()
+
+        self.epoch        += 1
+        self.total_samples += batch_size
+        self.scheduler.step(loss_val)
+
+        self.loss_history.append(round(loss_val, 5))
+        self.acc_history.append(round(acc, 4))
+        if len(self.loss_history) > 500:
+            self.loss_history = self.loss_history[-500:]
+            self.acc_history  = self.acc_history[-500:]
+
+        self.stats.update({
+            'epoch':           self.epoch,
+            'loss':            round(loss_val, 4),
+            'accuracy':        round(acc * 100, 1),
+            'total_samples':   self.total_samples,
+            'lr':              round(self.optimizer.param_groups[0]['lr'], 7),
+            'buffer_size':     len(self.data_buffer),
+            'last_text':       texts[0][:120],
+            'vocab_size':      len(self.vocab),
+            'knowledge_count': self.knowledge_count,
+        })
+
+        # Auto-save checkpoint every N epochs
+        if self.epoch % self.AUTO_SAVE_EVERY == 0:
+            self.save_checkpoint()
+
+        # Always write stats file so UI can read without waiting
+        self._write_stats_file()
+
+        return loss_val
+
+    def train_n_steps(self, n: int = 50) -> dict:
+        losses = []
+        for _ in range(n):
+            l = self.train_step()
+            if l is not None:
+                losses.append(l)
+        return {
+            'steps':    len(losses),
+            'avg_loss': round(sum(losses) / len(losses), 5) if losses else None,
+        }
+
+    # ── INFERENCE ───────────────────────────────────────────���─────────────────
+    def predict(self, text: str) -> dict:
+        if not self.vocab.is_built or not text.strip():
+            return {'error': 'Model not ready — start the network and let it train first'}
+        self.model.eval()
+        with torch.no_grad():
+            x      = torch.tensor([self.vocab.encode(text)], dtype=torch.long)
+            logits = self.model(x)
+            probs  = torch.softmax(logits, dim=1)[0].tolist()
+            pred   = int(logits.argmax(1).item())
+        return {
+            'prediction': CATEGORIES[pred],
+            'confidence': round(probs[pred] * 100, 1),
+            'all_probs':  {c: round(p * 100, 2) for c, p in zip(CATEGORIES, probs)},
+        }
+
+    # ── VIZ STATE ─────────────────────────────────────────────────────────────
+    def get_viz_state(self) -> dict:
+        self.model.eval()
+        with torch.no_grad():
+            dummy = torch.zeros(1, 64, dtype=torch.long)
+            self.model(dummy)
+        return {
+            'layer_sizes':    [self.model.embed_dim] + self.model.hidden_dims + [self.model.num_classes],
+            'activations':    self.model.get_activations(),
+            'weights':        self.model.get_weight_info(),
+            'loss_history':   self.loss_history[-100:],
+            'acc_history':    self.acc_history[-100:],
+            'stats':          self.stats,
+            'category_counts':dict(self.category_counts),
+        }
+
+    # ── PERSISTENCE ───────────────────────────────────────────────────────────
+    def save_checkpoint(self):
+        try:
+            torch.save({
+                'model':           self.model.state_dict(),
+                'optimizer':       self.optimizer.state_dict(),
+                'epoch':           self.epoch,
+                'total_samples':   self.total_samples,
+                'loss_history':    self.loss_history,
+                'acc_history':     self.acc_history,
+                'vocab_word2idx':  self.vocab.word2idx,
+                'category_counts': dict(self.category_counts),
+                'stats':           self.stats,
+            }, CHECKPOINT_FILE)
+            return True
+        except Exception:
+            return False
+
+    def _load_checkpoint(self):
+        if not os.path.exists(CHECKPOINT_FILE):
+            return False
+        try:
+            ck = torch.load(CHECKPOINT_FILE, map_location='cpu')
+            self.model.load_state_dict(ck['model'])
+            self.optimizer.load_state_dict(ck['optimizer'])
+            self.epoch           = ck.get('epoch', 0)
+            self.total_samples   = ck.get('total_samples', 0)
+            self.loss_history    = ck.get('loss_history', [])
+            self.acc_history     = ck.get('acc_history', [])
+            self.category_counts = defaultdict(int, ck.get('category_counts', {}))
+            self.stats           = ck.get('stats', self.stats)
+            w2i = ck.get('vocab_word2idx', {})
+            if w2i:
+                self.vocab.word2idx  = w2i
+                self.vocab.idx2word  = {v: k for k, v in w2i.items()}
+                self.vocab.is_built  = len(w2i) > 2
+            return True
+        except Exception:
+            return False
+
+    def _write_stats_file(self):
+        """Write human-readable stats to training_stats.json for easy debugging."""
+        try:
+            with open(STATS_FILE, 'w') as f:
+                json.dump({**self.stats, 'loss_last10': self.loss_history[-10:]}, f, indent=2)
+        except Exception:
+            pass

requirements.txt

@@ -0,0 +1,5 @@
+torch>=2.5.0
+feedparser==6.0.11
+requests==2.31.0
+beautifulsoup4==4.12.3
+numpy>=1.26.4