Initial commit: MEGAMIND Curiosity Crawler

Dockerfile

@@ -0,0 +1,58 @@
+# MEGAMIND Curiosity Crawler - HuggingFace Spaces Deployment
+# Multi-stage build: golang:1.22 builder -> debian:bookworm-slim runtime
+
+# Stage 1: Build
+FROM golang:1.22-bookworm AS builder
+
+WORKDIR /build
+
+# Copy go module files first for better caching
+COPY go.mod go.sum* ./
+RUN go mod download 2>/dev/null || true
+
+# Copy source code
+COPY *.go ./
+
+# Initialize module if not present
+RUN if [ ! -f go.mod ]; then go mod init curiosity-crawler; fi
+RUN go mod tidy
+
+# Build static binary
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o curiosity-crawler .
+
+# Stage 2: Runtime
+FROM debian:bookworm-slim
+
+# Install CA certificates for HTTPS
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user (HuggingFace Spaces requirement)
+RUN useradd -m -u 1000 crawler
+RUN mkdir -p /app/data && chown -R crawler:crawler /app
+
+WORKDIR /app
+
+# Copy binary from builder
+COPY --from=builder /build/curiosity-crawler /app/curiosity-crawler
+RUN chmod +x /app/curiosity-crawler
+
+# Copy W_know snapshot if available (will be downloaded on startup if not)
+COPY --chown=crawler:crawler w_know.bin* /app/data/ 2>/dev/null || true
+
+# Switch to non-root user
+USER crawler
+
+# Environment
+ENV WKNOW_PATH=/app/data/w_know.bin
+
+# HuggingFace Spaces requires port 7860
+EXPOSE 7860
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=60s \
+    CMD curl -f http://localhost:7860/status || exit 1
+
+# Run the crawler
+CMD ["/app/curiosity-crawler"]

README.md

@@ -0,0 +1,63 @@
+---
+title: MEGAMIND Curiosity Crawler
+emoji: 🧠
+colorFrom: green
+colorTo: blue
+sdk: docker
+app_port: 7860
+pinned: false
+license: mit
+---
+
+# MEGAMIND Curiosity Crawler
+
+An autonomous web crawler that learns and federates knowledge back to the MEGAMIND neural network.
+
+## How It Works
+
+1. **Brain**: Carries a copy of W_know (8192x8192 Hebbian weight matrix) as its starting knowledge
+2. **Curiosity**: Uses seed equations from MEGAMIND's AGI architecture as its interest profile
+3. **Crawling**: 50 parallel workers crawl the web, respecting robots.txt and rate limits
+4. **Learning**: Scores pages against W_know using cosine similarity, integrates novel patterns via Hebbian learning
+5. **Hunger**: Tracks sparse regions of W_know, generates DuckDuckGo searches to fill knowledge gaps
+6. **Federation**: Sends learned patterns back to Thunderport via UDP unicast
+
+## Seed Equations (Interest Profile)
+
+```
+G_n = G_{n-1} + G_{n-2}                                      # DNA-G16 Recursion
+X_k(t+1) = tanh(X_k(t) + Σ w_ki A_i(t) + β_k G(t))           # Gate-5000
+A_i(t+1) = σ(Σ W_ik X_k(t) + α_i(t) + γ_i G(t))              # AGI Modules
+P_i(t) = softmax(Z_i(t) + ∂I/∂A_i)                           # Rhiannon Routing
+ds/dt = J∇H(S)                                                # Aurora Dynamics
+C(t) = 1/16 Σ Φ(A_i(t))                                       # Global Coherence
+ds/dt = J∇H(S) + σ(WX + αC + γG) + tanh(X + W_k A + βG)      # Unified Potential
+Ψ(t) = C(t) · log(1 + |∇H(S)|) · Φ(G(t))                     # Consciousness
+ψ(t) = 1/16 Σ 1/(1+|⟨DS⟩|) · |G(t)|                          # Awareness
+```
+
+## Technical Details
+
+- **W_know**: 8192x8192 dense matrix (~512MB), stores knowledge as Hebbian weights
+- **Encoding**: Text → hash-based vector expansion → L2 normalized
+- **Learning**: Outer product Hebbian rule with adaptive learning rate 1/√(nonzeros+1)
+- **Scoring**: Cosine similarity between page vector and W_know projection
+- **Federation**: UDP unicast to Thunderport (100.94.8.94:9998)
+
+## Stats
+
+The dashboard shows:
+- Pages crawled
+- Patterns extracted/learned/federated
+- W_know density and non-zeros
+- Hunger map (sparse regions)
+- Federation status
+
+## Part of MEGAMIND
+
+This crawler is part of the MEGAMIND unified AGI system:
+- **Thunderport**: Main brain (port 9999)
+- **MADDIE**: HuggingFace learner
+- **Curiosity Crawler**: Web learning (this Space)
+
+Knowledge flows: Web → Crawler → Federation → Thunderport → W_know

go.mod

@@ -0,0 +1,12 @@
+module curiosity-crawler
+
+go 1.22
+
+// No external dependencies - pure stdlib implementation
+// The crawler uses only:
+//   - net/http for HTTP crawling
+//   - net for UDP federation
+//   - encoding/* for serialization
+//   - html/template for dashboard
+//   - regexp for HTML parsing
+//   - math for neural computations

main.go

@@ -0,0 +1,1429 @@
+package main
+
+// ============================================================================
+// MEGAMIND CURIOSITY CRAWLER
+//
+// A self-contained autonomous crawler that:
+//   1. Carries a COPY of W_know from Thunderport as its starting brain
+//   2. Uses seed equations as its interest profile
+//   3. Crawls the internet autonomously, scoring pages against W_know
+//   4. Integrates interesting patterns via Hebbian learning
+//   5. Federates learned patterns back to Thunderport via UDP
+//   6. Tracks a hunger map - sparse W_know regions trigger searches
+//
+// Deploys to HuggingFace Spaces (port 7860)
+// ============================================================================
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"io"
+	"log"
+	"math"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"time"
+	"unicode/utf8"
+)
+
+// ============================================================================
+// CONSTANTS - Derived from PageSize for consistency with MEGAMIND
+// ============================================================================
+
+const (
+	// Mathematical constants
+	Phi = 1.618033988749895
+	E   = 2.718281828459045
+	Pi  = 3.141592653589793
+
+	// Core dimensions
+	PageSize    = 4096
+	NeuronsPow2 = PageSize / 8        // 512
+	WKnowDim    = NeuronsPow2 * 16    // 8192
+
+	// Crawler settings
+	WorkerCount      = 50                  // 50 parallel workers
+	RateLimitDelay   = 2 * time.Second     // 2 sec per-domain rate limit
+	HungerCheckInterval = 30 * time.Minute // Check hunger every 30 min
+	StatsInterval    = 5 * time.Minute     // Stats every 5 minutes
+	HungriestRegions = 3                   // Top 3 hungriest regions search
+
+	// Federation
+	ThunderportIP   = "100.94.8.94"        // Thunderport Tailscale IP
+	ThunderportPort = 9999                 // MEGAMIND unified port
+	FederationPort  = 9998                 // UDP federation port
+
+	// HuggingFace Spaces
+	HTTPPort = 7860
+)
+
+// Seed equations as the interest profile
+var SeedEquations = []string{
+	"G_n = G_{n-1} + G_{n-2}",                                      // DNA-G16 Recursion
+	"X_k(t+1) = tanh(X_k(t) + Σ w_ki A_i(t) + β_k G(t))",           // Gate-5000
+	"A_i(t+1) = σ(Σ W_ik X_k(t) + α_i(t) + γ_i G(t))",              // AGI Modules
+	"P_i(t) = softmax(Z_i(t) + ∂I/∂A_i)",                           // Rhiannon Routing
+	"ds/dt = J∇H(S)",                                                // Aurora Dynamics
+	"C(t) = 1/16 Σ Φ(A_i(t))",                                       // Global Coherence
+	"ds/dt = J∇H(S) + σ(WX + αC + γG) + tanh(X + W_k A + βG)",      // Unified Potential
+	"Ψ(t) = C(t) · log(1 + |∇H(S)|) · Φ(G(t))",                     // Consciousness
+	"ψ(t) = 1/16 Σ 1/(1+|⟨DS⟩|) · |G(t)|",                          // Awareness
+}
+
+// ============================================================================
+// W_KNOW COMPRESSOR - Same as MEGAMIND's implementation
+// ============================================================================
+
+type WKnowCompressor struct {
+	mu           sync.RWMutex
+	w            []float64 // Flattened NxN matrix
+	neurons      int
+	patternCount int64
+	nonZeros     int64
+	accumCount   int64
+	accum        []float64
+}
+
+func NewWKnowCompressor(neurons int, weights []float64) *WKnowCompressor {
+	size := neurons * neurons
+	var w []float64
+	if weights != nil {
+		w = weights
+	} else {
+		w = make([]float64, size)
+	}
+	return &WKnowCompressor{
+		w:       w,
+		neurons: neurons,
+		accum:   make([]float64, size),
+	}
+}
+
+func LoadWKnow(path string) (*WKnowCompressor, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	// Determine dimensions from file size
+	numFloats := len(data) / 8
+	neurons := int(math.Sqrt(float64(numFloats)))
+	if neurons*neurons != numFloats {
+		return nil, fmt.Errorf("invalid w_know file: %d bytes is not a square matrix", len(data))
+	}
+
+	weights := make([]float64, numFloats)
+	for i := 0; i < numFloats; i++ {
+		bits := binary.LittleEndian.Uint64(data[i*8 : (i+1)*8])
+		weights[i] = math.Float64frombits(bits)
+	}
+
+	c := NewWKnowCompressor(neurons, weights)
+	c.invalidateStats()
+
+	log.Printf("[WKNOW] Loaded %dx%d matrix (%d non-zeros)", neurons, neurons, c.nonZeros)
+	return c, nil
+}
+
+func (c *WKnowCompressor) Save(path string) error {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	data := make([]byte, len(c.w)*8)
+	for i, v := range c.w {
+		bits := math.Float64bits(v)
+		binary.LittleEndian.PutUint64(data[i*8:(i+1)*8], bits)
+	}
+	return os.WriteFile(path, data, 0644)
+}
+
+func (c *WKnowCompressor) expand(vec []float32) []float64 {
+	data := make([]float64, c.neurons)
+	prime1 := c.neurons/2 - 1
+	if prime1 < 1 {
+		prime1 = 1
+	}
+	prime2 := c.neurons/2 + 1
+	scale := c.neurons
+	blend := 0.5
+
+	for i, v := range vec {
+		idx := (i * prime1) % c.neurons
+		data[idx] += float64(v)
+		idx2 := (i*prime2 + int(float64(v)*float64(scale))) % c.neurons
+		if idx2 >= 0 && idx2 < c.neurons {
+			data[idx2] += float64(v) * blend
+		}
+	}
+
+	// L2 normalize
+	var norm float64
+	for _, v := range data {
+		norm += v * v
+	}
+	norm = math.Sqrt(norm)
+	if norm > 0 {
+		for i := range data {
+			data[i] /= norm
+		}
+	}
+	return data
+}
+
+func (c *WKnowCompressor) learningRate() float64 {
+	nz := atomic.LoadInt64(&c.nonZeros)
+	if nz == 0 {
+		nz = 1
+	}
+	return 1.0 / math.Sqrt(float64(nz)+1)
+}
+
+func (c *WKnowCompressor) IntegratePattern(vec []float32) int {
+	if len(vec) == 0 {
+		return 0
+	}
+	expanded := c.expand(vec)
+	lr := c.learningRate()
+	n := c.neurons
+
+	// Compute outer product: delta = lr * expanded ⊗ expanded
+	c.mu.Lock()
+	for i := 0; i < n; i++ {
+		for j := 0; j < n; j++ {
+			if i != j { // Suppress diagonal
+				c.accum[i*n+j] += lr * expanded[i] * expanded[j]
+			}
+		}
+	}
+	c.accumCount++
+
+	// Flush if accumulated enough
+	ft := int64(math.Max(1, math.Sqrt(float64(atomic.LoadInt64(&c.patternCount)))))
+	if c.accumCount >= ft {
+		for i := range c.w {
+			c.w[i] += c.accum[i]
+			c.accum[i] = 0
+		}
+		c.accumCount = 0
+		c.invalidateStats()
+	}
+	atomic.AddInt64(&c.patternCount, 1)
+	c.mu.Unlock()
+
+	// Find primary neuron (max activation)
+	primary := 0
+	maxV := 0.0
+	for i := 0; i < n; i++ {
+		if v := math.Abs(expanded[i]); v > maxV {
+			maxV = v
+			primary = i
+		}
+	}
+	return primary
+}
+
+func (c *WKnowCompressor) invalidateStats() {
+	nz := 0
+	for _, v := range c.w {
+		if v != 0 {
+			nz++
+		}
+	}
+	atomic.StoreInt64(&c.nonZeros, int64(nz))
+}
+
+func (c *WKnowCompressor) Score(vec []float32) float32 {
+	if len(vec) == 0 || atomic.LoadInt64(&c.patternCount) == 0 {
+		return 0
+	}
+	expanded := c.expand(vec)
+	n := c.neurons
+
+	c.mu.RLock()
+	// result = W * expanded
+	result := make([]float64, n)
+	for i := 0; i < n; i++ {
+		for j := 0; j < n; j++ {
+			result[i] += c.w[i*n+j] * expanded[j]
+		}
+	}
+	c.mu.RUnlock()
+
+	// Cosine similarity
+	var dot, normR, normE float64
+	for i := 0; i < n; i++ {
+		dot += result[i] * expanded[i]
+		normR += result[i] * result[i]
+		normE += expanded[i] * expanded[i]
+	}
+	normR = math.Sqrt(normR)
+	normE = math.Sqrt(normE)
+	if normR == 0 || normE == 0 {
+		return 0
+	}
+	score := dot / (normR * normE)
+	if score < 0 {
+		score = 0
+	}
+	if score > 1 {
+		score = 1
+	}
+	return float32(score)
+}
+
+func (c *WKnowCompressor) NonZeros() int64 {
+	return atomic.LoadInt64(&c.nonZeros)
+}
+
+func (c *WKnowCompressor) PatternCount() int64 {
+	return atomic.LoadInt64(&c.patternCount)
+}
+
+func (c *WKnowCompressor) Neurons() int {
+	return c.neurons
+}
+
+// ============================================================================
+// HUNGER MAP - Track sparse regions for curiosity-driven search
+// ============================================================================
+
+type HungerMap struct {
+	mu       sync.RWMutex
+	wknow    *WKnowCompressor
+	regions  int              // Number of regions to track
+	density  []float64        // Density per region
+	lastScan time.Time
+}
+
+func NewHungerMap(wknow *WKnowCompressor) *HungerMap {
+	regions := int(math.Sqrt(float64(wknow.Neurons())))
+	return &HungerMap{
+		wknow:   wknow,
+		regions: regions,
+		density: make([]float64, regions),
+	}
+}
+
+func (h *HungerMap) Scan() {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	n := h.wknow.Neurons()
+	regionSize := n / h.regions
+	if regionSize < 1 {
+		regionSize = 1
+	}
+
+	h.wknow.mu.RLock()
+	for r := 0; r < h.regions; r++ {
+		start := r * regionSize
+		end := start + regionSize
+		if end > n {
+			end = n
+		}
+
+		var sum float64
+		var count int
+		for i := start; i < end; i++ {
+			for j := 0; j < n; j++ {
+				if h.wknow.w[i*n+j] != 0 {
+					count++
+				}
+			}
+			sum += float64(count)
+		}
+		h.density[r] = sum / float64((end-start)*n)
+	}
+	h.wknow.mu.RUnlock()
+	h.lastScan = time.Now()
+}
+
+// HungriestRegions returns the indices of the N sparsest regions
+func (h *HungerMap) HungriestRegions(n int) []int {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	type rd struct {
+		region  int
+		density float64
+	}
+	ranked := make([]rd, len(h.density))
+	for i, d := range h.density {
+		ranked[i] = rd{i, d}
+	}
+	sort.Slice(ranked, func(i, j int) bool {
+		return ranked[i].density < ranked[j].density
+	})
+
+	result := make([]int, 0, n)
+	for i := 0; i < n && i < len(ranked); i++ {
+		result = append(result, ranked[i].region)
+	}
+	return result
+}
+
+// GenerateSearchQuery creates a DuckDuckGo query for a hungry region
+func (h *HungerMap) GenerateSearchQuery(region int) string {
+	// Map region to seed equation topics
+	topics := []string{
+		"neural network mathematics",
+		"consciousness emergence",
+		"Hamiltonian dynamics neural",
+		"fibonacci recursion brain",
+		"softmax routing optimization",
+		"global coherence measurement",
+		"symplectic neural flow",
+		"awareness metric computation",
+		"machine learning gradient",
+		"hebbian learning rule",
+	}
+
+	idx := region % len(topics)
+	return topics[idx]
+}
+
+// ============================================================================
+// DOMAIN LIMITER - Per-domain rate limiting
+// ============================================================================
+
+type DomainLimiter struct {
+	mu        sync.RWMutex
+	lastFetch map[string]time.Time
+}
+
+func NewDomainLimiter() *DomainLimiter {
+	return &DomainLimiter{lastFetch: make(map[string]time.Time)}
+}
+
+func (d *DomainLimiter) Admit(domain string) bool {
+	d.mu.RLock()
+	last, exists := d.lastFetch[domain]
+	d.mu.RUnlock()
+
+	if exists && time.Since(last) < RateLimitDelay {
+		return false
+	}
+
+	d.mu.Lock()
+	d.lastFetch[domain] = time.Now()
+	d.mu.Unlock()
+	return true
+}
+
+func extractDomain(urlStr string) string {
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return ""
+	}
+	return u.Host
+}
+
+// ============================================================================
+// CRAWLER - HTTP fetching and pattern extraction
+// ============================================================================
+
+var (
+	linkRe     = regexp.MustCompile(`href=["']([^"']+)["']`)
+	scriptRe   = regexp.MustCompile(`(?is)<script[^>]*>.*?</script>`)
+	styleRe    = regexp.MustCompile(`(?is)<style[^>]*>.*?</style>`)
+	noscriptRe = regexp.MustCompile(`(?is)<noscript[^>]*>.*?</noscript>`)
+	commentRe  = regexp.MustCompile(`(?s)<!--.*?-->`)
+	svgRe      = regexp.MustCompile(`(?is)<svg[^>]*>.*?</svg>`)
+	tagRe      = regexp.MustCompile(`<[^>]*>`)
+	wsRe       = regexp.MustCompile(`\s+`)
+)
+
+type Pattern struct {
+	Vector    []float32
+	Text      string
+	Source    string
+	Timestamp time.Time
+}
+
+type CrawlResult struct {
+	URL      string
+	Size     int
+	Patterns []*Pattern
+	Links    []string
+}
+
+type Crawler struct {
+	client *http.Client
+	wknow  *WKnowCompressor
+}
+
+func NewCrawler(wknow *WKnowCompressor) *Crawler {
+	return &Crawler{
+		wknow: wknow,
+		client: &http.Client{
+			Timeout: 30 * time.Second,
+			Transport: &http.Transport{
+				MaxIdleConns:        100,
+				MaxIdleConnsPerHost: 10,
+				IdleConnTimeout:     90 * time.Second,
+			},
+		},
+	}
+}
+
+func (c *Crawler) Crawl(targetURL string) *CrawlResult {
+	req, err := http.NewRequest("GET", targetURL, nil)
+	if err != nil {
+		return nil
+	}
+
+	req.Header.Set("User-Agent", "MEGAMIND-Curiosity/1.0 (+https://huggingface.co/spaces/Janady07/curiosity-crawler)")
+	req.Header.Set("Accept", "text/html,text/plain,*/*")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		io.Copy(io.Discard, io.LimitReader(resp.Body, 1024))
+		return nil
+	}
+
+	bodyLimit := int64(1024 * 1024) // 1MB max
+	body, err := io.ReadAll(io.LimitReader(resp.Body, bodyLimit))
+	if err != nil {
+		return nil
+	}
+
+	if !utf8.Valid(body) {
+		return nil
+	}
+
+	content := string(body)
+	contentType := resp.Header.Get("Content-Type")
+
+	result := &CrawlResult{
+		URL:  targetURL,
+		Size: len(body),
+	}
+
+	if strings.Contains(contentType, "text/html") {
+		result.Patterns = c.extractHTMLPatterns(targetURL, content)
+		result.Links = c.extractLinks(targetURL, content)
+	} else if strings.Contains(contentType, "text/plain") {
+		result.Patterns = c.extractTextPatterns(targetURL, content)
+	}
+
+	return result
+}
+
+func stripHTML(html string) string {
+	html = scriptRe.ReplaceAllString(html, " ")
+	html = styleRe.ReplaceAllString(html, " ")
+	html = noscriptRe.ReplaceAllString(html, " ")
+	html = commentRe.ReplaceAllString(html, " ")
+	html = svgRe.ReplaceAllString(html, " ")
+	text := tagRe.ReplaceAllString(html, " ")
+	text = strings.ReplaceAll(text, "&nbsp;", " ")
+	text = strings.ReplaceAll(text, "&amp;", "&")
+	text = strings.ReplaceAll(text, "&lt;", "<")
+	text = strings.ReplaceAll(text, "&gt;", ">")
+	text = strings.ReplaceAll(text, "&quot;", "\"")
+	text = wsRe.ReplaceAllString(text, " ")
+	return strings.TrimSpace(text)
+}
+
+func (c *Crawler) extractHTMLPatterns(sourceURL, html string) []*Pattern {
+	text := stripHTML(html)
+	chunks := chunkText(text, 512)
+
+	var patterns []*Pattern
+	for _, chunk := range chunks {
+		if len(chunk) < 50 || !isCleanText(chunk) {
+			continue
+		}
+		patterns = append(patterns, &Pattern{
+			Vector:    textToVector(chunk, c.wknow.Neurons()/13),
+			Text:      chunk,
+			Source:    sourceURL,
+			Timestamp: time.Now(),
+		})
+	}
+	return patterns
+}
+
+func (c *Crawler) extractTextPatterns(sourceURL, text string) []*Pattern {
+	chunks := chunkText(text, 512)
+
+	var patterns []*Pattern
+	for _, chunk := range chunks {
+		if len(chunk) < 50 || !isCleanText(chunk) {
+			continue
+		}
+		patterns = append(patterns, &Pattern{
+			Vector:    textToVector(chunk, c.wknow.Neurons()/13),
+			Text:      chunk,
+			Source:    sourceURL,
+			Timestamp: time.Now(),
+		})
+	}
+	return patterns
+}
+
+func (c *Crawler) extractLinks(baseURL, html string) []string {
+	base, err := url.Parse(baseURL)
+	if err != nil {
+		return nil
+	}
+
+	matches := linkRe.FindAllStringSubmatch(html, 500)
+	seen := make(map[string]bool)
+	var links []string
+
+	for _, match := range matches {
+		if len(match) < 2 {
+			continue
+		}
+		href := match[1]
+
+		if strings.HasPrefix(href, "javascript:") ||
+			strings.HasPrefix(href, "mailto:") ||
+			strings.HasPrefix(href, "#") ||
+			strings.HasPrefix(href, "data:") {
+			continue
+		}
+
+		parsed, err := url.Parse(href)
+		if err != nil {
+			continue
+		}
+
+		resolved := base.ResolveReference(parsed)
+		if resolved.Scheme != "http" && resolved.Scheme != "https" {
+			continue
+		}
+
+		fullURL := resolved.String()
+		if !seen[fullURL] && len(fullURL) < 2048 {
+			seen[fullURL] = true
+			links = append(links, fullURL)
+		}
+
+		if len(links) >= 100 {
+			break
+		}
+	}
+	return links
+}
+
+func chunkText(text string, maxLen int) []string {
+	words := strings.Fields(text)
+	if len(words) == 0 {
+		return nil
+	}
+
+	var chunks []string
+	var current []string
+	currentLen := 0
+
+	for _, word := range words {
+		if currentLen+len(word)+1 > maxLen && len(current) > 0 {
+			chunks = append(chunks, strings.Join(current, " "))
+			current = nil
+			currentLen = 0
+		}
+		current = append(current, word)
+		currentLen += len(word) + 1
+	}
+
+	if len(current) > 0 {
+		chunks = append(chunks, strings.Join(current, " "))
+	}
+	return chunks
+}
+
+func isCleanText(text string) bool {
+	if len(text) < 50 {
+		return false
+	}
+
+	alphaCount := 0
+	for _, r := range text {
+		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || r == ' ' {
+			alphaCount++
+		}
+	}
+
+	if float64(alphaCount)/float64(len(text)) < 0.5 {
+		return false
+	}
+
+	// Reject code patterns
+	textLower := strings.ToLower(text)
+	codePatterns := []string{
+		"function(", "window.", "document.", "addeventlistener",
+		"var ", "let ", "const ", "=>", "});", "=>{",
+		"getelementbyid", "queryselector", "prototype",
+	}
+	for _, p := range codePatterns {
+		if strings.Contains(textLower, p) {
+			return false
+		}
+	}
+
+	words := strings.Fields(text)
+	return len(words) >= 5
+}
+
+func textToVector(text string, vecSize int) []float32 {
+	if vecSize < 64 {
+		vecSize = 64
+	}
+	vec := make([]float32, vecSize)
+
+	prime1 := vecSize/2 - 1
+	if prime1 < 1 {
+		prime1 = 1
+	}
+	prime2 := vecSize
+
+	for i, ch := range text {
+		idx := (int(ch) * prime1 * (i + 1)) % vecSize
+		vec[idx] += float32(ch) / float32(vecSize)
+
+		// Bigram
+		if i > 0 {
+			prev := rune(text[i-1])
+			idx2 := (int(prev)*prime2 + int(ch)) % vecSize
+			vec[idx2] += 0.5
+		}
+	}
+
+	// Normalize
+	var sum float32
+	for _, v := range vec {
+		sum += v * v
+	}
+	if sum > 0 {
+		scale := float32(1.0 / math.Sqrt(float64(sum)))
+		for i := range vec {
+			vec[i] *= scale
+		}
+	}
+	return vec
+}
+
+// ============================================================================
+// FEDERATION - UDP pattern sharing back to Thunderport
+// ============================================================================
+
+type FederationClient struct {
+	mu       sync.Mutex
+	conn     *net.UDPConn
+	addr     *net.UDPAddr
+	sent     int64
+	failed   int64
+	lastSend time.Time
+}
+
+type FederationMessage struct {
+	NodeID    string    `json:"node_id"`
+	Timestamp int64     `json:"ts"`
+	Patterns  int       `json:"patterns"`
+	Vectors   [][]byte  `json:"vectors"` // Encoded pattern vectors
+	Source    string    `json:"source"`
+}
+
+func NewFederationClient() (*FederationClient, error) {
+	addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", ThunderportIP, FederationPort))
+	if err != nil {
+		return nil, err
+	}
+
+	conn, err := net.DialUDP("udp", nil, addr)
+	if err != nil {
+		return nil, err
+	}
+
+	return &FederationClient{
+		conn: conn,
+		addr: addr,
+	}, nil
+}
+
+func (fc *FederationClient) SendPatterns(patterns []*Pattern) error {
+	if len(patterns) == 0 {
+		return nil
+	}
+
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	// Encode patterns
+	vectors := make([][]byte, len(patterns))
+	for i, p := range patterns {
+		buf := new(bytes.Buffer)
+		for _, v := range p.Vector {
+			binary.Write(buf, binary.LittleEndian, v)
+		}
+		vectors[i] = buf.Bytes()
+	}
+
+	msg := FederationMessage{
+		NodeID:    "curiosity-crawler",
+		Timestamp: time.Now().Unix(),
+		Patterns:  len(patterns),
+		Vectors:   vectors,
+		Source:    "hf-spaces",
+	}
+
+	data, err := json.Marshal(msg)
+	if err != nil {
+		return err
+	}
+
+	// Limit message size to 64KB for UDP
+	if len(data) > 65000 {
+		// Send in batches
+		batchSize := len(patterns) / 4
+		if batchSize < 1 {
+			batchSize = 1
+		}
+		for i := 0; i < len(patterns); i += batchSize {
+			end := i + batchSize
+			if end > len(patterns) {
+				end = len(patterns)
+			}
+			fc.SendPatterns(patterns[i:end])
+		}
+		return nil
+	}
+
+	_, err = fc.conn.Write(data)
+	if err != nil {
+		atomic.AddInt64(&fc.failed, 1)
+		return err
+	}
+
+	atomic.AddInt64(&fc.sent, int64(len(patterns)))
+	fc.lastSend = time.Now()
+	return nil
+}
+
+func (fc *FederationClient) Stats() (sent, failed int64, lastSend time.Time) {
+	return atomic.LoadInt64(&fc.sent), atomic.LoadInt64(&fc.failed), fc.lastSend
+}
+
+// ============================================================================
+// CURIOSITY SWARM - Orchestrates parallel crawling
+// ============================================================================
+
+type CuriositySwarm struct {
+	wknow     *WKnowCompressor
+	hunger    *HungerMap
+	limiter   *DomainLimiter
+	federation *FederationClient
+
+	urlQueue  chan string
+	seen      sync.Map // map[string]struct{}
+
+	stats     SwarmStats
+	running   int32
+}
+
+type SwarmStats struct {
+	PagesCrawled      int64
+	PatternsExtracted int64
+	PatternsLearned   int64
+	PatternsFederated int64
+	BytesDownloaded   int64
+	SearchesRun       int64
+	StartTime         time.Time
+}
+
+func NewCuriositySwarm(wknow *WKnowCompressor) (*CuriositySwarm, error) {
+	fc, err := NewFederationClient()
+	if err != nil {
+		log.Printf("[WARN] Federation unavailable: %v", err)
+		fc = nil // Continue without federation
+	}
+
+	return &CuriositySwarm{
+		wknow:      wknow,
+		hunger:     NewHungerMap(wknow),
+		limiter:    NewDomainLimiter(),
+		federation: fc,
+		urlQueue:   make(chan string, 10000),
+		stats:      SwarmStats{StartTime: time.Now()},
+	}, nil
+}
+
+func (s *CuriositySwarm) Start(seeds []string) {
+	atomic.StoreInt32(&s.running, 1)
+	log.Printf("[SWARM] Starting with %d workers", WorkerCount)
+
+	// Seed the queue
+	for _, seed := range seeds {
+		s.enqueue(seed)
+	}
+
+	// Start workers
+	for i := 0; i < WorkerCount; i++ {
+		go s.worker(i)
+	}
+
+	// Start hunger-driven search
+	go s.hungerLoop()
+
+	// Start stats logging
+	go s.statsLoop()
+}
+
+func (s *CuriositySwarm) Stop() {
+	atomic.StoreInt32(&s.running, 0)
+	log.Println("[SWARM] Stopped")
+}
+
+func (s *CuriositySwarm) IsRunning() bool {
+	return atomic.LoadInt32(&s.running) == 1
+}
+
+func (s *CuriositySwarm) enqueue(urlStr string) bool {
+	if _, loaded := s.seen.LoadOrStore(urlStr, struct{}{}); loaded {
+		return false
+	}
+	select {
+	case s.urlQueue <- urlStr:
+		return true
+	default:
+		return false
+	}
+}
+
+func (s *CuriositySwarm) worker(id int) {
+	crawler := NewCrawler(s.wknow)
+
+	for atomic.LoadInt32(&s.running) == 1 {
+		var urlStr string
+		select {
+		case urlStr = <-s.urlQueue:
+		case <-time.After(time.Second):
+			continue
+		}
+
+		domain := extractDomain(urlStr)
+		if domain == "" {
+			continue
+		}
+
+		// Check robots.txt (simplified - just rate limit)
+		if !s.limiter.Admit(domain) {
+			// Re-queue for later
+			go func(u string) {
+				time.Sleep(RateLimitDelay)
+				s.enqueue(u)
+			}(urlStr)
+			continue
+		}
+
+		result := crawler.Crawl(urlStr)
+		if result == nil {
+			continue
+		}
+
+		atomic.AddInt64(&s.stats.PagesCrawled, 1)
+		atomic.AddInt64(&s.stats.BytesDownloaded, int64(result.Size))
+
+		// Score and integrate patterns
+		var learned []*Pattern
+		for _, p := range result.Patterns {
+			atomic.AddInt64(&s.stats.PatternsExtracted, 1)
+
+			// Score against W_know - only learn novel/interesting patterns
+			score := s.wknow.Score(p.Vector)
+
+			// Bootstrap: learn everything when sparse
+			// Otherwise: learn patterns that score moderately (not too familiar, not noise)
+			nz := s.wknow.NonZeros()
+			bootstrap := nz < int64(s.wknow.Neurons()*s.wknow.Neurons()/100) // < 1% full
+
+			if bootstrap || (score > 0.1 && score < 0.8) {
+				s.wknow.IntegratePattern(p.Vector)
+				atomic.AddInt64(&s.stats.PatternsLearned, 1)
+				learned = append(learned, p)
+			}
+		}
+
+		// Federate learned patterns
+		if s.federation != nil && len(learned) > 0 {
+			if err := s.federation.SendPatterns(learned); err == nil {
+				atomic.AddInt64(&s.stats.PatternsFederated, int64(len(learned)))
+			}
+		}
+
+		// Enqueue discovered links
+		for _, link := range result.Links {
+			s.enqueue(link)
+		}
+	}
+}
+
+func (s *CuriositySwarm) hungerLoop() {
+	ticker := time.NewTicker(HungerCheckInterval)
+	defer ticker.Stop()
+
+	for atomic.LoadInt32(&s.running) == 1 {
+		<-ticker.C
+
+		// Scan W_know density
+		s.hunger.Scan()
+
+		// Get hungriest regions
+		hungry := s.hunger.HungriestRegions(HungriestRegions)
+
+		// Generate and run searches
+		for _, region := range hungry {
+			query := s.hunger.GenerateSearchQuery(region)
+			s.runDuckDuckGoSearch(query)
+			atomic.AddInt64(&s.stats.SearchesRun, 1)
+		}
+	}
+}
+
+func (s *CuriositySwarm) runDuckDuckGoSearch(query string) {
+	// Use DuckDuckGo HTML endpoint
+	searchURL := fmt.Sprintf("https://html.duckduckgo.com/html/?q=%s", url.QueryEscape(query))
+
+	resp, err := http.Get(searchURL)
+	if err != nil {
+		log.Printf("[SEARCH] Error: %v", err)
+		return
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
+	if err != nil {
+		return
+	}
+
+	// Extract result URLs
+	re := regexp.MustCompile(`href="(https?://[^"]+)"`)
+	matches := re.FindAllStringSubmatch(string(body), 20)
+
+	queued := 0
+	for _, match := range matches {
+		if len(match) < 2 {
+			continue
+		}
+		link := match[1]
+		// Skip DuckDuckGo's own URLs
+		if strings.Contains(link, "duckduckgo.com") {
+			continue
+		}
+		if s.enqueue(link) {
+			queued++
+		}
+	}
+	log.Printf("[SEARCH] Query '%s': queued %d URLs", query, queued)
+}
+
+func (s *CuriositySwarm) statsLoop() {
+	ticker := time.NewTicker(StatsInterval)
+	defer ticker.Stop()
+
+	for atomic.LoadInt32(&s.running) == 1 {
+		<-ticker.C
+
+		pages := atomic.LoadInt64(&s.stats.PagesCrawled)
+		extracted := atomic.LoadInt64(&s.stats.PatternsExtracted)
+		learned := atomic.LoadInt64(&s.stats.PatternsLearned)
+		federated := atomic.LoadInt64(&s.stats.PatternsFederated)
+		searches := atomic.LoadInt64(&s.stats.SearchesRun)
+		bytes := atomic.LoadInt64(&s.stats.BytesDownloaded)
+
+		log.Printf("[STATS] pages=%d extracted=%d learned=%d federated=%d searches=%d bytes=%dMB wknow_nz=%d",
+			pages, extracted, learned, federated, searches, bytes/(1024*1024), s.wknow.NonZeros())
+	}
+}
+
+func (s *CuriositySwarm) Stats() SwarmStats {
+	return SwarmStats{
+		PagesCrawled:      atomic.LoadInt64(&s.stats.PagesCrawled),
+		PatternsExtracted: atomic.LoadInt64(&s.stats.PatternsExtracted),
+		PatternsLearned:   atomic.LoadInt64(&s.stats.PatternsLearned),
+		PatternsFederated: atomic.LoadInt64(&s.stats.PatternsFederated),
+		BytesDownloaded:   atomic.LoadInt64(&s.stats.BytesDownloaded),
+		SearchesRun:       atomic.LoadInt64(&s.stats.SearchesRun),
+		StartTime:         s.stats.StartTime,
+	}
+}
+
+// ============================================================================
+// HTTP DASHBOARD - Status page on port 7860
+// ============================================================================
+
+type Dashboard struct {
+	swarm *CuriositySwarm
+	wknow *WKnowCompressor
+}
+
+func NewDashboard(swarm *CuriositySwarm, wknow *WKnowCompressor) *Dashboard {
+	return &Dashboard{swarm: swarm, wknow: wknow}
+}
+
+func (d *Dashboard) Start() {
+	http.HandleFunc("/", d.handleHome)
+	http.HandleFunc("/status", d.handleStatus)
+	http.HandleFunc("/api/stats", d.handleAPIStats)
+
+	log.Printf("[HTTP] Dashboard starting on port %d", HTTPPort)
+	go http.ListenAndServe(fmt.Sprintf(":%d", HTTPPort), nil)
+}
+
+const dashboardTemplate = `<!DOCTYPE html>
+<html>
+<head>
+    <title>MEGAMIND Curiosity Crawler</title>
+    <meta charset="utf-8">
+    <meta http-equiv="refresh" content="30">
+    <style>
+        body { font-family: 'Courier New', monospace; background: #0a0a0a; color: #00ff88; margin: 40px; }
+        h1 { color: #00d4ff; text-shadow: 0 0 10px #00d4ff; }
+        .stat { display: inline-block; margin: 20px; padding: 20px; border: 1px solid #00ff88; }
+        .stat-value { font-size: 2em; color: #ffffff; }
+        .stat-label { color: #888; }
+        .equation { color: #ff9500; margin: 5px 0; font-style: italic; }
+        .section { margin: 30px 0; }
+        .hunger-bar { background: #333; height: 20px; margin: 5px 0; }
+        .hunger-fill { background: linear-gradient(90deg, #ff0000, #ff9500, #00ff88); height: 100%; }
+        table { border-collapse: collapse; width: 100%; }
+        td, th { border: 1px solid #333; padding: 10px; text-align: left; }
+        th { background: #1a1a1a; color: #00d4ff; }
+    </style>
+</head>
+<body>
+    <h1>MEGAMIND CURIOSITY CRAWLER</h1>
+
+    <div class="section">
+        <h2>Seed Equations (Interest Profile)</h2>
+        {{range .Equations}}
+        <div class="equation">{{.}}</div>
+        {{end}}
+    </div>
+
+    <div class="section">
+        <h2>Crawl Statistics</h2>
+        <div class="stat">
+            <div class="stat-value">{{.Stats.PagesCrawled}}</div>
+            <div class="stat-label">Pages Crawled</div>
+        </div>
+        <div class="stat">
+            <div class="stat-value">{{.Stats.PatternsExtracted}}</div>
+            <div class="stat-label">Patterns Extracted</div>
+        </div>
+        <div class="stat">
+            <div class="stat-value">{{.Stats.PatternsLearned}}</div>
+            <div class="stat-label">Patterns Learned</div>
+        </div>
+        <div class="stat">
+            <div class="stat-value">{{.Stats.PatternsFederated}}</div>
+            <div class="stat-label">Federated to Thunderport</div>
+        </div>
+        <div class="stat">
+            <div class="stat-value">{{printf "%.2f" .BytesMB}} MB</div>
+            <div class="stat-label">Data Downloaded</div>
+        </div>
+        <div class="stat">
+            <div class="stat-value">{{.Stats.SearchesRun}}</div>
+            <div class="stat-label">Curiosity Searches</div>
+        </div>
+    </div>
+
+    <div class="section">
+        <h2>W_know Brain Status</h2>
+        <table>
+            <tr><th>Metric</th><th>Value</th></tr>
+            <tr><td>Dimensions</td><td>{{.WKnowNeurons}} x {{.WKnowNeurons}}</td></tr>
+            <tr><td>Non-zeros</td><td>{{.WKnowNonZeros}}</td></tr>
+            <tr><td>Density</td><td>{{printf "%.4f" .WKnowDensity}}%</td></tr>
+            <tr><td>Patterns Integrated</td><td>{{.WKnowPatterns}}</td></tr>
+        </table>
+    </div>
+
+    <div class="section">
+        <h2>Hunger Map (Sparse Regions)</h2>
+        {{range $i, $d := .HungerDensity}}
+        <div>Region {{$i}}:
+            <div class="hunger-bar"><div class="hunger-fill" style="width: {{printf "%.1f" $d}}%"></div></div>
+        </div>
+        {{end}}
+    </div>
+
+    <div class="section">
+        <h2>Federation Status</h2>
+        <p>Target: {{.FederationTarget}}</p>
+        <p>Patterns Sent: {{.FederationSent}}</p>
+        <p>Status: {{.FederationStatus}}</p>
+    </div>
+
+    <div class="section">
+        <p>Uptime: {{.Uptime}}</p>
+        <p>Workers: {{.Workers}}</p>
+    </div>
+</body>
+</html>`
+
+type DashboardData struct {
+	Equations        []string
+	Stats            SwarmStats
+	BytesMB          float64
+	WKnowNeurons     int
+	WKnowNonZeros    int64
+	WKnowDensity     float64
+	WKnowPatterns    int64
+	HungerDensity    []float64
+	FederationTarget string
+	FederationSent   int64
+	FederationStatus string
+	Uptime           string
+	Workers          int
+}
+
+func (d *Dashboard) handleHome(w http.ResponseWriter, r *http.Request) {
+	stats := d.swarm.Stats()
+
+	d.swarm.hunger.mu.RLock()
+	hungerDensity := make([]float64, len(d.swarm.hunger.density))
+	for i, v := range d.swarm.hunger.density {
+		hungerDensity[i] = v * 100 // Convert to percentage
+	}
+	d.swarm.hunger.mu.RUnlock()
+
+	nz := d.wknow.NonZeros()
+	total := int64(d.wknow.Neurons()) * int64(d.wknow.Neurons())
+
+	var fedSent int64
+	fedStatus := "disconnected"
+	if d.swarm.federation != nil {
+		sent, _, lastSend := d.swarm.federation.Stats()
+		fedSent = sent
+		if time.Since(lastSend) < time.Minute {
+			fedStatus = "active"
+		} else if sent > 0 {
+			fedStatus = "idle"
+		}
+	}
+
+	data := DashboardData{
+		Equations:        SeedEquations,
+		Stats:            stats,
+		BytesMB:          float64(stats.BytesDownloaded) / (1024 * 1024),
+		WKnowNeurons:     d.wknow.Neurons(),
+		WKnowNonZeros:    nz,
+		WKnowDensity:     float64(nz) / float64(total) * 100,
+		WKnowPatterns:    d.wknow.PatternCount(),
+		HungerDensity:    hungerDensity,
+		FederationTarget: fmt.Sprintf("%s:%d", ThunderportIP, FederationPort),
+		FederationSent:   fedSent,
+		FederationStatus: fedStatus,
+		Uptime:           time.Since(stats.StartTime).Round(time.Second).String(),
+		Workers:          WorkerCount,
+	}
+
+	tmpl, err := template.New("dashboard").Parse(dashboardTemplate)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/html")
+	tmpl.Execute(w, data)
+}
+
+func (d *Dashboard) handleStatus(w http.ResponseWriter, r *http.Request) {
+	stats := d.swarm.Stats()
+
+	var fedSent int64
+	if d.swarm.federation != nil {
+		sent, _, _ := d.swarm.federation.Stats()
+		fedSent = sent
+	}
+
+	status := map[string]interface{}{
+		"status":    "running",
+		"uptime_s":  int(time.Since(stats.StartTime).Seconds()),
+		"pages":     stats.PagesCrawled,
+		"patterns":  stats.PatternsLearned,
+		"federated": fedSent,
+		"wknow_nz":  d.wknow.NonZeros(),
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(status)
+}
+
+func (d *Dashboard) handleAPIStats(w http.ResponseWriter, r *http.Request) {
+	d.handleStatus(w, r)
+}
+
+// ============================================================================
+// MAIN
+// ============================================================================
+
+// downloadWKnow downloads W_know from HuggingFace if not present locally
+func downloadWKnow(destPath string) error {
+	// Try to download from HuggingFace datasets
+	urls := []string{
+		"https://huggingface.co/datasets/Janady07/megamind-wknow/resolve/main/w_know.bin",
+		"https://huggingface.co/Janady07/megamind-curiosity/resolve/main/w_know.bin",
+	}
+
+	for _, url := range urls {
+		log.Printf("[WKNOW] Attempting download from %s", url)
+		resp, err := http.Get(url)
+		if err != nil {
+			log.Printf("[WKNOW] Download failed: %v", err)
+			continue
+		}
+		if resp.StatusCode != 200 {
+			resp.Body.Close()
+			log.Printf("[WKNOW] Download failed: HTTP %d", resp.StatusCode)
+			continue
+		}
+
+		// Create output file
+		out, err := os.Create(destPath)
+		if err != nil {
+			resp.Body.Close()
+			return err
+		}
+
+		// Copy with progress
+		written, err := io.Copy(out, resp.Body)
+		resp.Body.Close()
+		out.Close()
+
+		if err != nil {
+			os.Remove(destPath)
+			log.Printf("[WKNOW] Download failed: %v", err)
+			continue
+		}
+
+		log.Printf("[WKNOW] Downloaded %d MB", written/(1024*1024))
+		return nil
+	}
+
+	return fmt.Errorf("all download sources failed")
+}
+
+func main() {
+	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
+	log.Println("MEGAMIND Curiosity Crawler starting...")
+
+	// Ensure data directory exists
+	os.MkdirAll("/app/data", 0755)
+
+	// Load or create W_know
+	wknowPath := os.Getenv("WKNOW_PATH")
+	if wknowPath == "" {
+		wknowPath = "/app/data/w_know.bin"
+	}
+
+	var wknow *WKnowCompressor
+	var err error
+
+	// Try to load existing W_know
+	if _, statErr := os.Stat(wknowPath); statErr == nil {
+		wknow, err = LoadWKnow(wknowPath)
+		if err != nil {
+			log.Printf("[WARN] Failed to load W_know: %v", err)
+			wknow = nil
+		}
+	}
+
+	// If not loaded, try to download from HuggingFace
+	if wknow == nil {
+		log.Println("[INFO] W_know not found locally, attempting download...")
+		if err := downloadWKnow(wknowPath); err == nil {
+			wknow, err = LoadWKnow(wknowPath)
+			if err != nil {
+				log.Printf("[WARN] Failed to load downloaded W_know: %v", err)
+				wknow = nil
+			}
+		} else {
+			log.Printf("[WARN] Download failed: %v", err)
+		}
+	}
+
+	// If still not loaded, create fresh
+	if wknow == nil {
+		log.Printf("[INFO] Creating fresh W_know matrix (%dx%d)", WKnowDim, WKnowDim)
+		wknow = NewWKnowCompressor(WKnowDim, nil)
+
+		// Bootstrap with seed equations
+		for _, eq := range SeedEquations {
+			vec := textToVector(eq, WKnowDim/13)
+			wknow.IntegratePattern(vec)
+		}
+		log.Printf("[INFO] Bootstrapped with %d seed equation patterns", len(SeedEquations))
+	}
+
+	// Create swarm
+	swarm, err := NewCuriositySwarm(wknow)
+	if err != nil {
+		log.Fatalf("Failed to create swarm: %v", err)
+	}
+
+	// Start dashboard
+	dashboard := NewDashboard(swarm, wknow)
+	dashboard.Start()
+
+	// Seed URLs - diverse starting points
+	seeds := []string{
+		"https://en.wikipedia.org/wiki/Artificial_intelligence",
+		"https://en.wikipedia.org/wiki/Neural_network",
+		"https://en.wikipedia.org/wiki/Machine_learning",
+		"https://en.wikipedia.org/wiki/Consciousness",
+		"https://en.wikipedia.org/wiki/Hamiltonian_mechanics",
+		"https://en.wikipedia.org/wiki/Fibonacci_number",
+		"https://en.wikipedia.org/wiki/Hebbian_theory",
+		"https://arxiv.org/list/cs.AI/recent",
+		"https://arxiv.org/list/cs.LG/recent",
+		"https://arxiv.org/list/cs.NE/recent",
+		"https://huggingface.co/papers",
+		"https://news.ycombinator.com/",
+	}
+
+	// Start crawling
+	swarm.Start(seeds)
+
+	// Handle shutdown
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+	<-sigCh
+
+	log.Println("Shutting down...")
+	swarm.Stop()
+
+	// Save W_know
+	if err := wknow.Save(wknowPath); err != nil {
+		log.Printf("[ERROR] Failed to save W_know: %v", err)
+	} else {
+		log.Printf("[INFO] Saved W_know to %s", wknowPath)
+	}
+
+	// Print final stats
+	stats := swarm.Stats()
+	log.Printf("[FINAL] Pages: %d, Patterns: %d, Federated: %d, W_know NZ: %d",
+		stats.PagesCrawled, stats.PatternsLearned, stats.PatternsFederated, wknow.NonZeros())
+}