File size: 24,653 Bytes

c617e7f

#!/usr/bin/env python3
"""
Flask Web GUI for the Self-Evolving Neural Network.
Provides real-time monitoring of evolutionary training with:
  - Live fitness charts (Chart.js)
  - Start/Stop/Reset controls
  - Genome architecture visualization
  - Evolution history browser
"""

import os
import sys
import json
import threading
import time
from pathlib import Path

import numpy as np
from flask import Flask, render_template_string, jsonify, request

# Import the evolution engine from our main module
from self_evolving_model import (
    EvolutionEngine, DataHandler, Genome, CHECKPOINT_DIR, log
)

# ═════════════════════════════════════════════════════════════════════
#  Global Application State
# ═════════════════════════════════════════════════════════════════════
app = Flask(__name__)

state = {
    "running": False,
    "generation": 0,
    "total_generations": 0,
    "history": [],
    "best_genome": None,
    "population_summary": [],
    "best_fitness": 0.0,
    "improvement": 0.0,
    "status_message": "Idle. Click Start to begin evolution.",
    "dataset_loaded": False,
    "dataset_info": None,
}

engine: EvolutionEngine = None
engine_thread: threading.Thread = None
X_global: np.ndarray = None
Y_global: np.ndarray = None

# ═════════════════════════════════════════════════════════════════════
#  Engine Callback
# ═════════════════════════════════════════════════════════════════════
def on_generation_complete(stats, best_genome):
    """Called by EvolutionEngine after each generation."""
    state["generation"] = stats["generation"] + 1
    state["history"].append(stats)
    state["best_fitness"] = stats["best_fitness"]
    if best_genome:
        state["best_genome"] = best_genome.to_dict()
    state["population_summary"] = [
        g.to_dict() for g in (engine.population if engine else [])
    ]
    if len(state["history"]) > 1 and state["history"][0]["best_fitness"] > 0:
        state["improvement"] = (
            (state["history"][-1]["best_fitness"] - state["history"][0]["best_fitness"])
            / state["history"][0]["best_fitness"] * 100
        )


def run_evolution_with_data(pop_size, generations, mutation_rate, train_epochs, n_samples):
    """Background thread: loads dataset (if needed) then runs evolution."""
    global engine, X_global, Y_global
    try:
        state["running"] = True
        state["total_generations"] = generations

        # Load dataset in background if not pre-loaded
        if X_global is None:
            state["status_message"] = "Loading FABLE.5 dataset..."
            try:
                X_global, Y_global = DataHandler.load_fable_dataset(n_samples=n_samples)
                state["dataset_loaded"] = True
                state["dataset_info"] = f"{X_global.shape[0]} samples, {X_global.shape[1]} features"
            except Exception as e:
                state["status_message"] = f"Dataset load failed: {e}. Using synthetic data."
                log(f"Dataset load failed: {e}. Falling back to synthetic.", "warn")
                X_global, Y_global = None, None

        state["status_message"] = f"Evolution in progress: 0/{generations} generations"

        engine = EvolutionEngine(
            pop_size=pop_size,
            generations=generations,
            mutation_rate=mutation_rate,
            train_epochs=train_epochs,
        )
        engine.on_generation_complete = on_generation_complete
        engine.run(X=X_global, Y=Y_global, reset=True)

        state["status_message"] = (
            f"Evolution complete! Best fitness: {state['best_fitness']:.2f} "
            f"(+{state['improvement']:.1f}%)"
        )
    except Exception as e:
        state["status_message"] = f"Error: {str(e)}"
        log(f"Evolution error: {e}", "error")
    finally:
        state["running"] = False


# ═════════════════════════════════════════════════════════════════════
#  HTML Template (Single-page app with embedded CSS/JS)
# ═════════════════════════════════════════════════════════════════════
HTML_TEMPLATE = r"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>🧬 Self-Evolving Neural Network</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
<style>
  :root {
    --bg: #0d1117; --surface: #161b22; --border: #30363d;
    --text: #e6edf3; --text-dim: #8b949e; --accent: #58a6ff;
    --green: #3fb950; --red: #f85149; --yellow: #d29922;
    --cyan: #39d353; --purple: #bc8cff;
  }
  * { margin:0; padding:0; box-sizing:border-box; }
  body {
    font-family: 'SF Mono', 'Fira Code', 'Cascadia Code', monospace;
    background: var(--bg); color: var(--text);
    min-height: 100vh; padding: 20px;
  }
  .header {
    text-align: center; padding: 20px 0 10px;
    border-bottom: 1px solid var(--border); margin-bottom: 20px;
  }
  .header h1 { font-size: 1.8em; color: var(--cyan); }
  .header p { color: var(--text-dim); font-size: 0.85em; margin-top: 5px; }
  .grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; max-width: 1400px; margin: 0 auto; }
  .card {
    background: var(--surface); border: 1px solid var(--border);
    border-radius: 8px; padding: 16px; position: relative;
  }
  .card h2 { font-size: 1em; color: var(--accent); margin-bottom: 12px; border-bottom: 1px solid var(--border); padding-bottom: 8px; }
  .card.full { grid-column: 1 / -1; }
  .stat-row { display: flex; justify-content: space-between; padding: 6px 0; border-bottom: 1px solid var(--border); }
  .stat-label { color: var(--text-dim); font-size: 0.85em; }
  .stat-value { font-weight: bold; font-size: 0.95em; }
  .stat-value.green { color: var(--green); }
  .stat-value.red { color: var(--red); }
  .stat-value.yellow { color: var(--yellow); }
  .stat-value.cyan { color: var(--cyan); }
  .stat-value.accent { color: var(--accent); }
  .controls { display: flex; gap: 10px; margin: 12px 0; flex-wrap: wrap; }
  .btn {
    padding: 10px 20px; border: 1px solid var(--border); border-radius: 6px;
    font-family: inherit; font-size: 0.85em; cursor: pointer;
    transition: all 0.2s; font-weight: 600;
  }
  .btn:hover { transform: translateY(-1px); box-shadow: 0 2px 8px rgba(0,0,0,0.3); }
  .btn-start { background: var(--green); color: #000; }
  .btn-start:hover { background: #2ea043; }
  .btn-stop { background: var(--red); color: #fff; }
  .btn-stop:hover { background: #da3633; }
  .btn:disabled { opacity: 0.4; cursor: not-allowed; transform: none; }
  .params { display: grid; grid-template-columns: 1fr 1fr; gap: 8px; margin: 10px 0; }
  .params label { font-size: 0.8em; color: var(--text-dim); }
  .params input {
    background: var(--bg); border: 1px solid var(--border); border-radius: 4px;
    color: var(--text); padding: 6px 8px; font-family: inherit; font-size: 0.85em; width: 100%;
  }
  .params input:focus { outline: none; border-color: var(--accent); }
  .status-bar {
    background: var(--bg); border: 1px solid var(--border); border-radius: 6px;
    padding: 10px 14px; margin: 10px 0; font-size: 0.85em;
    display: flex; align-items: center; gap: 8px;
  }
  .status-dot { width: 8px; height: 8px; border-radius: 50%; }
  .status-dot.running { background: var(--green); animation: pulse 1s infinite; }
  .status-dot.idle { background: var(--text-dim); }
  .status-dot.error { background: var(--red); }
  @keyframes pulse { 0%,100% { opacity:1; } 50% { opacity:0.4; } }
  .progress-bar {
    width: 100%; height: 6px; background: var(--border); border-radius: 3px; margin: 8px 0; overflow: hidden;
  }
  .progress-fill {
    height: 100%; background: var(--cyan); border-radius: 3px;
    transition: width 0.5s ease;
  }
  .genome-card {
    background: var(--bg); border: 1px solid var(--border); border-radius: 6px;
    padding: 10px; margin: 6px 0; font-size: 0.8em;
  }
  .genome-card .id { color: var(--purple); font-weight: bold; }
  .genome-card .fitness { color: var(--green); }
  .genome-layers {
    display: flex; gap: 4px; margin: 6px 0; flex-wrap: wrap; align-items: center;
  }
  .layer-chip {
    background: var(--surface); border: 1px solid var(--border); border-radius: 4px;
    padding: 3px 8px; font-size: 0.75em; color: var(--text-dim);
  }
  .layer-chip.active { border-color: var(--accent); color: var(--accent); }
  .arrow { color: var(--text-dim); font-size: 0.8em; }
  .chart-container { position: relative; height: 280px; }
  #log-area {
    background: var(--bg); border: 1px solid var(--border); border-radius: 6px;
    padding: 10px; max-height: 200px; overflow-y: auto; font-size: 0.75em;
    color: var(--text-dim); line-height: 1.6;
  }
  @media (max-width: 800px) { .grid { grid-template-columns: 1fr; } }
</style>
</head>
<body>

<div class="header">
  <h1>🧬 Self-Evolving Neural Network</h1>
  <p>Architecture evolves through mutation, crossover &amp; natural selection — trained on FABLE.5 traces</p>
</div>

<div class="grid">

  <!-- Status & Controls -->
  <div class="card">
    <h2>⚡ Controls</h2>
    <div class="status-bar">
      <div class="status-dot" id="statusDot"></div>
      <span id="statusText">Idle</span>
    </div>
    <div class="progress-bar">
      <div class="progress-fill" id="progressFill" style="width:0%"></div>
    </div>
    <div class="controls">
      <button class="btn btn-start" id="btnStart" onclick="startEvolution()">▶ Start Evolution</button>
      <button class="btn btn-stop" id="btnStop" onclick="stopEvolution()" disabled>■ Stop</button>
    </div>
    <div class="params">
      <div><label>Population Size</label><input type="number" id="popSize" value="6" min="2" max="50"></div>
      <div><label>Generations</label><input type="number" id="generations" value="15" min="1" max="500"></div>
      <div><label>Train Epochs</label><input type="number" id="trainEpochs" value="10" min="1" max="100"></div>
      <div><label>Mutation Rate</label><input type="number" id="mutationRate" value="0.3" min="0" max="1" step="0.05"></div>
      <div><label>Dataset Samples</label><input type="number" id="nSamples" value="5000" min="100" max="200000"></div>
    </div>
  </div>

  <!-- Statistics -->
  <div class="card">
    <h2>📊 Statistics</h2>
    <div class="stat-row"><span class="stat-label">Generation</span><span class="stat-value accent" id="statGen">0 / 0</span></div>
    <div class="stat-row"><span class="stat-label">Best Fitness</span><span class="stat-value green" id="statBestFit">—</span></div>
    <div class="stat-row"><span class="stat-label">Avg Fitness</span><span class="stat-value yellow" id="statAvgFit">—</span></div>
    <div class="stat-row"><span class="stat-label">Improvement</span><span class="stat-value cyan" id="statImprovement">—</span></div>
    <div class="stat-row"><span class="stat-label">Population</span><span class="stat-value" id="statPop">—</span></div>
    <div class="stat-row"><span class="stat-label">Best Genome ID</span><span class="stat-value purple" id="statBestId" style="color:var(--purple)">—</span></div>
    <div class="stat-row"><span class="stat-label">Dataset</span><span class="stat-value accent" id="statDataset">Not loaded</span></div>
  </div>

  <!-- Fitness Chart -->
  <div class="card full">
    <h2>📈 Fitness Over Generations</h2>
    <div class="chart-container">
      <canvas id="fitnessChart"></canvas>
    </div>
  </div>

  <!-- Best Genome -->
  <div class="card">
    <h2>🏆 Best Genome</h2>
    <div id="bestGenomeInfo">
      <div class="genome-card">
        <span style="color:var(--text-dim)">No evolution started yet</span>
      </div>
    </div>
  </div>

  <!-- Population -->
  <div class="card">
    <h2>🧬 Population</h2>
    <div id="populationInfo" style="max-height: 300px; overflow-y: auto;">
      <div style="color:var(--text-dim); font-size:0.85em;">Population will appear here during evolution...</div>
    </div>
  </div>

  <!-- Log -->
  <div class="card full">
    <h2>📋 Evolution Log</h2>
    <div id="log-area">Waiting for evolution to start...</div>
  </div>

</div>

<script>
// ── Chart Setup ────────────────────────────────────────────────────
const ctx = document.getElementById('fitnessChart').getContext('2d');
const fitnessChart = new Chart(ctx, {
  type: 'line',
  data: {
    labels: [],
    datasets: [
      {
        label: 'Best Fitness',
        data: [],
        borderColor: '#3fb950',
        backgroundColor: 'rgba(63,185,80,0.1)',
        fill: true, tension: 0.3, pointRadius: 4, borderWidth: 2,
      },
      {
        label: 'Avg Fitness',
        data: [],
        borderColor: '#d29922',
        backgroundColor: 'rgba(210,153,34,0.05)',
        fill: false, tension: 0.3, pointRadius: 3, borderWidth: 1.5,
        borderDash: [5, 3],
      },
    ],
  },
  options: {
    responsive: true, maintainAspectRatio: false,
    plugins: {
      legend: { labels: { color: '#8b949e', font: { family: 'monospace' } } },
    },
    scales: {
      x: { ticks: { color: '#8b949e' }, grid: { color: '#21262d' } },
      y: { ticks: { color: '#8b949e' }, grid: { color: '#21262d' }, beginAtZero: false },
    },
  },
});

// ── Polling ────────────────────────────────────────────────────────
let pollInterval = null;

function pollStatus() {
  fetch('/api/status')
    .then(r => r.json())
    .then(data => {
      // Status
      const dot = document.getElementById('statusDot');
      const txt = document.getElementById('statusText');
      if (data.running) {
        dot.className = 'status-dot running';
        document.getElementById('btnStart').disabled = true;
        document.getElementById('btnStop').disabled = false;
      } else {
        dot.className = data.status_message.includes('Error') ? 'status-dot error' : 'status-dot idle';
        document.getElementById('btnStart').disabled = false;
        document.getElementById('btnStop').disabled = true;
      }
      txt.textContent = data.status_message;

      // Progress
      const pct = data.total_generations > 0
        ? (data.generation / data.total_generations * 100) : 0;
      document.getElementById('progressFill').style.width = pct + '%';

      // Stats
      document.getElementById('statGen').textContent = `${data.generation} / ${data.total_generations}`;
      document.getElementById('statBestFit').textContent = data.best_fitness > 0 ? data.best_fitness.toFixed(4) : '—';
      document.getElementById('statImprovement').textContent = data.improvement !== 0 ? `${data.improvement > 0 ? '+' : ''}${data.improvement.toFixed(1)}%` : '—';
      document.getElementById('statPop').textContent = data.population_summary ? data.population_summary.length + ' genomes' : '—';
      document.getElementById('statBestId').textContent = data.best_genome ? data.best_genome.id : '—';

      // Average fitness from last history entry
      if (data.history && data.history.length > 0) {
        const last = data.history[data.history.length - 1];
        document.getElementById('statAvgFit').textContent = last.avg_fitness.toFixed(4);
      }

      // Dataset
      if (data.dataset_loaded) {
        document.getElementById('statDataset').textContent = data.dataset_info || 'Loaded';
      }

      // Chart update
      if (data.history && data.history.length > 0) {
        fitnessChart.data.labels = data.history.map((h, i) => `Gen ${i + 1}`);
        fitnessChart.data.datasets[0].data = data.history.map(h => h.best_fitness);
        fitnessChart.data.datasets[1].data = data.history.map(h => h.avg_fitness);
        fitnessChart.update('none');
      }

      // Best genome
      if (data.best_genome) {
        const g = data.best_genome;
        const cfg = g.config;
        let layersHtml = '';
        cfg.layers.forEach((l, i) => {
          layersHtml += `<span class="layer-chip active">${l.units} ${l.activation.slice(0,4)}</span>`;
          if (i < cfg.layers.length - 1) layersHtml += '<span class="arrow">→</span>';
        });
        document.getElementById('bestGenomeInfo').innerHTML = `
          <div class="genome-card">
            <span class="id">${g.id}</span> — <span class="fitness">Fitness: ${g.fitness.toFixed(4)}</span><br>
            <span style="color:var(--text-dim);font-size:0.8em">
              Layers: ${cfg.num_layers} | LR: ${cfg.learning_rate} | Opt: ${cfg.optimizer} | Born: Gen ${g.generation_born}
            </span>
            <div class="genome-layers">${layersHtml}</div>
            <div style="color:var(--accent);font-size:0.8em;margin-top:4px">
              ⚡ Gated: top-3 features [${cfg.top3_features ? cfg.top3_features.join(', ') : '0, 1, 2'}] → layer 1 | rest join layers 2+
            </div>
          </div>`;
      }

      // Population
      if (data.population_summary && data.population_summary.length > 0) {
        const sorted = [...data.population_summary].sort((a, b) => b.fitness - a.fitness);
        let popHtml = '';
        sorted.forEach((g, i) => {
          const cfg = g.config;
          const layersStr = cfg.layers.map(l => `${l.units}(${l.activation.slice(0,3)})`).join(', ');
          const medal = i === 0 ? '🥇' : i === 1 ? '🥈' : i === 2 ? '🥉' : '  ';
          popHtml += `<div class="genome-card" style="${i < 3 ? 'border-color:var(--accent)' : ''}">
            ${medal} <span class="id">${g.id}</span> — <span class="fitness">${g.fitness.toFixed(4)}</span>
            <span style="color:var(--text-dim);font-size:0.75em"> | ${cfg.num_layers}L [{${layersStr}}] ${cfg.optimizer} lr=${cfg.learning_rate}</span>
          </div>`;
        });
        document.getElementById('populationInfo').innerHTML = popHtml;
      }

      // Log
      if (data.history && data.history.length > 0) {
        const logDiv = document.getElementById('log-area');
        const last = data.history[data.history.length - 1];
        const entry = `[Gen ${last.generation + 1}] Best: ${last.best_fitness.toFixed(4)} | Avg: ${last.avg_fitness.toFixed(4)} | Std: ${last.std_fitness.toFixed(4)}\n`;
        if (!logDiv.textContent.includes(entry.trim())) {
          logDiv.textContent += entry;
          logDiv.scrollTop = logDiv.scrollHeight;
        }
      }
    })
    .catch(err => console.error('Poll error:', err));
}

function startEvolution() {
  const params = {
    pop_size: parseInt(document.getElementById('popSize').value),
    generations: parseInt(document.getElementById('generations').value),
    train_epochs: parseInt(document.getElementById('trainEpochs').value),
    mutation_rate: parseFloat(document.getElementById('mutationRate').value),
    n_samples: parseInt(document.getElementById('nSamples').value),
  };

  // Clear previous state
  fitnessChart.data.labels = [];
  fitnessChart.data.datasets[0].data = [];
  fitnessChart.data.datasets[1].data = [];
  fitnessChart.update();
  document.getElementById('log-area').textContent = 'Starting evolution...\n';

  fetch('/api/start', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify(params),
  })
  .then(r => r.json())
  .then(data => {
    console.log('Start:', data);
    if (!pollInterval) pollInterval = setInterval(pollStatus, 1500);
  })
  .catch(err => console.error('Start error:', err));
}

function stopEvolution() {
  fetch('/api/stop', { method: 'POST' })
    .then(r => r.json())
    .then(data => console.log('Stop:', data));
}

// Auto-poll on load
pollInterval = setInterval(pollStatus, 2000);
pollStatus();
</script>
</body>
</html>"""


# ═════════════════════════════════════════════════════════════════════
#  Flask Routes
# ═════════════════════════════════════════════════════════════════════
@app.route("/")
def index():
    return render_template_string(HTML_TEMPLATE)


@app.route("/api/status")
def api_status():
    return jsonify(state)


@app.route("/api/start", methods=["POST"])
def api_start():
    global engine_thread, X_global, Y_global

    if state["running"]:
        return jsonify({"error": "Evolution already running"}), 400

    params = request.get_json() or {}

    pop_size = params.get("pop_size", 6)
    generations = params.get("generations", 15)
    train_epochs = params.get("train_epochs", 10)
    mutation_rate = params.get("mutation_rate", 0.3)
    n_samples = params.get("n_samples", 5000)

    # Reset state
    state["history"] = []
    state["generation"] = 0
    state["best_genome"] = None
    state["population_summary"] = []
    state["best_fitness"] = 0.0
    state["improvement"] = 0.0

    # Start evolution in background thread
    # Dataset loading happens inside the thread to avoid blocking Flask
    engine_thread = threading.Thread(
        target=run_evolution_with_data,
        args=(pop_size, generations, mutation_rate, train_epochs, n_samples),
        daemon=True,
    )
    engine_thread.start()

    return jsonify({"status": "started", "generations": generations})


@app.route("/api/stop", methods=["POST"])
def api_stop():
    global engine
    if engine and state["running"]:
        engine.stop_event.set()
        state["status_message"] = "Stopping... (finishing current genome)"
        return jsonify({"status": "stopping"})
    return jsonify({"status": "not_running"})


# ═════════════════════════════════════════════════════════════════════
#  Launch
# ═════════════════════════════════════════════════════════════════════
def launch_gui(
    pop_size: int = 6,
    generations: int = 15,
    mutation_rate: float = 0.3,
    train_epochs: int = 10,
    X: np.ndarray = None,
    Y: np.ndarray = None,
    port: int = 5000,
):
    """Launch the Flask GUI server."""
    global X_global, Y_global

    X_global = X
    Y_global = Y

    if X_global is not None:
        state["dataset_loaded"] = True
        state["dataset_info"] = f"{X_global.shape[0]} samples, {X_global.shape[1]} features"
        log(f"Dataset pre-loaded: {state['dataset_info']}", "success")

    banner = f"""
╔══════════════════════════════════════════════════════════════╗
║  🧬 Self-Evolving Neural Network — Web GUI                  ║
║  Open http://localhost:{port} in your browser               ║
╚══════════════════════════════════════════════════════════════╝"""
    print(banner)

    app.run(host="0.0.0.0", port=port, debug=False, use_reloader=False)


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--port", type=int, default=5000)
    parser.add_argument("--n-samples", type=int, default=5000)
    args = parser.parse_args()

    X_data, Y_data = None, None
    try:
        X_data, Y_data = DataHandler.load_fable_dataset(n_samples=args.n_samples)
    except Exception as e:
        log(f"Could not load dataset: {e}. Will use synthetic data.", "warn")

    launch_gui(X=X_data, Y=Y_data, port=args.port)