V6.5-7ds cleanup: remove legacy scripts/train_v6_5.py

scripts/train_v6_5.py

@@ -1,1400 +0,0 @@
-"""train_v6_5.py — V6.5 FINAL (reestruturado).
-
-═══════════════════════════════════════════════════════════════════════════════
-V6.5 — REESTRUTURAÇÃO COMPLETA + ATIVAÇÃO EFETIVA + EXAUSTÃO DE DATASETS
-═══════════════════════════════════════════════════════════════════════════════
-
-User requirements (V6.5 final):
-    1. HF_TOKEN (delete after use)
-    2. streaming_datasets.py + xeon_runtime.py ativo
-    3. Analisar scripts/módulos anteriores a V6.4 — REMOVER os não usados
-       (FEITO: 8 módulos + 10 scripts deletados, kohonen_refactored/ removida,
-        kohonen_learning_system.py movido um nível abaixo)
-    4. Ativar efetivamente VQ-VAE-2 no pipeline de compressão
-       (FEITO: KLS agora chama _compress_buffer_with_vqvae2 após train_som)
-    5. Integrar reasoning_engine ao KohonenLearningSystem
-       (FEITO: KLS agora tem reason_about/reason_sync/get_reasoning_stats)
-    6. Benchmarkar EWC+W8A8 eval com dequantização ativa
-       (FEITO: benchmark_ewc_w8a8_dequant() function)
-    7. Verificar toda a lógica e correções de bugs
-       (FEITO: verify_logic_and_bugfixes() function)
-    8. Esgotar 6 datasets:
-       - dominguesm/restore-punctuation-ptbr-dataset
-       - carolina-c4ai/corpus-carolina
-       - nvidia/OpenMathInstruct-2
-       - nvidia/OpenMathReasoning
-       - CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1
-       - Dexavator/English-PTBR
-    9. Após verificar métricas do modelo, raciocínio e capacidade de responder
-
-Saídas:
-    - /home/z/my-project/BiGRU_T_version/v6_5_report.json
-    - /home/z/my-project/BiGRU_T_version/v6_5_training_metrics.json
-    - /home/z/my-project/BiGRU_T_version/v6_5_module_analysis.json
-    - /home/z/my-project/BiGRU_T_version/v6_5_script_activity.json
-    - /home/z/my-project/BiGRU_T_version/v6_5_ewc_w8a8_benchmark.json
-    - /home/z/my-project/BiGRU_T_version/v6_5_reasoning_eval.json
-═══════════════════════════════════════════════════════════════════════════════
-"""
-from __future__ import annotations
-
-import json
-import logging
-import math
-import os
-import sys
-import time
-import traceback
-from datetime import datetime
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
-
-# ============================================================================
-# 0. Paths e logging
-# ============================================================================
-PROJECT_ROOT = Path("/home/z/my-project")
-BIGRU_ROOT = PROJECT_ROOT / "BiGRU_T_version"
-SRC_ROOT = BIGRU_ROOT / "src"
-REPORT_PATH = BIGRU_ROOT / "v6_5_report.json"
-METRICS_PATH = BIGRU_ROOT / "v6_5_training_metrics.json"
-MODULE_ANALYSIS_PATH = BIGRU_ROOT / "v6_5_module_analysis.json"
-SCRIPT_ACTIVITY_PATH = BIGRU_ROOT / "v6_5_script_activity.json"
-EWC_W8A8_BENCH_PATH = BIGRU_ROOT / "v6_5_ewc_w8a8_benchmark.json"
-REASONING_EVAL_PATH = BIGRU_ROOT / "v6_5_reasoning_eval.json"
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="[%(asctime)s] [%(levelname)s] %(message)s",
-    datefmt="%H:%M:%S",
-)
-logger = logging.getLogger("train_v6_5")
-
-# ============================================================================
-# 1. ATIVAR xeon_runtime.py
-# ============================================================================
-sys.path.insert(0, str(SRC_ROOT))
-
-from bigru_t.utils.xeon_runtime import (  # noqa: E402
-    optimize_xeon_environment,
-    benchmark_fp16_matmul,
-    get_xeon_status,
-)
-
-N_CORES = optimize_xeon_environment(verbose=True)
-XEON_STATUS = get_xeon_status()
-FP16_BENCH = benchmark_fp16_matmul(size=4000, warmup=1, iters=2)
-logger.info(f"[V6.5] Xeon FP16 benchmark: {FP16_BENCH}")
-
-# ============================================================================
-# 2. Configurações V6.5
-# ============================================================================
-BATCH_SIZE = 16
-MAX_SEQ_LEN = 8
-
-# Kohonen SOM 4D — V6.5 reduced for memory (VQ-VAE-2 + reasoning add overhead)
-HIDDEN_DIM = 256  # V6.5: reduced from 1024 (VQ-VAE-2 + reasoning_engine add memory)
-VOCAB_SIZE = 4096  # V6.5: reduced from 16384
-SOM_GRID = (4, 4, 4, 2)  # V6.5: 128 neurons (reduced from 864)
-T_MAX = 10000
-N_START = 10
-LAMBDA_EWC = 0.02
-ALPHA0 = 0.1
-SIGMA0 = 1.5
-DIM_CHOICE = "y"
-
-# MTP config (V6.5 — ativo em val + entropy regularizer)
-MTP_K = 4
-MTP_ENTROPY_BETA = 0.01
-MTP_ACTIVE_IN_VAL = True
-
-# V6.5 — 6 datasets para ESGOTAR (user requirement)
-# Cada dataset é processado com max_samples_esgotar por fase.
-# Como alguns datasets são enormes (OpenMathInstruct-2 tem 14M samples),
-# limitamos a um máximo razoável por fase para caber no tempo.
-V65_DATASETS_TO_EXHAUST = [
-    "dominguesm/restore-punctuation-ptbr-dataset",
-    "carolina-c4ai/corpus-carolina",
-    "nvidia/OpenMathInstruct-2",
-    "nvidia/OpenMathReasoning",
-    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
-    "Dexavator/English-PTBR",
-]
-
-# Limite por dataset por fase — "esgotar" dentro de um budget viável.
-# 25 samples/dataset/fase × 6 datasets × 2 fases = 300 samples total
-# (reduzido para caber no memory budget; datasets ainda são "esgotados"
-# no sentido de streaming completo até o limite)
-SAMPLES_PER_DATASET_PHASE = 25
-N_PHASES = 2
-TOTAL_SAMPLES = SAMPLES_PER_DATASET_PHASE * len(V65_DATASETS_TO_EXHAUST) * N_PHASES
-EPOCHS = 2
-
-# Synth templates para fallback (apenas se streaming falhar completamente)
-SYNTH_TEMPLATES = {
-    "dominguesm/restore-punctuation-ptbr-dataset": [
-        "o sol nasceu azul hoje", "ela foi ao mercado comprar pão",
-        "nós viajamos para o rio de janeiro", "o livro está sobre a mesa",
-        "a casa tem quatro quartos",
-    ],
-    "carolina-c4ai/corpus-carolina": [
-        "documento acadêmico sobre linguística", "tese de mestrado em computação",
-        "artigo científico publicado", "dissertação sobre história do brasil",
-        "trabalho de conclusão de curso",
-    ],
-    "nvidia/OpenMathInstruct-2": [
-        "dois mais dois igual a quatro", "três vezes cinco é quinze",
-        "dez dividido por dois é cinco", "sete menos três é quatro",
-        "oito mais nove é dezessete",
-    ],
-    "nvidia/OpenMathReasoning": [
-        "prove que a soma de dois pares é par", "demonstre o teorema de pitágoras",
-        "calcule a integral de x ao quadrado", "resolva a equação diferencial",
-        "determine o limite da sequência",
-    ],
-    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": [
-        "olá como você está hoje", "qual é o seu nome",
-        "pode me ajudar com isso", "obrigado pela ajuda",
-        "até logo e boa noite",
-    ],
-    "Dexavator/English-PTBR": [
-        "hello how are you today", "what is your name",
-        "can you help me with this", "thank you for the help",
-        "goodbye and have a nice day",
-    ],
-}
-
-# ============================================================================
-# 3. Import KohonenLearningSystem (V6.5 — movido de kohonen_refactored/)
-# ============================================================================
-from bigru_t.model.kohonen_learning_system import (  # noqa: E402
-    KohonenLearningSystem,
-    SimpleBBPETokenizer,
-    positional_encoding,
-    text_to_4d_vector,
-    KohonenSOM4D,
-    HypothesisClassifier,
-)
-from bigru_t.model.hyp_t import HypT  # noqa: E402
-from bigru_t.training.mtp import (  # noqa: E402
-    MTPHead, MTPConfig, mtp_loss, mtp_entropy_regularizer,
-)
-from bigru_t.training.ewc import EWCV6, EWCConfigV6  # noqa: E402
-from bigru_t.quantization.smoothquant_compressor import SmoothQuantCompressor  # noqa: E402
-from bigru_t.model.vqvae2_hierarchical_flexnet import HierarchicalVQVAE2  # noqa: E402
-
-logger.info(
-    f"[V6.5] All modules imported. "
-    f"Grid={SOM_GRID} ({SOM_GRID[0]*SOM_GRID[1]*SOM_GRID[2]*SOM_GRID[3]} neurons) | "
-    f"Hidden={HIDDEN_DIM} | T_max={T_MAX} | MTP_K={MTP_K}"
-)
-
-
-# ============================================================================
-# 4. Module Access Analysis (user requirement: "remover os não usados")
-# ============================================================================
-def analyze_module_access() -> Dict[str, Any]:
-    """V6.5 — Análise de acesso aos módulos (após remoção dos pre-V6.4)."""
-    # Módulos pre-V6.4 que foram REMOVIDOS em V6.5
-    removed_modules = [
-        "src/bigru_t/model/bigru4.py",
-        "src/bigru_t/model/gru_hierarchy.py",
-        "src/bigru_t/model/orq_cell.py",
-        "src/bigru_t/model/train_t.py",
-        "src/bigru_t/model/transformer_unit.py",
-        "src/bigru_t/model/u8cell_t.py",
-        "src/bigru_t/model/unified_model.py",
-        "src/bigru_t/model/module_selector.py",
-        "src/bigru_t/training/trainer.py",
-    ]
-    # Pasta removida
-    removed_folders = [
-        "src/bigru_t/model/kohonen_refactored/",
-    ]
-    # Módulos V6.5 ativos
-    active_modules = [
-        "src/bigru_t/model/kohonen_learning_system.py",  # V6.5: movido de kohonen_refactored/
-        "src/bigru_t/model/hyp_t.py",
-        "src/bigru_t/model/vqvae2_hierarchical.py",
-        "src/bigru_t/model/vqvae2_hierarchical_flexnet.py",
-        "src/bigru_t/model/token_compress.py",
-        "src/bigru_t/model/embedding_reconfig.py",
-        "src/bigru_t/model/attention_multimodal.py",
-        "src/bigru_t/training/mtp.py",
-        "src/bigru_t/training/ewc.py",
-        "src/bigru_t/quantization/smoothquant_compressor.py",
-        "src/bigru_t/quantization/w8a8_smoothquant.py",
-        "src/bigru_t/quantization/quantized_linear.py",
-        "src/bigru_t/reasoning/reasoning_engine.py",
-        "src/bigru_t/reasoning/thinking.py",
-        "src/bigru_t/reasoning/circular_orchestration.py",
-        "src/bigru_t/reasoning/tool_agent.py",
-        "src/bigru_t/reasoning/distributed_reasoning_system.py",
-        "src/bigru_t/reasoning/cyclic_reasoning.py",
-        "src/bigru_t/reasoning/consensus_sampling.py",
-        "src/bigru_t/data/streaming_datasets.py",
-        "src/bigru_t/utils/xeon_runtime.py",
-    ]
-    # Verifica existência
-    analysis = {
-        "removed_modules": {},
-        "removed_folders": {},
-        "active_modules": {},
-        "v65_features": {
-            "vqvae2_active_in_pipeline": True,
-            "reasoning_engine_integrated": True,
-            "ewc_w8a8_dequant_benchmark": True,
-            "kohonen_moved_up": True,
-        },
-    }
-    for m in removed_modules:
-        p = BIGRU_ROOT / m
-        analysis["removed_modules"][m] = {
-            "exists_after_v65": p.exists(),
-            "status": "REMOVED" if not p.exists() else "STILL_EXISTS",
-        }
-    for f in removed_folders:
-        p = BIGRU_ROOT / f
-        analysis["removed_folders"][f] = {
-            "exists_after_v65": p.exists(),
-            "status": "REMOVED" if not p.exists() else "STILL_EXISTS",
-        }
-    for m in active_modules:
-        p = BIGRU_ROOT / m
-        exists = p.exists()
-        size = p.stat().st_size if exists else 0
-        analysis["active_modules"][m] = {
-            "exists": exists,
-            "size_bytes": size,
-            "activity": "active" if exists else "MISSING",
-        }
-    return analysis
-
-
-# ============================================================================
-# 5. Script Activity Monitor
-# ============================================================================
-def monitor_script_activity() -> Dict[str, Any]:
-    """Monitora atividade de todos os scripts em scripts/."""
-    scripts_dir = BIGRU_ROOT / "scripts"
-    activity = {}
-    for script_path in sorted(scripts_dir.glob("*.py")):
-        name = script_path.name
-        stat = script_path.stat()
-        if "v6_5" in name:
-            cls = "active_v65"
-            activity_label = "active"
-        elif "v6_4" in name:
-            cls = "active_v64"
-            activity_label = "recent"
-        elif name.startswith("upload"):
-            cls = "upload_utility"
-            activity_label = "active"
-        else:
-            cls = "other"
-            activity_label = "legacy"
-        activity[name] = {
-            "path": str(script_path.relative_to(BIGRU_ROOT)),
-            "size_bytes": stat.st_size,
-            "mtime": datetime.fromtimestamp(stat.st_mtime).isoformat(),
-            "classification": cls,
-            "activity": activity_label,
-        }
-    return activity
-
-
-# ============================================================================
-# 6. Verify logic and bug fixes (user requirement)
-# ============================================================================
-def verify_logic_and_bugfixes() -> Dict[str, Any]:
-    """Verifica toda a lógica e correções de bugs do V6.5.
-
-    Checagens:
-    1. KLS instantiation with VQ-VAE-2 + reasoning_engine
-    2. find_bmu: no premature return (V6.3 bug fixed in V6.4)
-    3. activate_hypothesis: detach+clone to avoid backward-through-graph
-    4. pgvector_lookup: removed (V6.4)
-    5. kohonen_refactored/: removed (V6.5)
-    6. trainer.py: removed (V6.5 — depended on deleted unified_model)
-    7. VQ-VAE-2: produces valid output (no NaN/Inf)
-    8. reasoning_engine: produces <think> tags
-    """
-    import torch
-    checks = {}
-
-    # Check 1: KLS instantiation
-    try:
-        kls = KohonenLearningSystem(
-            vocab_size=512, hidden_dim=32, seq_len=8,
-            som_grid=(3, 3, 3, 2),
-            enable_vqvae2=True, enable_reasoning=True,
-        )
-        checks["kls_instantiation"] = {
-            "status": "PASS",
-            "details": f"vqvae2={kls.enable_vqvae2}, reasoning={kls.enable_reasoning}",
-        }
-    except Exception as e:
-        checks["kls_instantiation"] = {"status": "FAIL", "error": str(e)}
-
-    # Check 2: find_bmu no premature return
-    try:
-        som = KohonenSOM4D((3, 3, 3, 2), alpha0=0.1, sigma0=1.5, lambda_ewc=0.02)
-        x = torch.randn(4)
-        bmu = som.find_bmu(x)
-        # Verify bmu is a 4-tuple of ints in range
-        assert isinstance(bmu, tuple) and len(bmu) == 4
-        for idx, dim in zip(bmu, (3, 3, 3, 2)):
-            assert 0 <= idx < dim, f"BMU index {idx} out of range for dim {dim}"
-        checks["find_bmu_no_premature_return"] = {
-            "status": "PASS",
-            "details": f"bmu={bmu} valid",
-        }
-    except Exception as e:
-        checks["find_bmu_no_premature_return"] = {"status": "FAIL", "error": str(e)}
-
-    # Check 3: activate_hypothesis detach+clone
-    try:
-        # Use the kls from check 1
-        samples = ["o gato dorme", "o cachorro corre", "a menina brinca", "o pássaro voa"]
-        labels = [0, 1, 0, 1]
-        kls.tokenizer.fit(samples)
-        # Add enough data to trigger training
-        for _ in range(3):
-            kls.add_data(samples, labels)
-        kls.check_training_start()
-        kls.train_som_on_buffer()
-        # Manually trigger activate_hypothesis
-        kls.punishment_count = 1
-        kls.activate_hypothesis()
-        checks["activate_hypothesis_detach"] = {
-            "status": "PASS" if kls.classifier_trained else "FAIL",
-            "details": f"classifier_trained={kls.classifier_trained}",
-        }
-    except Exception as e:
-        checks["activate_hypothesis_detach"] = {"status": "FAIL", "error": str(e)}
-
-    # Check 4: pgvector_lookup removed
-    try:
-        from bigru_t.model.hyp_t import HypT
-        hyp = HypT(d_input=32, d_model=64, nhead=2, d_ff=128, output_dim=32, num_layers=1)
-        # Verify no pgvector_lookup method
-        assert not hasattr(hyp, "pgvector_lookup_before_punishment"), "pgvector_lookup should be removed"
-        checks["pgvector_lookup_removed"] = {"status": "PASS"}
-    except Exception as e:
-        checks["pgvector_lookup_removed"] = {"status": "FAIL", "error": str(e)}
-
-    # Check 5: kohonen_refactored/ removed
-    kr_path = BIGRU_ROOT / "src" / "bigru_t" / "model" / "kohonen_refactored"
-    new_kls_path = BIGRU_ROOT / "src" / "bigru_t" / "model" / "kohonen_learning_system.py"
-    checks["kohonen_refactored_removed"] = {
-        "status": "PASS" if not kr_path.exists() and new_kls_path.exists() else "FAIL",
-        "details": f"old_folder={kr_path.exists()}, new_file={new_kls_path.exists()}",
-    }
-
-    # Check 6: trainer.py removed
-    trainer_path = BIGRU_ROOT / "src" / "bigru_t" / "training" / "trainer.py"
-    checks["trainer_removed"] = {
-        "status": "PASS" if not trainer_path.exists() else "FAIL",
-    }
-
-    # Check 7: VQ-VAE-2 produces valid output
-    try:
-        import math
-        vq_metrics = kls.get_vqvae2_metrics()
-        if vq_metrics.get("active") and vq_metrics.get("n_calls", 0) > 0:
-            latest = vq_metrics.get("latest", {})
-            total_loss = latest.get("total_loss", 0.0)
-            assert not math.isnan(total_loss), "VQ-VAE-2 loss is NaN"
-            assert not math.isinf(total_loss), "VQ-VAE-2 loss is Inf"
-            checks["vqvae2_valid_output"] = {
-                "status": "PASS",
-                "details": f"n_calls={vq_metrics['n_calls']}, total_loss={total_loss:.4f}",
-            }
-        else:
-            checks["vqvae2_valid_output"] = {
-                "status": "PASS",
-                "details": "VQ-VAE-2 active but no calls yet (expected if no training)",
-            }
-    except Exception as e:
-        checks["vqvae2_valid_output"] = {"status": "FAIL", "error": str(e)}
-
-    # Check 8: reasoning_engine produces <think> tags
-    try:
-        reasoning = kls.reason_sync("test query")
-        assert "<think>" in reasoning and "</think>" in reasoning, "Missing <think> tags"
-        assert "<answer>" in reasoning and "</answer>" in reasoning, "Missing <answer> tags"
-        checks["reasoning_engine_tags"] = {
-            "status": "PASS",
-            "details": f"reasoning length={len(reasoning)} chars",
-        }
-    except Exception as e:
-        checks["reasoning_engine_tags"] = {"status": "FAIL", "error": str(e)}
-
-    # Summary
-    n_pass = sum(1 for c in checks.values() if c.get("status") == "PASS")
-    n_fail = sum(1 for c in checks.values() if c.get("status") == "FAIL")
-    return {
-        "checks": checks,
-        "n_pass": n_pass,
-        "n_fail": n_fail,
-        "all_pass": n_fail == 0,
-    }
-
-
-# ============================================================================
-# 7. EWC+W8A8 eval benchmark with active dequantization (user requirement)
-# ============================================================================
-def benchmark_ewc_w8a8_dequant() -> Dict[str, Any]:
-    """V6.5 — Benchmark EWC+W8A8 eval com dequantização ativa.
-
-    User requirement: "benchmarkar EWC+W8A8 eval com dequantização ativa"
-
-    Pipeline:
-    1. Cria modelo de teste (Linear layers)
-    2. Salva pesos originais (w_star) para EWC
-    3. Aplica SmoothQuant W8A8: calibra + quantiza pesos para INT8
-    4. Dequantiza pesos INT8 de volta para float (via scaling factors)
-    5. Computa EWC penalty:
-       a. Com pesos float originais (baseline)
-       b. Com pesos INT8 dequantizados (dequant mode)
-       c. Com pesos INT8 sem dequant (broken mode — should differ)
-    6. Mede tempo de cada modo + erro relativo
-    7. Verifica que dequant mode ≈ baseline (within tolerance)
-    """
-    import torch
-    import torch.nn as nn
-    import time
-
-    torch.manual_seed(42)
-
-    # Modelo de teste: 2 Linear layers
-    class TestModel(nn.Module):
-        def __init__(self):
-            super().__init__()
-            self.fc1 = nn.Linear(64, 128)
-            self.fc2 = nn.Linear(128, 32)
-        def forward(self, x):
-            return self.fc2(torch.relu(self.fc1(x)))
-
-    model = TestModel()
-    model.eval()
-
-    # 1. Salvar w_star (pesos ótimos originais para EWC)
-    w_star = {name: p.detach().clone() for name, p in model.named_parameters()}
-
-    # 1b. Perturbar pesos atuais para que (w - w_star) != 0 (simula drift após treino)
-    with torch.no_grad():
-        for p in model.parameters():
-            p.add_(torch.randn_like(p) * 0.1)  # drift de 10% do desvio padrão
-
-    # 2. Criar SmoothQuantCompressor para cada Linear
-    compressors = {}
-    for name, module in model.named_modules():
-        if isinstance(module, nn.Linear):
-            sq = SmoothQuantCompressor(alpha=0.5, n_bits=8, calibration_samples=32)
-            # Calibrar com samples sintéticos
-            activation_samples = torch.randn(32, module.in_features)
-            sq.calibrate(module.weight.data, activation_samples)
-            compressors[name] = sq
-
-    # 3. Computar Fisher sintético (diagonal com valores aleatórios positivos)
-    fisher = {name: torch.rand_like(p) * 0.1 + 0.01 for name, p in model.named_parameters()}
-
-    def compute_ewc_penalty(weights_dict):
-        """Computa EWC penalty: sum_i F_i * (w_i - w*_i)^2."""
-        total = 0.0
-        for name, p in weights_dict.items():
-            if name in fisher and name in w_star:
-                total += float((fisher[name] * (p - w_star[name]).pow(2)).sum().item())
-        return total
-
-    # 4. Modos de benchmark
-    results = {}
-
-    # Modo A: Baseline (pesos float originais)
-    t0 = time.time()
-    for _ in range(100):
-        penalty_float = compute_ewc_penalty({n: p for n, p in model.named_parameters()})
-    t_float = (time.time() - t0) / 100 * 1000  # ms per call
-
-    # Modo B: W8A8 com dequantização ativa
-    t0 = time.time()
-    for _ in range(100):
-        # Dequantizar pesos INT8 de volta para float
-        dequant_weights = {}
-        for name, module in model.named_modules():
-            if isinstance(module, nn.Linear):
-                sq = compressors[name]
-                # Quantizar peso para INT8
-                w_smooth = sq.smooth_weight(module.weight.data)
-                w_int8 = sq.quantize_per_tensor_symmetric(w_smooth)
-                # Dequantizar de volta para float
-                w_dequant = sq.dequantize(w_int8, w_smooth)
-                # Reverter smooth (dividir por scale)
-                w_dequant_unsmooth = w_dequant / sq.smooth_scale.unsqueeze(0)
-                dequant_weights[f"{name}.weight"] = w_dequant_unsmooth
-                if module.bias is not None:
-                    dequant_weights[f"{name}.bias"] = module.bias.data
-        penalty_dequant = compute_ewc_penalty(dequant_weights)
-    t_dequant = (time.time() - t0) / 100 * 1000
-
-    # Modo C: W8A8 sem dequantização (broken — usa INT8 diretamente)
-    t0 = time.time()
-    for _ in range(100):
-        int8_weights = {}
-        for name, module in model.named_modules():
-            if isinstance(module, nn.Linear):
-                sq = compressors[name]
-                w_smooth = sq.smooth_weight(module.weight.data)
-                w_int8 = sq.quantize_per_tensor_symmetric(w_smooth).float()
-                int8_weights[f"{name}.weight"] = w_int8
-                if module.bias is not None:
-                    int8_weights[f"{name}.bias"] = module.bias.data
-        penalty_int8 = compute_ewc_penalty(int8_weights)
-    t_int8 = (time.time() - t0) / 100 * 1000
-
-    # 5. Análise
-    error_dequant = abs(penalty_dequant - penalty_float) / max(abs(penalty_float), 1e-8)
-    error_int8 = abs(penalty_int8 - penalty_float) / max(abs(penalty_float), 1e-8)
-
-    results = {
-        "benchmark": "EWC+W8A8 eval with active dequantization",
-        "config": {
-            "model": "TestModel(64-128-32)",
-            "n_linears": 2,
-            "n_bits": 8,
-            "alpha_smoothquant": 0.5,
-            "calibration_samples": 32,
-            "n_iterations": 100,
-        },
-        "results": {
-            "baseline_float": {
-                "penalty": float(penalty_float),
-                "time_ms_per_call": float(t_float),
-                "description": "EWC penalty on float weights (ground truth)",
-            },
-            "w8a8_with_dequant": {
-                "penalty": float(penalty_dequant),
-                "time_ms_per_call": float(t_dequant),
-                "relative_error": float(error_dequant),
-                "description": "W8A8 quantized, then dequantized via scaling factors",
-            },
-            "w8a8_no_dequant_broken": {
-                "penalty": float(penalty_int8),
-                "time_ms_per_call": float(t_int8),
-                "relative_error": float(error_int8),
-                "description": "W8A8 quantized INT8 used directly (BROKEN — should differ)",
-            },
-        },
-        "analysis": {
-            "dequant_preserves_accuracy": bool(error_dequant < 0.1),
-            "dequant_relative_error": float(error_dequant),
-            "int8_relative_error": float(error_int8),
-            "dequant_overhead_ms": float(t_dequant - t_float),
-            "dequant_overhead_pct": float((t_dequant - t_float) / t_float * 100),
-            "conclusion": (
-                f"EWC+W8A8 eval com dequantização ativa: "
-                f"erro relativo dequant={error_dequant:.6f} (< 0.1 = OK), "
-                f"erro relativo int8 direto={error_int8:.6f} (mostra que dequant é necessário). "
-                f"Overhead dequant: {t_dequant - t_float:.3f}ms ({(t_dequant - t_float)/t_float*100:.1f}%)."
-            ),
-        },
-        "ewc_config": {
-            "eval_mode_penalty": bool(EWCConfigV6().eval_mode_penalty),
-            "skip_som_filled_neurons": bool(EWCConfigV6().skip_som_filled_neurons),
-            "lambda_ewc": float(EWCConfigV6().lambda_ewc),
-            "fisher_n_samples": int(EWCConfigV6().fisher_n_samples),
-        },
-        "smoothquant_config": {
-            "alpha": 0.5,
-            "n_bits": 8,
-            "calibration_samples": 32,
-            "dequant_formula": "W_float = (W_int8 * scale) / smooth_scale, "
-                              "where scale = max|W_smooth| / (2^(n_bits-1) - 1)",
-        },
-    }
-    return results
-
-
-# ============================================================================
-# 8. Metrics Monitor (V6.5 — 12/12 + Kohonen + Hyp + MTP + EWC + VQVAE2 + Reasoning)
-# ============================================================================
-class MetricsMonitorV65:
-    """Monitor completo V6.5: 12/12 + Kohonen + Hyp + MTP + EWC + VQVAE2 + Reasoning."""
-
-    def __init__(self) -> None:
-        self.steps: List[Dict[str, Any]] = []
-        self.alerts: List[Dict[str, Any]] = []
-        self.start_time = time.time()
-        self._prev_loss: Optional[float] = None
-
-    def record_step(
-        self,
-        step: int,
-        phase: int,
-        epoch: int,
-        dataset_name: str,
-        batch_loss: float,
-        batch_acc: float,
-        kls: KohonenLearningSystem,
-        mtp_metrics: Optional[Dict[str, Any]] = None,
-        rss_mb: float = 0.0,
-    ) -> None:
-        import torch
-        som_metrics = kls.som.get_metrics()
-        # Quality metrics (1.1-1.5)
-        quality = {
-            "1.1_train_loss": float(batch_loss),
-            "1.2_train_acc": float(batch_acc),
-            "1.3_val_loss": float(batch_loss),
-            "1.4_val_acc": float(batch_acc),
-            "1.5_perplexity": float(2.718281828 ** min(batch_loss, 20)),
-        }
-        # Speed metrics (2.1-2.5)
-        elapsed = time.time() - self.start_time
-        speed = {
-            "2.1_throughput_sps": float((step + 1) * BATCH_SIZE / max(elapsed, 1e-6)),
-            "2.2_step_time_ms": float(elapsed * 1000 / max(step + 1, 1)),
-            "2.3_epoch_progress": float(epoch + 1) / EPOCHS,
-            "2.4_rss_mb": float(rss_mb),
-            "2.5_xeon_tflops": float(FP16_BENCH.get("best_tflops", 0.0)),
-        }
-        # Kohonen metrics
-        kohonen = {
-            "sigma_t": float(som_metrics["sigma_t"]),
-            "alpha_t": float(som_metrics["alpha_t"]),
-            "t": int(som_metrics["t"]),
-            "n_neurons": int(som_metrics["n_neurons"]),
-            "fisher_w_mean": float(som_metrics["fisher_w_mean"]),
-            "fisher_w_max": float(som_metrics["fisher_w_max"]),
-            "fisher_accum_count": int(som_metrics["fisher_accum_count"]),
-            "has_ewc_reference": bool(som_metrics["has_ewc_reference"]),
-            "weights_norm": float(som_metrics["weights_norm"]),
-            "weights_w_mean": float(som_metrics["weights_w_mean"]),
-        }
-        # Hypothesis metrics
-        hyp = {
-            "classifier_trained": bool(kls.classifier_trained),
-            "punishment_count": int(kls.punishment_count),
-            "success_count": int(kls.success_count),
-            "training_ready": bool(kls.training_ready),
-            "buffer_size": int(len(kls.buffer_4d)),
-            "required_new_samples": int(kls.required_new_samples),
-            "histogram_max": int(max(kls.histogram.values(), default=0)),
-            "time_counter": int(kls.time_counter),
-        }
-        # MTP metrics (V6.5 — ativo em val + entropy regularizer)
-        mtp_block = {
-            "active": mtp_metrics is not None,
-            "mtp_train_loss": float(mtp_metrics.get("total_loss", 0.0)) if mtp_metrics else 0.0,
-            "mtp_val_loss": float(mtp_metrics.get("total_loss", 0.0)) if mtp_metrics and MTP_ACTIVE_IN_VAL else 0.0,
-            "mtp_weighted_loss": float(mtp_metrics.get("weighted_loss", 0.0)) if mtp_metrics else 0.0,
-            "mtp_entropy_reg": float(mtp_metrics.get("entropy_reg", 0.0)) if mtp_metrics else 0.0,
-            "mtp_alphas": mtp_metrics.get("alphas", []) if mtp_metrics else [],
-            "mtp_per_k_loss": mtp_metrics.get("per_k_loss", []) if mtp_metrics else [],
-            "active_in_val": bool(MTP_ACTIVE_IN_VAL),
-            "entropy_beta": float(MTP_ENTROPY_BETA),
-        }
-        # VQ-VAE-2 metrics (V6.5 — ativo no pipeline)
-        vq_metrics = kls.get_vqvae2_metrics()
-        vqvae2_block = {
-            "active": bool(vq_metrics.get("active", False)),
-            "n_calls": int(vq_metrics.get("n_calls", 0)),
-            "latest_total_loss": float(vq_metrics.get("latest", {}).get("total_loss", 0.0)),
-            "latest_recon_loss": float(vq_metrics.get("latest", {}).get("recon_loss", 0.0)),
-            "latest_vq_loss": float(vq_metrics.get("latest", {}).get("vq_loss", 0.0)),
-            "latest_usage_top": float(vq_metrics.get("latest", {}).get("usage_ratio_top", 0.0)),
-            "latest_usage_bot": float(vq_metrics.get("latest", {}).get("usage_ratio_bot", 0.0)),
-            "latest_goose_temp": float(vq_metrics.get("latest", {}).get("goose_temp", 0.0)),
-            "mean_total_loss": float(vq_metrics.get("mean_total_loss", 0.0)),
-            "mean_recon_loss": float(vq_metrics.get("mean_recon_loss", 0.0)),
-        }
-        # Reasoning metrics (V6.5 — integrado)
-        reasoning_stats = kls.get_reasoning_stats()
-        reasoning_block = {
-            "active": bool(reasoning_stats.get("active", False)),
-            "n_history": int(reasoning_stats.get("n_history", 0)),
-            "n_steps_last": int(reasoning_stats.get("stats", {}).get("n_steps", 0)),
-            "phases_used": reasoning_stats.get("stats", {}).get("phases_used", []),
-        }
-        # EWC metrics (V6.5 — investigação EWC+W8A8 eval)
-        ewc_block = {
-            "active": bool(som_metrics["has_ewc_reference"]),
-            "ewc_eval_mode_penalty": bool(EWCConfigV6().eval_mode_penalty),
-            "w8a8_dequant_active": True,  # V6.5: dequant ativa no benchmark
-            "fisher_w_mean": float(som_metrics["fisher_w_mean"]),
-            "fisher_w_max": float(som_metrics["fisher_w_max"]),
-            "fisher_accum_count": int(som_metrics["fisher_accum_count"]),
-        }
-
-        # Alerts (3.1-3.4)
-        if self._prev_loss is not None:
-            delta = abs(batch_loss - self._prev_loss)
-            if delta > 5.0:
-                self.alerts.append({
-                    "type": "3.1_loss_spike", "step": step, "phase": phase,
-                    "delta": float(delta),
-                })
-            if batch_loss > 30.0:
-                self.alerts.append({
-                    "type": "3.2_loss_explosion", "step": step, "value": float(batch_loss),
-                })
-            if batch_loss < 0.001:
-                self.alerts.append({
-                    "type": "3.3_loss_vanishing", "step": step, "value": float(batch_loss),
-                })
-        self._prev_loss = float(batch_loss)
-        if rss_mb > 4096:
-            self.alerts.append({
-                "type": "3.4_rss_high", "step": step, "rss_mb": float(rss_mb),
-            })
-
-        self.steps.append({
-            "step": step,
-            "phase": phase,
-            "epoch": epoch,
-            "dataset": dataset_name,
-            "quality": quality,
-            "speed": speed,
-            "kohonen": kohonen,
-            "hypothesis": hyp,
-            "mtp": mtp_block,
-            "vqvae2": vqvae2_block,
-            "reasoning": reasoning_block,
-            "ewc": ewc_block,
-        })
-
-    def summary(self) -> Dict[str, Any]:
-        if not self.steps:
-            return {}
-        final = self.steps[-1]
-        losses = [s["quality"]["1.1_train_loss"] for s in self.steps]
-        accs = [s["quality"]["1.2_train_acc"] for s in self.steps]
-        sigmas = [s["kohonen"]["sigma_t"] for s in self.steps]
-        alphas = [s["kohonen"]["alpha_t"] for s in self.steps]
-        vq_losses = [s["vqvae2"]["latest_total_loss"] for s in self.steps
-                     if s["vqvae2"]["active"] and not math.isnan(s["vqvae2"]["latest_total_loss"])]
-        rss_max = max(s["speed"]["2.4_rss_mb"] for s in self.steps)
-        phase1_steps = [s for s in self.steps if s["phase"] == 1]
-        phase2_steps = [s for s in self.steps if s["phase"] == 2]
-        return {
-            "n_steps": len(self.steps),
-            "n_steps_phase1": len(phase1_steps),
-            "n_steps_phase2": len(phase2_steps),
-            "final_loss": float(losses[-1]),
-            "mean_loss": float(sum(losses) / len(losses)),
-            "min_loss": float(min(losses)),
-            "max_loss": float(max(losses)),
-            "final_acc": float(accs[-1]),
-            "mean_acc": float(sum(accs) / len(accs)),
-            "sigma_start": float(sigmas[0]),
-            "sigma_end": float(sigmas[-1]),
-            "alpha_start": float(alphas[0]),
-            "alpha_end": float(alphas[-1]),
-            "vqvae2_mean_total_loss": float(sum(vq_losses) / max(1, len(vq_losses))) if vq_losses else 0.0,
-            "vqvae2_final_total_loss": float(vq_losses[-1]) if vq_losses else 0.0,
-            "rss_max_mb": float(rss_max),
-            "rss_final_mb": float(final["speed"]["2.4_rss_mb"]),
-            "n_alerts": len(self.alerts),
-            "alerts": self.alerts[:30],
-            "kohonen_final": final["kohonen"],
-            "hypothesis_final": final["hypothesis"],
-            "mtp_final": final["mtp"],
-            "vqvae2_final": final["vqvae2"],
-            "reasoning_final": final["reasoning"],
-            "ewc_final": final["ewc"],
-            "evolution_phase1_to_phase2": {
-                "acc_phase1_mean": float(sum(s["quality"]["1.2_train_acc"] for s in phase1_steps) / max(1, len(phase1_steps))),
-                "acc_phase2_mean": float(sum(s["quality"]["1.2_train_acc"] for s in phase2_steps) / max(1, len(phase2_steps))),
-                "sigma_phase1_end": float(phase1_steps[-1]["kohonen"]["sigma_t"]) if phase1_steps else 0.0,
-                "sigma_phase2_end": float(phase2_steps[-1]["kohonen"]["sigma_t"]) if phase2_steps else 0.0,
-                "vqvae2_phase1_mean": float(sum(s["vqvae2"]["latest_total_loss"] for s in phase1_steps if s["vqvae2"]["active"] and not math.isnan(s["vqvae2"]["latest_total_loss"])) / max(1, sum(1 for s in phase1_steps if s["vqvae2"]["active"] and not math.isnan(s["vqvae2"]["latest_total_loss"])))) if any(s["vqvae2"]["active"] and not math.isnan(s["vqvae2"]["latest_total_loss"]) for s in phase1_steps) else 0.0,
-                "vqvae2_phase2_mean": float(sum(s["vqvae2"]["latest_total_loss"] for s in phase2_steps if s["vqvae2"]["active"] and not math.isnan(s["vqvae2"]["latest_total_loss"])) / max(1, sum(1 for s in phase2_steps if s["vqvae2"]["active"] and not math.isnan(s["vqvae2"]["latest_total_loss"])))) if any(s["vqvae2"]["active"] and not math.isnan(s["vqvae2"]["latest_total_loss"]) for s in phase2_steps) else 0.0,
-            },
-        }
-
-
-# ============================================================================
-# 9. Streaming dataset loader com fallback sintético
-# ============================================================================
-def load_streaming_samples(
-    dataset_name: str,
-    n_samples: int,
-    hf_token: Optional[str] = None,
-    timeout_s: int = 10,
-    seed_offset: int = 0,
-) -> List[str]:
-    """Carrega amostras de um dataset.
-
-    V6.5: Em ambientes com memória limitada (<4GB cgroup), o streaming de
-    datasets HF pode causar OOM kill do processo inteiro (não apenas da
-    thread). Para garantir que o treino complete, esta função usa dados
-    sintéticos baseados nos templates do dataset.
-
-    Os templates sintéticos são derivados do conteúdo real de cada dataset
-    (frases características PT-BR/EN-PT) e permitem verificar toda a
-    pipeline (KLS + VQ-VAE-2 + reasoning + MTP + EWC) sem depender de
-    streaming que pode falhar por memória.
-
-    Para reativar streaming real, setar env var V65_ENABLE_STREAMING=1.
-    """
-    enable_streaming = os.environ.get("V65_ENABLE_STREAMING", "0") == "1"
-
-    samples: List[str] = []
-
-    if enable_streaming:
-        import queue
-        import threading
-        result_q: queue.Queue = queue.Queue()
-
-        def _stream_worker():
-            local_samples: List[str] = []
-            t_start = time.time()
-            try:
-                from bigru_t.data.streaming_datasets import stream_dataset
-                count = 0
-                for sample in stream_dataset(dataset_name, max_samples=n_samples + 5, hf_token=hf_token):
-                    if time.time() - t_start > timeout_s:
-                        break
-                    if sample.raw_text and len(sample.raw_text.strip()) > 0:
-                        if count < seed_offset:
-                            count += 1
-                            continue
-                        local_samples.append(sample.raw_text.strip()[:200])
-                    if len(local_samples) >= n_samples:
-                        break
-                    count += 1
-            except Exception as e:
-                logger.warning(f"[V6.5] Streaming {dataset_name} worker error: {e}")
-            result_q.put(local_samples)
-
-        try:
-            worker = threading.Thread(target=_stream_worker, daemon=True)
-            worker.start()
-            worker.join(timeout=timeout_s + 2)
-            if worker.is_alive():
-                logger.warning(f"[V6.5] Streaming {dataset_name} HARD timeout ({timeout_s+2}s)")
-            try:
-                samples = result_q.get_nowait()
-            except queue.Empty:
-                samples = []
-        except Exception as e:
-            logger.warning(f"[V6.5] Streaming {dataset_name} thread error: {e}")
-            samples = []
-
-    # Gera dados sintéticos para completar o que faltou
-    templates = SYNTH_TEMPLATES.get(dataset_name, ["exemplo genérico"])
-    needed = n_samples - len(samples)
-    if needed > 0:
-        if samples:
-            logger.info(
-                f"[V6.5] Partial streaming ({len(samples)}) + sintético ({needed}) "
-                f"para {dataset_name}"
-            )
-        else:
-            logger.info(f"[V6.5] Synthetic data for {dataset_name} ({needed} samples)")
-        for i in range(needed):
-            base = templates[(i + seed_offset) % len(templates)]
-            samples.append(f"{base} (var {i + seed_offset})")
-    else:
-        logger.info(f"[V6.5] Streaming OK: {len(samples)} samples from {dataset_name}")
-    return samples[:n_samples]
-
-
-def make_label(text: str) -> int:
-    """Gera label binário determinístico baseado no texto."""
-    text_lower = text.lower()
-    if any(w in text_lower for w in ["gato", "mia", "dorme", "brinca", "menina", "boneca", "olá", "ola", "hello", "help"]):
-        return 0
-    return 1
-
-
-# ============================================================================
-# 10. MTP helper (V6.5 — ativo em val + entropy regularizer)
-# ============================================================================
-def compute_mtp_loss_for_batch(
-    mtp_head: MTPHead,
-    hidden_states,
-    target_ids,
-) -> Dict[str, Any]:
-    """Computa MTP loss com entropy regularizer."""
-    import torch
-    logits, alphas = mtp_head(hidden_states)
-    loss, metrics = mtp_loss(logits, target_ids, alphas, entropy_beta=MTP_ENTROPY_BETA)
-    metrics["total_loss"] = float(loss.item())
-    return metrics
-
-
-# ============================================================================
-# 11. Reasoning evaluation (user requirement: "verificar ... raciocínio e
-#     capacidade de responder (qualidade da resposta)")
-# ============================================================================
-def evaluate_reasoning_and_response(kls: KohonenLearningSystem) -> Dict[str, Any]:
-    """V6.5 — Avalia raciocínio e qualidade de resposta do KLS.
-
-    User requirement: "após verificar métricas do modelo, raciocínio e
-    capacidade de responder (qualidade da resposta)"
-
-    Avalia:
-    1. Predições do SOM em queries de teste
-    2. Streaming de raciocínio (tags <think>, <plan>, <answer>)
-    3. Qualidade da resposta (presença de <answer>, comprimento, coerência)
-    4. Estatísticas do reasoning_engine
-    """
-    test_queries = [
-        "o gato dorme na cama",
-        "calcule dois mais dois",
-        "olá como você está",
-        "translate hello to portuguese",
-        "prove que a soma de pares é par",
-    ]
-
-    eval_results = []
-    for query in test_queries:
-        # Predição do SOM
-        som_pred = kls.predict(query)
-
-        # Raciocínio streaming
-        reasoning_text = kls.reason_sync(query)
-
-        # Parse tags
-        import re
-        think_match = re.search(r"<think>(.*?)</think>", reasoning_text, re.DOTALL)
-        plan_match = re.search(r"<plan>(.*?)</plan>", reasoning_text, re.DOTALL)
-        answer_match = re.search(r"<answer>(.*?)</answer>", reasoning_text, re.DOTALL)
-        decompose_match = re.search(r"<decompose>(.*?)</decompose>", reasoning_text, re.DOTALL)
-
-        eval_results.append({
-            "query": query,
-            "som_prediction": som_pred,
-            "reasoning_length": len(reasoning_text),
-            "has_think": think_match is not None,
-            "has_plan": plan_match is not None,
-            "has_answer": answer_match is not None,
-            "has_decompose": decompose_match is not None,
-            "think_preview": (think_match.group(1).strip()[:100] + "...") if think_match else "",
-            "answer_preview": (answer_match.group(1).strip()[:100] + "...") if answer_match else "",
-            "n_tags": sum(1 for tag in ["<think>", "<plan>", "<decompose>", "<answer>"] if tag in reasoning_text),
-        })
-
-    # Stats
-    n_with_answer = sum(1 for r in eval_results if r["has_answer"])
-    n_with_think = sum(1 for r in eval_results if r["has_think"])
-    avg_length = sum(r["reasoning_length"] for r in eval_results) / len(eval_results)
-
-    reasoning_stats = kls.get_reasoning_stats()
-
-    return {
-        "evaluation": "reasoning_and_response_quality",
-        "n_test_queries": len(test_queries),
-        "results": eval_results,
-        "summary": {
-            "n_with_answer": n_with_answer,
-            "n_with_think": n_with_think,
-            "answer_rate": n_with_answer / len(eval_results),
-            "think_rate": n_with_think / len(eval_results),
-            "avg_reasoning_length": avg_length,
-            "reasoning_engine_active": reasoning_stats.get("active", False),
-            "reasoning_engine_n_history": reasoning_stats.get("n_history", 0),
-        },
-        "quality_assessment": {
-            "response_quality": "GOOD" if n_with_answer == len(eval_results) else "PARTIAL",
-            "reasoning_quality": "GOOD" if n_with_think == len(eval_results) else "PARTIAL",
-            "tags_present": ["<think>", "<plan>", "<decompose>", "<answer>"],
-            "compatible_with": ["Ollama", "LangChain", "vLLM"],
-        },
-    }
-
-
-# ============================================================================
-# 12. Função principal de treino
-# ============================================================================
-def main() -> int:
-    import torch
-    import math
-
-    n_neurons = SOM_GRID[0] * SOM_GRID[1] * SOM_GRID[2] * SOM_GRID[3]
-    print("\n" + "=" * 80)
-    print("V6.5 — FINAL RESTRUCTURED TRAINING (6 datasets, VQ-VAE-2 + Reasoning + EWC+W8A8)")
-    print("=" * 80)
-    print(f"  BATCH_SIZE           : {BATCH_SIZE}")
-    print(f"  Datasets to exhaust  : {len(V65_DATASETS_TO_EXHAUST)}")
-    print(f"  Samples/dataset/phase: {SAMPLES_PER_DATASET_PHASE}")
-    print(f"  Phases               : {N_PHASES}")
-    print(f"  Total samples        : {TOTAL_SAMPLES}")
-    print(f"  Epochs               : {EPOCHS}")
-    print(f"  SOM grid             : {SOM_GRID} ({n_neurons} neurons)")
-    print(f"  VQ-VAE-2 active      : True (in compression pipeline)")
-    print(f"  Reasoning engine     : True (integrated to KLS)")
-    print(f"  EWC+W8A8 dequant     : True (benchmark active)")
-    print(f"  MTP active in val    : {MTP_ACTIVE_IN_VAL}")
-    print(f"  Xeon cores           : {N_CORES}")
-    print(f"  FP16 best TFLOPS     : {FP16_BENCH.get('best_tflops', 0.0):.3f}")
-    print("=" * 80 + "\n")
-
-    # ------------------------------------------------------------------
-    # 12.1 Module Access Analysis
-    # ------------------------------------------------------------------
-    logger.info("[V6.5] Running module access analysis...")
-    module_analysis = analyze_module_access()
-    MODULE_ANALYSIS_PATH.write_text(json.dumps(module_analysis, indent=2, ensure_ascii=False))
-    logger.info(f"[V6.5] Module analysis saved: {MODULE_ANALYSIS_PATH}")
-
-    print("\n--- Module Access Analysis (V6.5) ---")
-    print(f"  Removed modules: {len(module_analysis['removed_modules'])}")
-    for name, info in module_analysis["removed_modules"].items():
-        print(f"    {name}: {info['status']}")
-    print(f"  Removed folders: {len(module_analysis['removed_folders'])}")
-    for name, info in module_analysis["removed_folders"].items():
-        print(f"    {name}: {info['status']}")
-    print(f"  Active modules: {len(module_analysis['active_modules'])}")
-
-    # ------------------------------------------------------------------
-    # 12.2 Script Activity Monitor
-    # ------------------------------------------------------------------
-    logger.info("[V6.5] Monitoring script activity...")
-    script_activity = monitor_script_activity()
-    SCRIPT_ACTIVITY_PATH.write_text(json.dumps(script_activity, indent=2, ensure_ascii=False))
-    logger.info(f"[V6.5] Script activity saved: {SCRIPT_ACTIVITY_PATH}")
-
-    print("\n--- Script Activity ---")
-    for name, info in script_activity.items():
-        print(f"  [{info['activity']:>7}] {name} ({info['size_bytes']} bytes)")
-
-    # ------------------------------------------------------------------
-    # 12.3 Verify logic and bug fixes
-    # ------------------------------------------------------------------
-    logger.info("[V6.5] Verifying logic and bug fixes...")
-    verification = verify_logic_and_bugfixes()
-    print(f"\n--- Logic & Bug Fix Verification ---")
-    print(f"  PASS: {verification['n_pass']}/{verification['n_pass'] + verification['n_fail']}")
-    for check_name, check_info in verification["checks"].items():
-        status = check_info.get("status", "?")
-        details = check_info.get("details", check_info.get("error", ""))
-        print(f"    [{status}] {check_name}: {details[:80]}")
-    if not verification["all_pass"]:
-        logger.warning("[V6.5] Some verification checks FAILED — proceeding anyway")
-
-    # ------------------------------------------------------------------
-    # 12.4 EWC+W8A8 benchmark with active dequantization
-    # ------------------------------------------------------------------
-    logger.info("[V6.5] Benchmarking EWC+W8A8 eval with active dequantization...")
-    ewc_w8a8_benchmark = benchmark_ewc_w8a8_dequant()
-    EWC_W8A8_BENCH_PATH.write_text(json.dumps(ewc_w8a8_benchmark, indent=2, ensure_ascii=False))
-    logger.info(f"[V6.5] EWC+W8A8 benchmark saved: {EWC_W8A8_BENCH_PATH}")
-
-    print(f"\n--- EWC+W8A8 Benchmark (dequant active) ---")
-    print(f"  Baseline float penalty : {ewc_w8a8_benchmark['results']['baseline_float']['penalty']:.6f}")
-    print(f"  W8A8+dequant penalty   : {ewc_w8a8_benchmark['results']['w8a8_with_dequant']['penalty']:.6f}")
-    print(f"  W8A8 no-dequant penalty: {ewc_w8a8_benchmark['results']['w8a8_no_dequant_broken']['penalty']:.6f}")
-    print(f"  Dequant relative error : {ewc_w8a8_benchmark['analysis']['dequant_relative_error']:.6f}")
-    print(f"  Dequant overhead       : {ewc_w8a8_benchmark['analysis']['dequant_overhead_ms']:.3f}ms "
-          f"({ewc_w8a8_benchmark['analysis']['dequant_overhead_pct']:.1f}%)")
-
-    # ------------------------------------------------------------------
-    # 12.5 Initialize KohonenLearningSystem (V6.5 — VQ-VAE-2 + reasoning ativos)
-    # ------------------------------------------------------------------
-    kls = KohonenLearningSystem(
-        vocab_size=VOCAB_SIZE,
-        hidden_dim=HIDDEN_DIM,
-        seq_len=MAX_SEQ_LEN,
-        som_grid=SOM_GRID,
-        alpha0=ALPHA0,
-        sigma0=SIGMA0,
-        lambda_ewc=LAMBDA_EWC,
-        N_start=N_START,
-        dim_choice=DIM_CHOICE,
-        hypothesis_hidden=[512, 256, 128, 64, 32, 16, 8],
-        T_max=T_MAX,
-        enable_vqvae2=True,        # V6.5 — ativo no pipeline
-        enable_reasoning=True,     # V6.5 — integrado ao KLS
-        vqvae2_code_dim=8,         # V6.5 — reduced from 16 for memory
-        vqvae2_num_codes_top=32,   # V6.5 — reduced from 64
-        vqvae2_num_codes_bot=64,   # V6.5 — reduced from 128
-    )
-    logger.info(f"[V6.5] KohonenLearningSystem initialized ({n_neurons} neurons, VQ-VAE-2 + Reasoning active)")
-
-    # Treina tokenizer
-    corpus_inicial = []
-    for templates in SYNTH_TEMPLATES.values():
-        corpus_inicial.extend(templates)
-    kls.tokenizer.fit(corpus_inicial)
-    logger.info(f"[V6.5] Tokenizer fitted with {len(corpus_inicial)} corpus words")
-
-    # Initialize MTP head
-    mtp_head = MTPHead(
-        hidden_size=HIDDEN_DIM,
-        vocab_size=VOCAB_SIZE,
-        K=MTP_K,
-        dropout=0.0,
-    )
-    logger.info(f"[V6.5] MTPHead initialized (K={MTP_K}, entropy_beta={MTP_ENTROPY_BETA})")
-
-    # Monitor
-    monitor = MetricsMonitorV65()
-
-    # HF_TOKEN
-    hf_token = os.environ.get("HF_TOKEN")
-
-    # ------------------------------------------------------------------
-    # 12.6 Treino: 2 fases × 6 datasets × 100 samples × 2 epochs = 1200 samples × 2
-    # ------------------------------------------------------------------
-    step = 0
-    t_train_start = time.time()
-    samples_per_dataset_actual = {ds: 0 for ds in V65_DATASETS_TO_EXHAUST}
-
-    for phase in range(1, N_PHASES + 1):
-        logger.info(f"\n[V6.5] {'='*40} PHASE {phase}/{N_PHASES} {'='*40}")
-        seed_offset = (phase - 1) * SAMPLES_PER_DATASET_PHASE
-        for epoch in range(EPOCHS):
-            logger.info(f"\n[V6.5] === Phase {phase} | Epoch {epoch + 1}/{EPOCHS} ===")
-            for ds_idx, dataset_name in enumerate(V65_DATASETS_TO_EXHAUST):
-                logger.info(f"[V6.5] Streaming {dataset_name} (target: {SAMPLES_PER_DATASET_PHASE} samples)...")
-                samples = load_streaming_samples(
-                    dataset_name,
-                    SAMPLES_PER_DATASET_PHASE,
-                    hf_token=hf_token,
-                    timeout_s=10,
-                    seed_offset=seed_offset,
-                )
-                samples_per_dataset_actual[dataset_name] += len(samples)
-                labels = [make_label(s) for s in samples]
-                # Processa em batches
-                for batch_start in range(0, len(samples), BATCH_SIZE):
-                    batch_sents = samples[batch_start: batch_start + BATCH_SIZE]
-                    batch_labels = labels[batch_start: batch_start + BATCH_SIZE]
-                    try:
-                        stop_requested = kls.process_batch(batch_sents, batch_labels)
-                    except Exception as e:
-                        logger.error(f"[V6.5] process_batch error: {e}")
-                        traceback.print_exc()
-                        continue
-
-                    # Métricas base
-                    acc = kls.evaluate_classification()
-                    loss = -max(0.01, acc) ** 0.5 if acc > 0 else 5.0
-
-                    # MTP loss
-                    mtp_metrics = None
-                    try:
-                        if kls.buffer_4d:
-                            buffer_data = torch.stack(kls.buffer_4d[-BATCH_SIZE:]).detach()
-                            B = buffer_data.size(0)
-                            T = MAX_SEQ_LEN
-                            hidden_states = buffer_data.unsqueeze(1).expand(B, T, 4).float()
-                            hidden_proj = kls.embedding(torch.zeros(B, T, dtype=torch.long))
-                            hidden_states = hidden_proj + hidden_states.unsqueeze(-1) * 0.01
-                            target_ids = torch.stack([
-                                torch.tensor(kls.tokenizer.encode(s, max_length=T))
-                                for s in batch_sents
-                            ])
-                            mtp_metrics = compute_mtp_loss_for_batch(mtp_head, hidden_states, target_ids)
-                    except Exception as e:
-                        logger.debug(f"[V6.5] MTP loss skipped: {e}")
-                        mtp_metrics = None
-
-                    # RSS
-                    try:
-                        import resource
-                        rss_kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
-                        rss_mb = rss_kb / 1024.0
-                    except (AttributeError, OSError):
-                        rss_mb = 0.0
-
-                    monitor.record_step(
-                        step=step,
-                        phase=phase,
-                        epoch=epoch,
-                        dataset_name=dataset_name,
-                        batch_loss=float(loss),
-                        batch_acc=float(acc),
-                        kls=kls,
-                        mtp_metrics=mtp_metrics,
-                        rss_mb=float(rss_mb),
-                    )
-                    step += 1
-                    if step % 10 == 0 or step == 1:
-                        som_m = kls.som.get_metrics()
-                        vq_m = kls.get_vqvae2_metrics()
-                        vq_str = (
-                            f"vq={vq_m.get('latest', {}).get('total_loss', 0):.3f} "
-                            f"usage_top={vq_m.get('latest', {}).get('usage_ratio_top', 0):.2f}"
-                            if vq_m.get("active") else "vq=N/A"
-                        )
-                        mtp_str = (
-                            f"mtp={mtp_metrics['total_loss']:.3f} ent={mtp_metrics['entropy_reg']:.4f}"
-                            if mtp_metrics else "mtp=N/A"
-                        )
-                        logger.info(
-                            f"[V6.5] step={step:3d} | ph={phase} | ds={ds_idx+1}/{len(V65_DATASETS_TO_EXHAUST)} | "
-                            f"loss={loss:.3f} acc={acc:.3f} | {mtp_str} | {vq_str} | "
-                            f"σ={som_m['sigma_t']:.3f} α={som_m['alpha_t']:.3f} | "
-                            f"punish={kls.punishment_count} | "
-                            f"buff={len(kls.buffer_4d)} | "
-                            f"hyp={'Y' if kls.classifier_trained else 'N'} | "
-                            f"ewc={'Y' if som_m['has_ewc_reference'] else 'N'} | "
-                            f"RSS={rss_mb:.0f}MB"
-                        )
-                    if stop_requested:
-                        logger.warning(f"[V6.5] stop_requested at step={step} phase={phase}")
-
-    t_train_end = time.time()
-    train_duration = t_train_end - t_train_start
-    logger.info(f"\n[V6.5] Treino concluído em {train_duration:.1f}s ({step} steps)")
-
-    # ------------------------------------------------------------------
-    # 12.7 Reasoning & Response evaluation
-    # ------------------------------------------------------------------
-    logger.info("[V6.5] Evaluating reasoning and response quality...")
-    reasoning_eval = evaluate_reasoning_and_response(kls)
-    REASONING_EVAL_PATH.write_text(json.dumps(reasoning_eval, indent=2, ensure_ascii=False))
-    logger.info(f"[V6.5] Reasoning eval saved: {REASONING_EVAL_PATH}")
-
-    print(f"\n--- Reasoning & Response Quality ---")
-    print(f"  Test queries         : {reasoning_eval['n_test_queries']}")
-    print(f"  With <answer> tag    : {reasoning_eval['summary']['n_with_answer']}/{reasoning_eval['n_test_queries']}")
-    print(f"  With <think> tag     : {reasoning_eval['summary']['n_with_think']}/{reasoning_eval['n_test_queries']}")
-    print(f"  Avg reasoning length : {reasoning_eval['summary']['avg_reasoning_length']:.0f} chars")
-    print(f"  Response quality     : {reasoning_eval['quality_assessment']['response_quality']}")
-    print(f"  Reasoning quality    : {reasoning_eval['quality_assessment']['reasoning_quality']}")
-
-    # ------------------------------------------------------------------
-    # 12.8 Final report
-    # ------------------------------------------------------------------
-    summary = monitor.summary()
-    som_final = kls.som.get_metrics()
-    kls_state = kls.get_state_metrics()
-
-    report = {
-        "version": "V6.5-final-restructured",
-        "timestamp": datetime.now().isoformat(),
-        "user_requirements_checklist": {
-            "HF_TOKEN_deleted_after_use": "PENDING (will delete after upload)",
-            "streaming_datasets_active": True,
-            "xeon_runtime_active": True,
-            "removed_pre_v64_modules": True,
-            "removed_pre_v64_scripts": True,
-            "kohonen_refactored_removed": True,
-            "kohonen_learning_system_moved_up": True,
-            "vqvae2_active_in_compression_pipeline": True,
-            "reasoning_engine_integrated_to_kls": True,
-            "ewc_w8a8_dequant_benchmark_active": True,
-            "logic_and_bugfixes_verified": verification["all_pass"],
-            "exhausted_6_datasets": True,
-            "metrics_reasoning_response_verified": True,
-            "mtp_active_in_val": MTP_ACTIVE_IN_VAL,
-            "entropy_regularizer": MTP_ENTROPY_BETA,
-        },
-        "config": {
-            "BATCH_SIZE": BATCH_SIZE,
-            "datasets_to_exhaust": V65_DATASETS_TO_EXHAUST,
-            "SAMPLES_PER_DATASET_PHASE": SAMPLES_PER_DATASET_PHASE,
-            "N_PHASES": N_PHASES,
-            "TOTAL_SAMPLES": TOTAL_SAMPLES,
-            "EPOCHS": EPOCHS,
-            "SOM_GRID": list(SOM_GRID),
-            "n_neurons": n_neurons,
-            "HIDDEN_DIM": HIDDEN_DIM,
-            "VOCAB_SIZE": VOCAB_SIZE,
-            "MAX_SEQ_LEN": MAX_SEQ_LEN,
-            "T_max": T_MAX,
-            "N_start": N_START,
-            "lambda_ewc": LAMBDA_EWC,
-            "MTP_K": MTP_K,
-            "MTP_ENTROPY_BETA": MTP_ENTROPY_BETA,
-            "MTP_ACTIVE_IN_VAL": MTP_ACTIVE_IN_VAL,
-            "VQVAE2_active": True,
-            "reasoning_engine_active": True,
-        },
-        "xeon_status": XEON_STATUS,
-        "fp16_benchmark": FP16_BENCH,
-        "training": {
-            "duration_s": float(train_duration),
-            "n_steps": int(step),
-            "n_epochs": EPOCHS,
-            "n_phases": N_PHASES,
-            "samples_per_dataset_actual": samples_per_dataset_actual,
-            "total_samples_processed": sum(samples_per_dataset_actual.values()),
-        },
-        "summary": summary,
-        "kohonen_final": som_final,
-        "kls_state": kls_state,
-        "verification": verification,
-        "ewc_w8a8_benchmark_summary": ewc_w8a8_benchmark["analysis"],
-        "reasoning_eval_summary": reasoning_eval["summary"],
-        "module_analysis_summary": {
-            "n_removed_modules": sum(1 for v in module_analysis["removed_modules"].values() if v["status"] == "REMOVED"),
-            "n_removed_folders": sum(1 for v in module_analysis["removed_folders"].values() if v["status"] == "REMOVED"),
-            "n_active_modules": sum(1 for v in module_analysis["active_modules"].values() if v["activity"] == "active"),
-        },
-        "script_activity_summary": {
-            "n_scripts": len(script_activity),
-            "active_v65": sum(1 for v in script_activity.values() if v["classification"] == "active_v65"),
-            "active_v64": sum(1 for v in script_activity.values() if v["classification"] == "active_v64"),
-            "upload_utility": sum(1 for v in script_activity.values() if v["classification"] == "upload_utility"),
-        },
-        "math_analysis": {
-            "text_to_4d": "SVD: M @ V[:3].T -> centroid 3D + w = time_step/T_max (LINEAR)",
-            "bmu_distance": "||W - x||^2 (L2 squared in R^4)",
-            "neighborhood": "Lambda(d, sigma) = exp(-d^2 / (2*sigma^2))",
-            "weight_update": "dW = alpha * Lambda * (x - W)",
-            "sigma_decay": "sigma_t = sigma0 * exp(-t/1000)",
-            "alpha_decay": "alpha_t = alpha0 * exp(-t/2000)",
-            "ewc_only_dim4": "penalty = lambda * F * (W_w - W*_w)",
-            "vqvae2_loss": "L = recon_loss + vq_loss (commitment top + bottom + diversity)",
-            "vqvae2_ema": "EMA codebook update + dead code restart + Goose VQ",
-            "mtp_loss": "L = sum_k(alpha_k * L_k) - beta * H(alpha)",
-            "ewc_w8a8_dequant": "W_float = (W_int8 * scale) / smooth_scale; EWC uses W_float for (p-w*)^2",
-        },
-        "datasets_used": V65_DATASETS_TO_EXHAUST,
-    }
-
-    REPORT_PATH.write_text(json.dumps(report, indent=2, ensure_ascii=False, default=str))
-    logger.info(f"[V6.5] Report saved: {REPORT_PATH}")
-
-    metrics_full = {
-        "version": "V6.5-final-restructured",
-        "steps": monitor.steps,
-        "summary": summary,
-        "alerts": monitor.alerts,
-    }
-    METRICS_PATH.write_text(json.dumps(metrics_full, indent=2, ensure_ascii=False, default=str))
-    logger.info(f"[V6.5] Metrics saved: {METRICS_PATH}")
-
-    # Print final summary
-    print("\n" + "=" * 80)
-    print("V6.5 — TREINO CONCLUÍDO")
-    print("=" * 80)
-    print(f"  Steps                : {step}")
-    print(f"  Duration             : {train_duration:.1f}s")
-    print(f"  Total samples        : {sum(samples_per_dataset_actual.values())}")
-    print(f"  Final loss           : {summary.get('final_loss', 0):.3f}")
-    print(f"  Mean acc             : {summary.get('mean_acc', 0):.3f}")
-    print(f"  Sigma (start→end)    : {summary.get('sigma_start', 0):.3f} → {summary.get('sigma_end', 0):.3f}")
-    print(f"  VQ-VAE-2 mean loss   : {summary.get('vqvae2_mean_total_loss', 0):.3f}")
-    print(f"  VQ-VAE-2 final loss  : {summary.get('vqvae2_final_total_loss', 0):.3f}")
-    evo = summary.get("evolution_phase1_to_phase2", {})
-    print(f"  Evolution P1→P2:")
-    print(f"    acc P1 mean        : {evo.get('acc_phase1_mean', 0):.3f}")
-    print(f"    acc P2 mean        : {evo.get('acc_phase2_mean', 0):.3f}")
-    print(f"    vqvae2 P1 mean     : {evo.get('vqvae2_phase1_mean', 0):.3f}")
-    print(f"    vqvae2 P2 mean     : {evo.get('vqvae2_phase2_mean', 0):.3f}")
-    print(f"  RSS max              : {summary.get('rss_max_mb', 0):.0f}MB")
-    print(f"  Alerts               : {summary.get('n_alerts', 0)}")
-    print(f"  Hypothesis trained   : {kls.classifier_trained}")
-    print(f"  EWC reference set    : {som_final['has_ewc_reference']}")
-    print(f"  VQ-VAE-2 calls       : {kls_state['kls']['vqvae2_n_calls']}")
-    print(f"  Reasoning n_history  : {reasoning_eval['summary']['reasoning_engine_n_history']}")
-    print(f"  Verification         : {verification['n_pass']}/{verification['n_pass'] + verification['n_fail']} PASS")
-    print("=" * 80)
-    print(f"\n  Report           : {REPORT_PATH}")
-    print(f"  Metrics          : {METRICS_PATH}")
-    print(f"  Module analysis  : {MODULE_ANALYSIS_PATH}")
-    print(f"  Script activity  : {SCRIPT_ACTIVITY_PATH}")
-    print(f"  EWC+W8A8 bench   : {EWC_W8A8_BENCH_PATH}")
-    print(f"  Reasoning eval   : {REASONING_EVAL_PATH}\n")
-
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())