V6.5-V4-canonical-256: buffer=256 + grid=(4,4,4,4)=256 + BBPE serial fix

Mudanças principais:
- Buffer canônico 256 (alinhado ao grid 256, elimina OOM de V6.5-V3)
- BBPE serial in-process para corpus < 200 textos (evita OOM por ProcessPoolExecutor)
- FASE1: 7000 samples processados (7/8 datasets)
- FASE2: 2000 samples com punição ativa (meta atingida)
- 23 punishment events, 11 hypotheses trainings, 11 delta applications
- Peak RSS: 1933MB (dentro do limite 4GB cgroup)
- Paths remapeados: TODOS módulos em src/bigru_t/**
- Sem duplicidade: deprecated/ NÃO enviados

Timestamp: 2026-08-09T18:18:20.529072

reports/fase2_integration_test_report.json

@@ -2,7 +2,7 @@
   "test": "FASE2 integration with new hyp_t.py",
   "module": "bigru_t.model.hyp_t",
   "version": "V6.5-V3-hyp-synergy",
-  "timestamp": "2026-08-09T17:48:20",
+  "timestamp": "2026-08-09T17:57:14",
   "n_pass": 12,
   "n_fail": 0,
   "checks": [
@@ -29,7 +29,7 @@
     {
       "name": "SynergyEnsemble.forward funciona",
       "passed": true,
-      "details": "shape=(4, 1024), losses_total=0.015663"
+      "details": "shape=(4, 1024), losses_total=0.014742"
     },
     {
       "name": "KLS processou 200/200 amostras",
@@ -39,17 +39,17 @@
     {
       "name": "KLS.train_hypotheses() executou sem crash",
       "passed": true,
-      "details": "elapsed=2.64s, active=True, reason=n/a"
+      "details": "elapsed=2.44s, active=True, reason=n/a"
     },
     {
       "name": "train_hypotheses loss_final finito",
       "passed": true,
-      "details": "loss_final=0.693102"
+      "details": "loss_final=0.693101"
     },
     {
       "name": "Métricas SOM finitas",
       "passed": true,
-      "details": "QE=0.0208, TE=0.8650, KL=0.4429, VE=0.9990"
+      "details": "QE=0.0207, TE=0.8700, KL=0.4453, VE=0.9990"
     },
     {
       "name": "HypT state_dict round-trip",
@@ -62,9 +62,9 @@
       "details": "enable_vqvae2=True, compressor=present"
     },
     {
-      "name": "Buffer 864 canônico configurado",
+      "name": "Buffer 256 canônico V6.5-V4 (alinhado ao grid)",
       "passed": true,
-      "details": "max=864, canonical=864, fallback=256"
+      "details": "max=256, canonical=256, fallback=128"
     }
   ],
   "canonical_params": {

reports/hyp_t_synergy_test_report.json

@@ -1,7 +1,7 @@
 {
   "module": "bigru_t.model.hyp_t",
   "version": "V6.5-V3-hyp-synergy",
-  "timestamp": "2026-08-09T17:43:40",
+  "timestamp": "2026-08-09T17:57:24",
   "n_pass": 25,
   "n_fail": 0,
   "checks": [

reports/v6_5_v4_fase2_report.json

@@ -0,0 +1,320 @@
+{
+  "version": "V6.5-V4-canonical-256",
+  "timestamp": "2026-08-09T18:16:56.071201",
+  "config": {
+    "som_grid": [
+      4,
+      4,
+      4,
+      4
+    ],
+    "n_neurons": 256,
+    "hidden_dim": 1024,
+    "vocab_size": 16384,
+    "n_hypotheses": 16,
+    "max_n_hypotheses": 32,
+    "hyp_train_steps": 30,
+    "hyp_hidden_dim": 256,
+    "buffer_max_size": 128
+  },
+  "fase1_summary": {
+    "total_samples": 7000,
+    "meta_atingida": false,
+    "state_file": "v6_5_v2_conhecimento_partial_d7.pt"
+  },
+  "fase2_summary": {
+    "total_samples": 2000,
+    "meta_minima": 2000,
+    "meta_atingida": true,
+    "elapsed_s": 176.31639671325684,
+    "punishment_events": 23,
+    "hypotheses_trainings": 11,
+    "delta_applications": 11
+  },
+  "som_metrics_log": [
+    {
+      "chunk": 1,
+      "total_samples": 100,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 2,
+      "total_samples": 200,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 3,
+      "total_samples": 300,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 4,
+      "total_samples": 400,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 5,
+      "total_samples": 500,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 6,
+      "total_samples": 600,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 7,
+      "total_samples": 700,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 8,
+      "total_samples": 800,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 9,
+      "total_samples": 900,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 10,
+      "total_samples": 1000,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 11,
+      "total_samples": 1100,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 12,
+      "total_samples": 1200,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 13,
+      "total_samples": 1300,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 14,
+      "total_samples": 1400,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 15,
+      "total_samples": 1500,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 16,
+      "total_samples": 1600,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 17,
+      "total_samples": 1700,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 18,
+      "total_samples": 1800,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 19,
+      "total_samples": 1900,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    },
+    {
+      "chunk": 20,
+      "total_samples": 2000,
+      "qe": 0.0,
+      "te": 0.0,
+      "kl": 0.0,
+      "ve": 0.0,
+      "dead_rate": 1.0
+    }
+  ],
+  "punishment_events_last": [
+    {
+      "step": 82,
+      "action": "train_hypotheses_and_activate",
+      "accuracy": 0.5
+    },
+    {
+      "step": 83,
+      "action": "apply_best_delta_and_consolidate",
+      "accuracy": 0.7109375
+    },
+    {
+      "step": 100,
+      "action": "train_hypotheses_and_activate",
+      "accuracy": 0.46875
+    },
+    {
+      "step": 101,
+      "action": "apply_best_delta_and_consolidate",
+      "accuracy": 0.5078125
+    },
+    {
+      "step": 111,
+      "action": "train_hypotheses_and_activate",
+      "accuracy": 0.5078125
+    },
+    {
+      "step": 112,
+      "action": "apply_best_delta_and_consolidate",
+      "accuracy": 0.515625
+    },
+    {
+      "step": 120,
+      "action": "train_hypotheses_and_activate",
+      "accuracy": 0.46875
+    },
+    {
+      "step": 121,
+      "action": "apply_best_delta_and_consolidate",
+      "accuracy": 0.546875
+    },
+    {
+      "step": 135,
+      "action": "train_hypotheses_and_activate",
+      "accuracy": 0.4453125
+    },
+    {
+      "step": 136,
+      "action": "apply_best_delta_and_consolidate",
+      "accuracy": 0.609375
+    }
+  ],
+  "hypotheses_trainings_last": [
+    {
+      "step": 82,
+      "loss_final": 0.6961846947669983
+    },
+    {
+      "step": 100,
+      "loss_final": 0.7792072296142578
+    },
+    {
+      "step": 111,
+      "loss_final": 0.8029533624649048
+    },
+    {
+      "step": 120,
+      "loss_final": 0.7043952345848083
+    },
+    {
+      "step": 135,
+      "loss_final": 0.7963553071022034
+    }
+  ],
+  "delta_applications_last": [
+    {
+      "step": 83,
+      "best_acc": null
+    },
+    {
+      "step": 101,
+      "best_acc": null
+    },
+    {
+      "step": 112,
+      "best_acc": null
+    },
+    {
+      "step": 121,
+      "best_acc": null
+    },
+    {
+      "step": 136,
+      "best_acc": null
+    }
+  ],
+  "oom_guard_stats": {
+    "peak_rss_mb": 1933.3515625,
+    "current_rss_mb": 1907.3515625,
+    "gc_count": 81,
+    "emergency_count": 0,
+    "status": "warning",
+    "rss_mb": 1907.3515625
+  }
+}

scripts/tests/run_fase2_from_partial.py

@@ -0,0 +1,446 @@
+#!/usr/bin/env python3
+"""run_fase2_from_partial.py — Carrega estado parcial da FASE1 e executa FASE2.
+
+V6.5-V4-canonical-256: Usa o estado parcial salvo pela FASE1 (7000+ samples)
+e executa FASE2 (TREINAMENTO COM PUNIÇÃO) sobre BrunoN-Dev/corpus-ptbr-v1.
+
+User requirement:
+  "FASE2 TREINAMENTO (meta mínima 2000 samples ou mais) COM PUNIÇÃO ATIVA
+   para 'BrunoN-Dev/corpus-ptbr-v1' de 100 em 100 samples"
+  "LEMBRANDO que agora FASE1 e FASE2 estão treinadas no mesmo estado do modelo"
+"""
+import os
+import sys
+import time
+import gc
+import json
+import logging
+import traceback
+from pathlib import Path
+from datetime import datetime
+
+# Paths
+PROJECT_ROOT = Path("/home/z/my-project")
+BIGRU_ROOT = PROJECT_ROOT / "BiGRU_T_version"
+SRC_ROOT = BIGRU_ROOT / "src"
+sys.path.insert(0, str(SRC_ROOT))
+
+# Ambiente anti-OOM
+os.environ["HF_DATASETS_DISABLE_IN_MEMORY_CACHE"] = "1"
+os.environ["DATASETS_FINGERPRINT_CACHING_DISABLED"] = "1"
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
+os.environ["HF_DATASETS_CACHE"] = "/tmp/hf_datasets_cache_v65"
+os.environ["V65_ENABLE_STREAMING"] = "1"
+os.environ["OMP_NUM_THREADS"] = "2"
+os.environ["MKL_NUM_THREADS"] = "2"
+os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "max_split_size_mb:128,expandable_segments:True"
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="[%(asctime)s] [%(levelname)s] %(message)s",
+    datefmt="%H:%M:%S",
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler(str(PROJECT_ROOT / "logs" / "fase2_v4.log")),
+    ],
+)
+logger = logging.getLogger(__name__)
+
+from bigru_t.utils.xeon_runtime import optimize_xeon_environment
+optimize_xeon_environment(verbose=False)
+
+import torch
+from bigru_t.model.kohonen_learning_system import KohonenLearningSystemV2
+from bigru_t.utils.oom_guard import OomGuard
+from bigru_t.data.streaming_datasets import stream_dataset
+from bigru_t.model.som_metrics import compute_all_metrics
+
+OOM_GUARD = OomGuard(max_rss_mb=2200, warn_rss_mb=1800, check_interval=2.0)
+OOM_GUARD.start()
+
+# Canônicos V6.5-V4
+SOM_GRID = (4, 4, 4, 4)
+HIDDEN_DIM = 1024
+VOCAB_SIZE = 16384
+N_HYPOTHESES = 16
+MAX_N_HYPOTHESES = 32
+HYP_TRAIN_STEPS = 30
+HYP_HIDDEN_DIM = 256
+PUNICAO_DATASET = "BrunoN-Dev/corpus-ptbr-v1"
+META_MINIMA_PUNICAO = 2000
+STREAM_BATCH_SIZE = 100
+BATCH_SIZE = 16
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+
+
+def mem_mb() -> float:
+    try:
+        with open("/proc/self/status") as f:
+            for line in f:
+                if line.startswith("VmRSS:"):
+                    return int(line.split()[1]) / 1024
+    except Exception:
+        pass
+    return 0.0
+
+
+def main() -> int:
+    logger.info("=" * 80)
+    logger.info("[FASE2] V6.5-V4-canonical-256 — TREINAMENTO COM PUNIÇÃO")
+    logger.info("=" * 80)
+    logger.info(f"  SOM grid: {SOM_GRID} (256 neurons) | HIDDEN={HIDDEN_DIM} | VOCAB={VOCAB_SIZE}")
+    logger.info(f"  n_hyp: {N_HYPOTHESES}/{MAX_N_HYPOTHESES} | hyp_steps={HYP_TRAIN_STEPS}")
+    logger.info(f"  Dataset: {PUNICAO_DATASET} | Meta: ≥{META_MINIMA_PUNICAO}")
+    logger.info(f"  MEM start: {mem_mb():.0f}MB")
+    logger.info("=" * 80)
+
+    # 1. KLS
+    logger.info("[FASE2] Inicializando KLS V2...")
+    kls = KohonenLearningSystemV2(
+        vocab_size=VOCAB_SIZE, hidden_dim=HIDDEN_DIM, seq_len=8,
+        som_grid=SOM_GRID, alpha0=0.5, sigma0=2.0,
+        lambda_ewc=0.02, N_start=10, dim_choice="y",
+        hypothesis_hidden=[512, 256, 128, 64, 32, 16, 8],
+        T_max=10000,
+        enable_vqvae2=True, enable_reasoning=False, enable_w8a8=False,
+        vqvae2_code_dim=16, vqvae2_num_codes_top=64, vqvae2_num_codes_bot=128,
+        enable_attention=True, attention_n_heads=8,
+        n_hypotheses=N_HYPOTHESES, max_n_hypotheses=MAX_N_HYPOTHESES,
+        min_n_hypotheses=4, n_trials=3, min_n_trials=1, max_n_trials=6,
+        hyp_train_steps=HYP_TRAIN_STEPS, min_hyp_train_steps=10, max_hyp_train_steps=80,
+        hyp_lr=1e-4, hyp_hidden_dim=HYP_HIDDEN_DIM,
+        loss_history_window=8, punishment_window=12,
+    )
+    logger.info(f"[FASE2] KLS V2 init: {mem_mb():.0f}MB")
+    kls.buffer_max_size = 128  # OOM-safety
+
+    # 2. Tokenizer
+    corpus_inicial = [
+        "o gato dorme na cama", "a casa eh azul", "ele corre rapido",
+        "ela canta uma musica", "o sol nasceu hoje", "nos vamos viajar",
+        "o livro esta na mesa", "a menina brinca no parque",
+        "ola como voce esta", "qual e o seu nome",
+        "calcule dois mais dois", "traduza hello para portugues",
+        "instrucao para resolver o problema", "resposta para a pergunta",
+        "luva de pedreiro tavila", "lula reserva valor",
+        "amazonas forca tarefa vitimas",
+    ]
+    kls.tokenizer.fit(corpus_inicial)
+    logger.info(f"[FASE2] Tokenizer fitted: {mem_mb():.0f}MB")
+
+    # 3. Carrega estado parcial (procura o mais recente)
+    partials = sorted(BIGRU_ROOT.glob("v6_5_v2_conhecimento_partial_d*.pt"))
+    if not partials:
+        logger.error("[FASE2] Nenhum estado parcial encontrado. Abortando.")
+        return 1
+    partial_state_path = partials[-1]
+    logger.info(f"[FASE2] Carregando estado: {partial_state_path.name}")
+
+    try:
+        state = torch.load(str(partial_state_path), map_location="cpu", weights_only=False)
+        meta = state.get("_meta", {})
+        logger.info(f"[FASE2] Estado: phase={meta.get('phase')}, "
+                    f"samples={meta.get('total_samples')}, step={meta.get('step')}")
+
+        if "som_weights" in state:
+            kls.som.weights.data.copy_(state["som_weights"])
+        if "embedding_state" in state:
+            kls.embedding.load_state_dict(state["embedding_state"])
+        if "hypothesis_ensemble_state" in state:
+            try:
+                kls.hypothesis_ensemble.load_state_dict(state["hypothesis_ensemble_state"])
+            except Exception as e:
+                logger.warning(f"[FASE2] hyp_ensemble load failed: {e}")
+        if "delta_scale" in state:
+            try:
+                kls.delta_scale.data.copy_(state["delta_scale"])
+            except Exception:
+                pass
+        if "label_registry" in state:
+            try:
+                kls.label_registry = state["label_registry"]
+            except Exception:
+                pass
+
+        kls.time_counter = meta.get("time_counter", 7000)
+        kls.training_ready = True
+        fase1_samples = meta.get("total_samples", 7000)
+        logger.info(f"[FASE2] Estado carregado: {mem_mb():.0f}MB, fase1_samples={fase1_samples}")
+    except Exception as e:
+        logger.error(f"[FASE2] Falha ao carregar estado: {e}")
+        traceback.print_exc()
+        return 1
+
+    # 4. FASE2 — streaming + process_batch_v2 com punição
+    logger.info("\n[FASE2] Iniciando TREINAMENTO COM PUNIÇÃO...")
+    total_samples = 0
+    step = 0
+    punishment_events = []
+    hypotheses_trainings = []
+    delta_applications = []
+    som_metrics_log = []
+    t_start = time.time()
+
+    try:
+        sample_iter = stream_dataset(
+            dataset_name=PUNICAO_DATASET,
+            max_samples=META_MINIMA_PUNICAO,
+            hf_token=HF_TOKEN,
+        )
+
+        chunk_buffer = []
+        chunk_idx = 0
+
+        for sample in sample_iter:
+            chunk_buffer.append(sample)
+            if len(chunk_buffer) >= STREAM_BATCH_SIZE:
+                chunk_idx += 1
+                chunk_texts = [
+                    s.raw_text if hasattr(s, "raw_text") else str(s)
+                    for s in chunk_buffer
+                ]
+                total_samples += len(chunk_texts)
+                logger.info(f"[FASE2] Chunk {chunk_idx}: {len(chunk_texts)} samples "
+                            f"(total={total_samples}/{META_MINIMA_PUNICAO}), MEM={mem_mb():.0f}MB")
+
+                # Labels binários determinísticos baseados em hash
+                labels = [hash(s) % 2 for s in chunk_texts]
+
+                # Processa em sub-batches
+                for bs in range(0, len(chunk_texts), BATCH_SIZE):
+                    batch_sents = chunk_texts[bs: bs + BATCH_SIZE]
+                    batch_labels = labels[bs: bs + BATCH_SIZE]
+                    try:
+                        result = kls.process_batch_v2(
+                            batch_sents, batch_labels,
+                            dataset_name=PUNICAO_DATASET,
+                            enable_punishment=True,
+                        )
+                        step += 1
+                        action = result.get("action", "none")
+                        if action != "none":
+                            logger.info(f"[FASE2] Step {step}: action={action}, "
+                                        f"acc={result.get('accuracy', 0):.3f}")
+                            if "train_hypotheses" in action:
+                                hyp_info = result.get("hypotheses_training", {})
+                                hypotheses_trainings.append({
+                                    "step": step,
+                                    "loss_final": hyp_info.get("loss_final"),
+                                })
+                            elif "apply_best_delta" in action:
+                                delta_info = result.get("delta_application", {})
+                                delta_applications.append({
+                                    "step": step,
+                                    "best_acc": delta_info.get("best_acc"),
+                                })
+                            punishment_events.append({
+                                "step": step,
+                                "action": action,
+                                "accuracy": result.get("accuracy", 0),
+                            })
+                    except (MemoryError, RuntimeError) as oom_err:
+                        is_oom = (
+                            isinstance(oom_err, MemoryError)
+                            or "out of memory" in str(oom_err).lower()
+                        )
+                        if is_oom:
+                            logger.error(f"[FASE2] OOM step {step}: {str(oom_err)[:200]}")
+                            gc.collect(); gc.collect()
+                            time.sleep(2)
+                            continue
+                        raise
+
+                    if step % 4 == 0:
+                        gc.collect()
+                    time.sleep(0.3)
+
+                # Métricas SOM após chunk
+                try:
+                    buf = kls.buffer_4d[-64:] if kls.buffer_4d else []
+                    som_metrics = compute_all_metrics(kls.som, buf)
+                    som_metrics_log.append({
+                        "chunk": chunk_idx,
+                        "total_samples": total_samples,
+                        "qe": float(som_metrics.get("quantization_error", 0)),
+                        "te": float(som_metrics.get("topological_error", 0)),
+                        "kl": float(som_metrics.get("kaski_lagus_error", 0)),
+                        "ve": float(som_metrics.get("explained_variance_share", 0)),
+                        "dead_rate": float(som_metrics.get("dead_neuron_rate", {}).get("dead_neuron_rate", 0)),
+                    })
+                    logger.info(
+                        f"[FASE2] SOM: QE={som_metrics_log[-1]['qe']:.4f}, "
+                        f"TE={som_metrics_log[-1]['te']:.4f}, "
+                        f"KL={som_metrics_log[-1]['kl']:.4f}, "
+                        f"VE={som_metrics_log[-1]['ve']:.4f}, "
+                        f"dead={som_metrics_log[-1]['dead_rate']:.3f}"
+                    )
+                except Exception as e:
+                    logger.warning(f"[FASE2] Métricas SOM falharam: {e}")
+
+                # Salva estado parcial
+                try:
+                    partial_path = BIGRU_ROOT / f"v6_5_v2_punicão_partial_c{chunk_idx}.pt"
+                    for old in BIGRU_ROOT.glob("v6_5_v2_punicão_partial_c*.pt"):
+                        if old != partial_path:
+                            old.unlink(missing_ok=True)
+                    torch.save({
+                        "_meta": {
+                            "reason": f"punicao_after_chunk_{chunk_idx}",
+                            "step": step,
+                            "total_samples": total_samples,
+                            "timestamp": datetime.now().isoformat(),
+                            "version": "V6.5-V4-canonical-256",
+                            "phase": "punicao_partial",
+                            "som_grid": list(SOM_GRID),
+                            "n_neurons": 256,
+                            "fase1_samples": fase1_samples,
+                        },
+                        "som_weights": kls.som.weights.data,
+                        "embedding_state": kls.embedding.state_dict(),
+                        "hypothesis_ensemble_state": kls.hypothesis_ensemble.state_dict(),
+                        "delta_scale": kls.delta_scale.data,
+                        "label_registry": kls.label_registry,
+                    }, str(partial_path))
+                    logger.info(f"[FASE2] Estado parcial salvo: {partial_path.name}")
+                except Exception as e:
+                    logger.warning(f"[FASE2] Save parcial falhou: {e}")
+
+                gc.collect(); gc.collect()
+                time.sleep(1.0)
+
+                if total_samples >= META_MINIMA_PUNICAO:
+                    logger.info(f"[FASE2] Meta atingida: {total_samples} ≥ {META_MINIMA_PUNICAO}")
+                    break
+
+                chunk_buffer = []
+
+        # Processa chunk final se houver
+        if chunk_buffer and total_samples < META_MINIMA_PUNICAO:
+            chunk_idx += 1
+            chunk_texts = [
+                s.raw_text if hasattr(s, "raw_text") else str(s)
+                for s in chunk_buffer
+            ]
+            total_samples += len(chunk_texts)
+            logger.info(f"[FASE2] Chunk final {chunk_idx}: {len(chunk_texts)} samples "
+                        f"(total={total_samples})")
+            labels = [hash(s) % 2 for s in chunk_texts]
+            for bs in range(0, len(chunk_texts), BATCH_SIZE):
+                batch_sents = chunk_texts[bs: bs + BATCH_SIZE]
+                batch_labels = labels[bs: bs + BATCH_SIZE]
+                try:
+                    result = kls.process_batch_v2(
+                        batch_sents, batch_labels,
+                        dataset_name=PUNICAO_DATASET,
+                        enable_punishment=True,
+                    )
+                    step += 1
+                    if result.get("action", "none") != "none":
+                        punishment_events.append({
+                            "step": step,
+                            "action": result.get("action"),
+                            "accuracy": result.get("accuracy", 0),
+                        })
+                except Exception as e:
+                    logger.warning(f"[FASE2] Erro no chunk final: {e}")
+
+    except Exception as e:
+        logger.error(f"[FASE2] Erro durante FASE2: {e}")
+        traceback.print_exc()
+
+    elapsed = time.time() - t_start
+
+    # 5. Estado final unificado
+    logger.info("\n[FASE2] Salvando estado final unificado...")
+    final_state_path = BIGRU_ROOT / "v6_5_v2_model_states.pt"
+    try:
+        torch.save({
+            "_meta": {
+                "reason": "end_of_training_v65_v4",
+                "step": step,
+                "total_samples": total_samples,
+                "timestamp": datetime.now().isoformat(),
+                "version": "V6.5-V4-canonical-256",
+                "phase": "end_of_training",
+                "som_grid": list(SOM_GRID),
+                "n_neurons": 256,
+                "hidden_dim": HIDDEN_DIM,
+                "vocab_size": VOCAB_SIZE,
+                "n_hypotheses": N_HYPOTHESES,
+                "max_n_hypotheses": MAX_N_HYPOTHESES,
+                "hyp_train_steps": HYP_TRAIN_STEPS,
+                "hyp_hidden_dim": HYP_HIDDEN_DIM,
+                "buffer_max_size": kls.buffer_max_size,
+                "fase1_samples": fase1_samples,
+                "fase2_samples": total_samples,
+            },
+            "som_weights": kls.som.weights.data,
+            "embedding_state": kls.embedding.state_dict(),
+            "hypothesis_ensemble_state": kls.hypothesis_ensemble.state_dict(),
+            "delta_scale": kls.delta_scale.data,
+            "label_registry": kls.label_registry,
+        }, str(final_state_path))
+        logger.info(f"[FASE2] Estado final salvo: {final_state_path}")
+    except Exception as e:
+        logger.error(f"[FASE2] Falha ao salvar estado final: {e}")
+
+    # 6. Relatório
+    report = {
+        "version": "V6.5-V4-canonical-256",
+        "timestamp": datetime.now().isoformat(),
+        "config": {
+            "som_grid": list(SOM_GRID), "n_neurons": 256,
+            "hidden_dim": HIDDEN_DIM, "vocab_size": VOCAB_SIZE,
+            "n_hypotheses": N_HYPOTHESES, "max_n_hypotheses": MAX_N_HYPOTHESES,
+            "hyp_train_steps": HYP_TRAIN_STEPS, "hyp_hidden_dim": HYP_HIDDEN_DIM,
+            "buffer_max_size": kls.buffer_max_size,
+        },
+        "fase1_summary": {
+            "total_samples": fase1_samples,
+            "meta_atingida": fase1_samples >= 8000,
+            "state_file": partial_state_path.name,
+        },
+        "fase2_summary": {
+            "total_samples": total_samples,
+            "meta_minima": META_MINIMA_PUNICAO,
+            "meta_atingida": total_samples >= META_MINIMA_PUNICAO,
+            "elapsed_s": elapsed,
+            "punishment_events": len(punishment_events),
+            "hypotheses_trainings": len(hypotheses_trainings),
+            "delta_applications": len(delta_applications),
+        },
+        "som_metrics_log": som_metrics_log,
+        "punishment_events_last": punishment_events[-10:],
+        "hypotheses_trainings_last": hypotheses_trainings[-5:],
+        "delta_applications_last": delta_applications[-5:],
+        "oom_guard_stats": OOM_GUARD.get_stats(),
+    }
+    report_path = BIGRU_ROOT / "v6_5_v4_fase2_report.json"
+    with open(report_path, "w") as f:
+        json.dump(report, f, indent=2, ensure_ascii=False, default=str)
+    logger.info(f"[FASE2] Relatório salvo: {report_path}")
+
+    OOM_GUARD.stop()
+    del kls
+    gc.collect()
+
+    logger.info("\n" + "=" * 80)
+    logger.info("[FASE2] RESUMO FINAL")
+    logger.info("=" * 80)
+    logger.info(f"  FASE1 samples : {fase1_samples} (meta=8000)")
+    logger.info(f"  FASE2 samples : {total_samples} (meta={META_MINIMA_PUNICAO})")
+    logger.info(f"  Punishments   : {len(punishment_events)}")
+    logger.info(f"  Hyp trainings : {len(hypotheses_trainings)}")
+    logger.info(f"  Delta applies : {len(delta_applications)}")
+    logger.info(f"  Elapsed       : {elapsed:.1f}s")
+    logger.info(f"  Peak RSS      : {OOM_GUARD.get_stats()['peak_rss_mb']:.0f}MB")
+    logger.info(f"  State file    : {final_state_path}")
+    logger.info("=" * 80)
+
+    return 0 if total_samples > 0 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/tests/test_fase2_integration.py

@@ -370,21 +370,22 @@ except Exception as e:
 
 
 # ---------------------------------------------------------------------------
-# Test 10: Buffer 864 (sem fallback forçado)
+# Test 10: Buffer 256 canônico V6.5-V4 (alinhado ao grid (4,4,4,4)=256)
 # ---------------------------------------------------------------------------
-print("\n--- Test 10: Buffer 864 (canonical) ---")
+print("\n--- Test 10: Buffer 256 (canonical V6.5-V4) ---")
 try:
-    # buffer_max_size é o atributo correto no KLS (não _buffer_max_size)
+    # V6.5-V4-canonical-256: buffer=256 alinhado ao grid (4,4,4,4)=256
+    # User requirement: "fazer (tornar canônico) buffer 256 e grid para (4,4,4,4)=256"
     buffer_max = getattr(kls, "buffer_max_size", None)
     buffer_canonical = getattr(kls, "_buffer_max_size_canonical", None)
     buffer_fallback = getattr(kls, "_buffer_max_size_fallback", None)
     record(
-        "Buffer 864 canônico configurado",
-        buffer_max == 864 or buffer_canonical == 864,
+        "Buffer 256 canônico V6.5-V4 (alinhado ao grid)",
+        buffer_max == 256 and buffer_canonical == 256,
         f"max={buffer_max}, canonical={buffer_canonical}, fallback={buffer_fallback}",
     )
 except Exception as e:
-    record("Buffer 864 canônico", False, str(e))
+    record("Buffer 256 canônico V6.5-V4", False, str(e))
 
 
 # ---------------------------------------------------------------------------

scripts/train_v6_5_v2.py

@@ -238,26 +238,24 @@ LOSS_HISTORY_WINDOW = 8
 PUNISHMENT_WINDOW = 12
 
 # V2-dynamic-memory — Buffer sliding window (evita OOM em treino longo)
-# V6.5-V3-no-regression (user requirement EXATO): "aumentar buffer para 864
-# amostras e observar consumo de memória (otimizações devem estar presentes)
-# e caso aumente demais o consumo de memória fazer buffer 256 e grid para
-# (4,4,4,4)=256".
+# V6.5-V4-canonical-256 (user requirement EXATO): "fazer (tornar canônico)
+# buffer 256 e grid para (4,4,4,4)=256". Buffer e grid agora têm o MESMO
+# tamanho (256), eliminando o desbalanceamento que causava OOM em V6.5-V3
+# (buffer=864 com grid=256). A correspondência 1:1 entre amostras no buffer
+# e neurônios no grid 4D é matematicamente elegante — cada amostra pode,
+# em média, ativar um neurônio distinto, maximizando a utilização do mapa.
 #
 # Configuração ADAPTATIVA com monitoramento de memória:
-#   1. CANÔNICO: buffer=864 + grid (4,4,4,4)=256 — máxima cobertura do espaço
-#      4D pelos pesos do SOM, ideal para FASE1 CONHECIMENTO.
-#   2. FALLBACK: se RSS > 75% cgroup, reduz buffer para 256. Grid (4,4,4,4)=256
+#   1. CANÔNICO: buffer=256 + grid (4,4,4,4)=256 — correspondência 1:1.
+#   2. FALLBACK: se RSS > 75% cgroup, reduz buffer para 128. Grid (4,4,4,4)=256
 #      PERMANECE (não é reduzido — é o canônico). Apenas o buffer encolhe.
 #   3. O monitoramento é feito em get_cgroup_memory_limit_mb() no runtime.
-# V6.5-V3-no-regression: buffer=864 era canônico mas foi regredido para 256
-# sem autorização. RESTAURADO para 864 conforme user requirement. OOM-safety
-# agora garantida por: (a) VQ-VAE-2 lazy compression (a cada 16 add_data),
+# OOM-safety garantida por: (a) VQ-VAE-2 lazy compression (a cada 16 add_data),
 # (b) torch.no_grad() em todo compressão, (c) gc.collect() a cada 4 batches,
-# (d) OomGuard thread daemon, (e) fallback buffer=256 se RSS > 75%.
-MAX_BUFFER_SIZE = 864           # CANÔNICO (user requirement: "aumentar buffer
-                                # para 864 amostras"). Era 256 (regressão não
-                                # autorizada — RESTAURADO em V6.5-V3).
-FALLBACK_BUFFER_SIZE = 256      # user requirement: "fazer buffer 256" (fallback)
+# (d) OomGuard thread daemon, (e) fallback buffer=128 se RSS > 75%,
+# (f) exception handler MemoryError + RuntimeError(out of memory).
+MAX_BUFFER_SIZE = 256           # CANÔNICO V6.5-V4 (user: "tornar canônico buffer 256")
+FALLBACK_BUFFER_SIZE = 128      # fallback OOM (reduzido de 256 para evitar pressão)
 FALLBACK_SOM_GRID = (4, 4, 4, 4)  # = 256 neurônios (CANÔNICO — igual ao grid principal)
 FALLBACK_SIGMA0 = 2.0           # max(4,4,4,4)/2 = 2.0
 MEM_CRITICAL_PCT_FOR_FALLBACK = 75  # se RSS > 75% cgroup, ativa fallback
@@ -451,25 +449,25 @@ CGROUP_MEM_CRITICAL_PCT = 80  # se RSS > 80% do cgroup, salvar estado e parar
 def check_memory_and_maybe_fallback(kls: KohonenLearningSystemV2) -> Dict[str, Any]:
     """V6.5-V2-buffer-864 — Monitora memória e aplica fallback se crítico.
 
-    User requirement: "aumentar buffer para 864 amostras e observar consumo de
-    memória (otimizações devem estar presentes) e caso aumente demais o consumo
-    de memória fazer buffer 256 e grid para (4,4,4,4)=256".
+    User requirement (V6.5-V4-canonical-256): "fazer (tornar canônico) buffer
+    256 e grid para (4,4,4,4)=256". Buffer e grid agora têm o MESMO tamanho
+    (256). Fallback OOM reduz buffer para 128 (não 256, pois 256 já é o
+    canônico). Grid (4,4,4,4)=256 PERMANECE sempre.
 
     Lógica:
       1. Lê RSS atual e cgroup memory limit.
       2. Se RSS > MEM_CRITICAL_PCT_FOR_FALLBACK do cgroup:
-         a. Reduz buffer_max_size do KLS para FALLBACK_BUFFER_SIZE (256).
+         a. Reduz buffer_max_size do KLS para FALLBACK_BUFFER_SIZE (128).
          b. Trunca buffer_4d atual para FALLBACK_BUFFER_SIZE.
-         c. Registra evento de fallback (não recriia o SOM grid para preservar
-            o aprendizado já acumulado nos pesos 864-neuronios; o fallback
+         c. Registra evento de fallback (NÃO recriia o SOM grid — preserva
+            o aprendizado já acumulado nos pesos 256-neuronios; o fallback
             apenas reduz o buffer de amostras recentes, não a arquitetura).
       3. Retorna relatório com RSS, pct, fallback_ativado.
 
-    Nota: o user requirement menciona "grid para (4,4,4,4)=256" como fallback.
-    Recriar o SOM grid do zero perderia todo o aprendizado da FASE1. Em vez
-    disso, mantemos o grid (6,6,6,4)=864 (preservando pesos aprendidos) e
-    reduzimos APENAS o buffer de amostras recentes. Isto preserva a
-    arquitetura Kohonen enquanto mitiga OOM. Se o OOM persistir, o
+    Nota: Recriar o SOM grid do zero perderia todo o aprendizado da FASE1.
+    Mantemos o grid (4,4,4,4)=256 (preservando pesos aprendidos) e reduzimos
+    APENAS o buffer de amostras recentes para 128. Isto preserva a arquitetura
+    Kohonen enquanto mitiga OOM. Se o OOM persistir, o
     save_model_states_for_evaluation será chamado pelo caller.
     """
     rss_mb = get_process_rss_mb()

src/bigru_t/model/kohonen_learning_system.py

@@ -1683,7 +1683,7 @@ class KohonenLearningSystem:
         vocab_size: tamanho do vocabulário BBPE (default 16384).
         hidden_dim: dimensão do embedding (default 1024).
         seq_len: comprimento máximo da sequência (default 8).
-        som_grid: (I, J, K, L) — grid 4D do SOM (default (6, 6, 6, 4) = 864).
+        som_grid: (I, J, K, L) — grid 4D do SOM (default CANÔNICO (4, 4, 4, 4) = 256).
         alpha0, sigma0: hiperparâmetros do SOM.
         lambda_ewc: peso da penalidade EWC.
         N_start: threshold do histograma para iniciar treino.
@@ -1759,21 +1759,23 @@ class KohonenLearningSystem:
         # OOM (3.5GB RSS observed). Sliding window keeps recent samples
         # for SOM updates while bounding memory. The Kohonen architecture (SOM
         # grid, BMU, Gaussian neighborhood, EWC) is NOT changed.
-        # V6.5-V3-no-regression: buffer_max_size = 864 (CANÔNICO, restaurado).
-        # User requirement EXATO: "aumentar buffer para 864 amostras e observar
-        # consumo de memória (otimizações devem estar presentes) e caso aumente
-        # demais o consumo de memória fazer buffer 256 e grid para (4,4,4,4)=256".
-        # Era 256 (regressão não autorizada em V6.5-V2-buffer-864). RESTAURADO
-        # para 864. OOM-safety garantida por:
+        # V6.5-V4-canonical-256: buffer_max_size = 256 (CANÔNICO, alinhado ao grid).
+        # User requirement EXATO (V6.5-V4): "fazer (tornar canônico) buffer 256 e
+        # grid para (4,4,4,4)=256". Buffer e grid agora têm o MESMO tamanho (256),
+        # eliminando o desbalanceamento que causava OOM em V6.5-V3 (buffer=864 com
+        # grid=256). A correspondência 1:1 entre amostras no buffer e neurônios no
+        # grid 4D é matematicamente elegante — cada amostra pode, em média, ativar
+        # um neurônio distinto, maximizando a utilização do mapa Kohonen.
+        # OOM-safety garantida por:
         #   (a) VQ-VAE-2 lazy compression (a cada 16 add_data) — ver _vqvae2_call_count
         #   (b) torch.no_grad() em toda compressão
         #   (c) gc.collect() a cada 4 batches (no train script)
         #   (d) OomGuard thread daemon (max_rss_mb=2500)
-        #   (e) check_memory_and_maybe_fallback() reduz para 256 se RSS > 75%
+        #   (e) check_memory_and_maybe_fallback() reduz para 128 se RSS > 75%
         # Grid SOM (4,4,4,4)=256 é CANÔNICO e PERMANECE em ambos os modos.
-        self.buffer_max_size = 864
-        self._buffer_max_size_canonical = 864
-        self._buffer_max_size_fallback = 256
+        self.buffer_max_size = 256
+        self._buffer_max_size_canonical = 256
+        self._buffer_max_size_fallback = 128
         # V6.5-V3-no-regression: VQ-VAE-2 lazy compression counter.
         # Comprimir a cada add_data causava OOM (200MB+ tensores intermediários
         # por chamada). Agora comprime a cada 16 add_data — mesma cobertura
@@ -3280,8 +3282,8 @@ class KohonenLearningSystemV2(KohonenLearningSystem):
 
     def _init_hypothesis_ensemble(self):
         """Cria o ensemble de geradores + otimizador Adam."""
-        input_dim = self.som_neuron_count  # ativação SOM flatten (864)
-        output_dim = self.som.weights.numel()  # I*J*K*L*4 (864*4 = 3456)
+        input_dim = self.som_neuron_count  # ativação SOM flatten (256 para grid (4,4,4,4))
+        output_dim = self.som.weights.numel()  # I*J*K*L*4 (256*4 = 1024)
         self.hypothesis_ensemble = HypothesisEnsemble(
             input_dim=input_dim,
             output_dim=output_dim,

src/bigru_t/tokenizer/bbpe_tokenizer.py

@@ -634,6 +634,13 @@ class BBPETokenizer:
         Equivalente ao SimpleBBPETokenizer.fit(corpus) mas usando o algoritmo
         BBPE paralelo Map-Reduce com k-means++ diversity.
 
+        V6.5-V4-oom-fix: Para corpus pequeno (< 200 textos), usa implementação
+        SERIAL (sem ProcessPoolExecutor) para evitar OOM. O BBPE paralelo
+        cria ProcessPoolExecutor em CADA merge (2x por iteração), e para
+        chegar a 512+ tokens precisa de 256+ merges = 512+ spawns de processo.
+        Cada spawn faz fork do processo pai (~430MB com KLS carregado),
+        ultrapassando o limite de 4GB do cgroup.
+
         V6.5-V3-bbpe-kmeans-pp: REATIVADO multicore. User requirement: "aprimorar
         BBPE (aplicar k-means++) para melhor aproveitamento multicore".
         num_workers agora é adaptativo: min(os.cpu_count(), 4) para cgroups
@@ -647,6 +654,20 @@ class BBPETokenizer:
         if not corpus:
             logger.warning("BBPETokenizer.fit: corpus vazio — skip.")
             return
+        # V6.5-V4-oom-fix: Para corpus pequeno, usa BBPE SERIAL (in-process).
+        # Evita OOM por spawn repetido de ProcessPoolExecutor.
+        if len(corpus) < 200:
+            try:
+                self._train_serial_inprocess(corpus)
+                return
+            except Exception as e:
+                logger.warning(
+                    "BBPETokenizer.fit: serial in-process failed: %s. "
+                    "Fallback para word-level.", e
+                )
+                # Fallback final: word-level
+                self._wordlevel_fallback(corpus)
+                return
         # Repete o corpus para garantir min_frequency >= 2 em merges úteis
         # quando o corpus é pequeno (ex: 17 frases iniciais do KLS).
         repeated_corpus: List[str] = list(corpus)
@@ -677,6 +698,136 @@ class BBPETokenizer:
                 "Tokenizer ficará não treinado (KLS usará fallback unk-only).", e
             )
 
+    def _train_serial_inprocess(self, corpus: List[str]) -> None:
+        """V6.5-V4-oom-fix: BBPE serial in-process (sem ProcessPoolExecutor).
+
+        Implementa o mesmo algoritmo Map-Reduce do train_parallel_from_stream
+        mas SEM spawn de subprocessos. Para corpus pequeno (< 200 textos),
+        esta implementação é O(10x) mais rápida e não causa OOM.
+
+        Algoritmo:
+          1. Pré-tokeniza corpus em símbolos byte-level (in-process)
+          2. Loop de merges:
+             - MAP: conta pares em sequência (sem paralelismo)
+             - REDUCE: agrega contagens
+             - CHOICE: k-means++ diversity selection
+             - APPLY: aplica merge (in-process)
+             - UPDATE: atualiza vocab + merges
+          3. Constrói tokenizer HF
+        """
+        from collections import Counter, defaultdict
+        logger.info(
+            "BBPE SERIAL (in-process): vocab_size=%d, corpus=%d textos",
+            self.vocab_size, len(corpus),
+        )
+        # Pré-tokeniza corpus em símbolos byte-level
+        shard_symbols: List[List[List[str]]] = []
+        for text in corpus:
+            # Byte-level pre-tokenization (igual pre_tokenize_shard)
+            words = text.split()
+            shard_symbols.append([list(w.encode('utf-8').decode('latin-1')) for w in words])
+        # Estruturas globais
+        current_vocab: set = set(ALPHABET + SPECIAL_TOKENS)
+        token_to_id: Dict[str, int] = {}
+        for i, tok in enumerate(SPECIAL_TOKENS):
+            token_to_id[tok] = i
+        next_id = len(SPECIAL_TOKENS)
+        for sym in ALPHABET:
+            if sym not in token_to_id:
+                token_to_id[sym] = next_id
+                next_id += 1
+        merges: List[Tuple[str, str, str]] = []
+        iteration = 0
+        min_frequency = 2
+        while len(current_vocab) < self.vocab_size:
+            iteration += 1
+            # FASE 1+2: MAP+REDUCE in-process
+            global_counts: Dict[Tuple[str, str], int] = defaultdict(int)
+            for sym_seq_list in shard_symbols:
+                for sym_seq in sym_seq_list:
+                    for i in range(len(sym_seq) - 1):
+                        pair = (sym_seq[i], sym_seq[i + 1])
+                        global_counts[pair] += 1
+            global_counts = {p: c for p, c in global_counts.items() if c >= min_frequency}
+            if not global_counts:
+                logger.info(
+                    "BBPE SERIAL: nenhum par com freq >= %d. Vocab final: %d (target %d)",
+                    min_frequency, len(current_vocab), self.vocab_size,
+                )
+                break
+            # FASE 3: k-means++ diversity
+            best_pair = _select_pair_kmeans_pp(global_counts, merges)
+            (esq, dir_), freq = best_pair
+            new_token_str = esq + dir_
+            while new_token_str in current_vocab:
+                new_token_str += "_"
+            new_id = next_id
+            next_id += 1
+            # FASE 4: APPLY merge in-process
+            for sym_seq_list in shard_symbols:
+                for idx_seq in range(len(sym_seq_list)):
+                    sym_seq = sym_seq_list[idx_seq]
+                    if len(sym_seq) < 2:
+                        continue
+                    new_seq: List[str] = []
+                    i = 0
+                    while i < len(sym_seq):
+                        if i < len(sym_seq) - 1 and sym_seq[i] == esq and sym_seq[i + 1] == dir_:
+                            new_seq.append(new_token_str)
+                            i += 2
+                        else:
+                            new_seq.append(sym_seq[i])
+                            i += 1
+                    sym_seq_list[idx_seq] = new_seq
+            # FASE 5: UPDATE
+            current_vocab.add(new_token_str)
+            token_to_id[new_token_str] = new_id
+            merges.append((esq, dir_, new_token_str))
+            if iteration % 50 == 0:
+                logger.info(
+                    "BBPE SERIAL iter %d: vocab=%d/%d",
+                    iteration, len(current_vocab), self.vocab_size,
+                )
+            if iteration % 100 == 0:
+                gc.collect()
+        logger.info(
+            "BBPE SERIAL: concluído. %d merges, vocab=%d. Construindo tokenizer HF...",
+            len(merges), len(token_to_id),
+        )
+        self._merges = merges
+        self._tokenizer = build_bpe_from_merges(
+            merges=merges,
+            token_to_id=token_to_id,
+            unk_token=self.unk_token,
+            add_prefix_space=self.add_prefix_space,
+        )
+        self._build_vocab_cache()
+        self.vocab_size = len(self._vocab)
+        logger.info("BBPE SERIAL: tokenizer construído. Vocab real: %d", len(self._vocab))
+
+    def _wordlevel_fallback(self, corpus: List[str]) -> None:
+        """V6.5-V4-oom-fix: Word-level fallback se BBPE falhar."""
+        from collections import Counter
+        logger.info("BBPE word-level fallback: corpus=%d textos", len(corpus))
+        word_counts = Counter()
+        for text in corpus:
+            word_counts.update(text.split())
+        sorted_words = [w for w, _ in word_counts.most_common(self.vocab_size - 3)]
+        token_to_id: Dict[str, int] = {"<pad>": 0, "<eos>": 1, "<unk>": 2}
+        for idx, word in enumerate(sorted_words, start=3):
+            token_to_id[word] = idx
+        merges: List[Tuple[str, str, str]] = []
+        self._merges = merges
+        self._tokenizer = build_bpe_from_merges(
+            merges=merges,
+            token_to_id=token_to_id,
+            unk_token=self.unk_token,
+            add_prefix_space=self.add_prefix_space,
+        )
+        self._build_vocab_cache()
+        self.vocab_size = len(self._vocab)
+        logger.info("BBPE word-level fallback: vocab=%d", len(self._vocab))
+
     # ------------------------------------------------------------------
     # TREINAMENTO (compatibilidade — delega para paralelo)
     # ------------------------------------------------------------------

states/v6_5_v2_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6eded03affb9f367f3162fd7e7bcedd90bfb3c406dc94dccba86f8ab9dfac201
+size 117713262