V6.5-7ds cleanup: remove legacy scripts/train_v6_3.py

scripts/train_v6_3.py

@@ -1,558 +0,0 @@
-"""train_v6_3.py — V6.3 REFACTORED Kohonen Learning System training.
-
-═══════════════════════════════════════════════════════════════════════════════
-V6.3 — REFATORAÇÃO COMPLETA COM CÓDIGO FORNECIDO PELO USUÁRIO
-═══════════════════════════════════════════════════════════════════════════════
-
-Refatoração sobrescreve os módulos usando o código Kohonen SOM 4D fornecido
-pelo usuário, com análise matemática detalhada e correção de bugs
-identificados (find_bmu com return prematuro via Ellipsis).
-
-Componentes ativados:
-    1. xeon_runtime.py (AVX512 + AMX_INT8 + IPEX + OneDNN + FP16)
-    2. streaming_datasets.py (5 datasets × 50 samples = 250 total)
-    3. KohonenLearningSystem (refatorado, código do usuário)
-    4. BATCH_SIZE = 16 (user requirement)
-    5. Captura de métricas: 12/12 + Kohonen (sigma, alpha, BMU, fisher)
-    6. Monitoramento e informe de valores obtidos
-
-User requirements (V6.3):
-    - "refatorar sobrescrevendo os módulos usando (analisar matematicamente)"
-    - "HF_TOKEN=hf_<REDACTED_TOKEN> que deve ser apagada após uso"
-    - "usar streaming_datasets.py"
-    - "ativar xeon_runtime.py"
-    - "BATCH_SIZE = 16"
-    - "5 datasets × 50 samples"
-    - "monitorar e informar valores obtidos"
-
-Saídas:
-    - /home/z/my-project/BiGRU_T_version/v6_3_report.json
-    - /home/z/my-project/BiGRU_T_version/v6_3_training_metrics.json
-    - Log no worklog.md
-═══════════════════════════════════════════════════════════════════════════════
-"""
-from __future__ import annotations
-
-import json
-import logging
-import os
-import sys
-import time
-import traceback
-from datetime import datetime
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
-
-# ============================================================================
-# 0. Paths e logging
-# ============================================================================
-PROJECT_ROOT = Path("/home/z/my-project")
-BIGRU_ROOT = PROJECT_ROOT / "BiGRU_T_version"
-SRC_ROOT = BIGRU_ROOT / "src"
-REPORT_PATH = BIGRU_ROOT / "v6_3_report.json"
-METRICS_PATH = BIGRU_ROOT / "v6_3_training_metrics.json"
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="[%(asctime)s] [%(levelname)s] %(message)s",
-    datefmt="%H:%M:%S",
-)
-logger = logging.getLogger("train_v6_3")
-
-# ============================================================================
-# 1. ATIVAR xeon_runtime.py (user requirement)
-# ============================================================================
-# Insere src/ no sys.path antes de importar bigru_t
-sys.path.insert(0, str(SRC_ROOT))
-
-from bigru_t.utils.xeon_runtime import (  # noqa: E402
-    optimize_xeon_environment,
-    benchmark_fp16_matmul,
-    get_xeon_status,
-)
-
-N_CORES = optimize_xeon_environment(verbose=True)
-XEON_STATUS = get_xeon_status()
-FP16_BENCH = benchmark_fp16_matmul(size=4000, warmup=1, iters=2)
-logger.info(f"[V6.3] Xeon FP16 benchmark: {FP16_BENCH}")
-
-# ============================================================================
-# 2. Configurações V6.3
-# ============================================================================
-# User requirement: BATCH_SIZE = 16
-BATCH_SIZE = 16
-# User requirement: 5 datasets × 50 samples = 250 total
-N_DATASETS = 5
-SAMPLES_PER_DATASET = 50
-TOTAL_SAMPLES = N_DATASETS * SAMPLES_PER_DATASET  # 250
-EPOCHS = 2
-MAX_SEQ_LEN = 8
-HIDDEN_DIM = 256  # reduzido de 1024 para caber em memória com BATCH_SIZE=16
-
-# Kohonen SOM 4D grid: (I, J, K, L) — 8 neurônios no total
-# User forneceu grid (5,5,5,3)=375 neurônios no exemplo; mas para compatibilidade
-# com BiGRU_T (8 BiGRU4 neurônios), usamos grid (2,2,2,1)=8 neurônios.
-SOM_GRID = (2, 2, 2, 1)  # 8 neurônios
-
-# 5 datasets PT-BR para streaming (do streaming_datasets.py)
-V63_DATASETS = [
-    "TucanoBR/GigaVerbo",
-    "dominguesm/restore-punctuation-ptbr-dataset",
-    "Madras1/corpus-ptbr-v2",
-    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1",
-    "nvidia/OpenMathInstruct-2",
-]
-
-# Templates sintéticos para fallback (caso streaming falhe/lento)
-SYNTH_TEMPLATES = {
-    "TucanoBR/GigaVerbo": [
-        "o gato dorme na cama",
-        "o cachorro corre no parque",
-        "o pássaro voa no céu",
-        "a menina brinca com a boneca",
-        "o menino joga bola",
-    ],
-    "dominguesm/restore-punctuation-ptbr-dataset": [
-        "o sol nasceu azul hoje",
-        "ela foi ao mercado comprar pão",
-        "nós viajamos para o rio de janeiro",
-        "o livro está sobre a mesa",
-        "a casa tem quatro quartos",
-    ],
-    "Madras1/corpus-ptbr-v2": [
-        "o brasil é um país tropical",
-        "a música popular brasileira é rica",
-        "o carnaval acontece em fevereiro",
-        "a floresta amazônica é vasta",
-        "o futebol é o esporte favorito",
-    ],
-    "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1": [
-        "olá como você está hoje",
-        "qual é o seu nome",
-        "pode me ajudar com isso",
-        "obrigado pela ajuda",
-        "até logo e boa noite",
-    ],
-    "nvidia/OpenMathInstruct-2": [
-        "dois mais dois igual a quatro",
-        "três vezes cinco é quinze",
-        "dez dividido por dois é cinco",
-        "sete menos três é quatro",
-        "oito mais nove é dezessete",
-    ],
-}
-
-# ============================================================================
-# 3. Import KohonenLearningSystem (refatorado)
-# ============================================================================
-from bigru_t.model.kohonen_refactored.kohonen_learning_system import (  # noqa: E402
-    KohonenLearningSystem,
-    SimpleBBPETokenizer,
-    text_to_4d_vector,
-    HypothesisClassifier,
-    KohonenSOM4D,
-)
-
-logger.info(
-    f"[V6.3] KohonenLearningSystem imported. "
-    f"Grid={SOM_GRID} ({SOM_GRID[0]*SOM_GRID[1]*SOM_GRID[2]*SOM_GRID[3]} neurons)"
-)
-
-
-# ============================================================================
-# 4. Metrics Monitor (12/12 + Kohonen + Hypothesis)
-# ============================================================================
-class MetricsMonitorV63:
-    """Monitor 12/12 + Kohonen + Hypothesis para V6.3."""
-
-    def __init__(self) -> None:
-        self.steps: List[Dict[str, Any]] = []
-        self.alerts: List[Dict[str, Any]] = []
-        self.start_time = time.time()
-        self._prev_loss: Optional[float] = None
-
-    def record_step(
-        self,
-        step: int,
-        epoch: int,
-        dataset_name: str,
-        batch_loss: float,
-        batch_acc: float,
-        kls: KohonenLearningSystem,
-        rss_mb: float,
-    ) -> None:
-        som_status = kls.som.get_status()
-        # Quality metrics (1.1-1.5)
-        quality = {
-            "1.1_train_loss": float(batch_loss),
-            "1.2_train_acc": float(batch_acc),
-            "1.3_val_loss": float(batch_loss),  # placeholder (no separate val)
-            "1.4_val_acc": float(batch_acc),
-            "1.5_perplexity": float(2.718281828 ** min(batch_loss, 20)),
-        }
-        # Speed metrics (2.1-2.5)
-        elapsed = time.time() - self.start_time
-        speed = {
-            "2.1_throughput_sps": float((step + 1) * BATCH_SIZE / max(elapsed, 1e-6)),
-            "2.2_step_time_ms": float(elapsed * 1000 / max(step + 1, 1)),
-            "2.3_epoch_progress": float(epoch + 1) / EPOCHS,
-            "2.4_rss_mb": float(rss_mb),
-            "2.5_xeon_tflops": float(FP16_BENCH.get("best_tflops", 0.0)),
-        }
-        # Kohonen metrics
-        kohonen = {
-            "sigma_t": float(som_status["sigma_t"]),
-            "alpha_t": float(som_status["alpha_t"]),
-            "t": int(som_status["t"]),
-            "n_neurons": int(som_status["n_neurons"]),
-            "fisher_w_mean": float(som_status["fisher_w_mean"]),
-            "has_ewc_reference": bool(som_status["has_ewc_reference"]),
-        }
-        # Hypothesis metrics
-        hyp = {
-            "classifier_trained": bool(kls.classifier_trained),
-            "punishment_count": int(kls.punishment_count),
-            "success_count": int(kls.success_count),
-            "training_ready": bool(kls.training_ready),
-            "buffer_size": int(len(kls.buffer_4d)),
-            "required_new_samples": int(kls.required_new_samples),
-        }
-        # Alerts (3.1-3.3)
-        if self._prev_loss is not None:
-            delta = abs(batch_loss - self._prev_loss)
-            if delta > 5.0:
-                self.alerts.append({
-                    "type": "3.1_loss_spike",
-                    "step": step,
-                    "delta": float(delta),
-                    "prev": float(self._prev_loss),
-                    "curr": float(batch_loss),
-                })
-            if batch_loss > 30.0:
-                self.alerts.append({
-                    "type": "3.2_loss_explosion",
-                    "step": step,
-                    "value": float(batch_loss),
-                })
-            if batch_loss < 0.001:
-                self.alerts.append({
-                    "type": "3.3_loss_vanishing",
-                    "step": step,
-                    "value": float(batch_loss),
-                })
-        self._prev_loss = float(batch_loss)
-        if rss_mb > 4096:
-            self.alerts.append({
-                "type": "3.4_rss_high",
-                "step": step,
-                "rss_mb": float(rss_mb),
-            })
-
-        self.steps.append({
-            "step": step,
-            "epoch": epoch,
-            "dataset": dataset_name,
-            "quality": quality,
-            "speed": speed,
-            "kohonen": kohonen,
-            "hypothesis": hyp,
-        })
-
-    def summary(self) -> Dict[str, Any]:
-        if not self.steps:
-            return {}
-        final = self.steps[-1]
-        losses = [s["quality"]["1.1_train_loss"] for s in self.steps]
-        accs = [s["quality"]["1.2_train_acc"] for s in self.steps]
-        sigmas = [s["kohonen"]["sigma_t"] for s in self.steps]
-        alphas = [s["kohonen"]["alpha_t"] for s in self.steps]
-        rss_max = max(s["speed"]["2.4_rss_mb"] for s in self.steps)
-        rss_final = final["speed"]["2.4_rss_mb"]
-        return {
-            "n_steps": len(self.steps),
-            "final_loss": float(losses[-1]),
-            "mean_loss": float(sum(losses) / len(losses)),
-            "min_loss": float(min(losses)),
-            "max_loss": float(max(losses)),
-            "final_acc": float(accs[-1]),
-            "mean_acc": float(sum(accs) / len(accs)),
-            "sigma_start": float(sigmas[0]),
-            "sigma_end": float(sigmas[-1]),
-            "alpha_start": float(alphas[0]),
-            "alpha_end": float(alphas[-1]),
-            "rss_max_mb": float(rss_max),
-            "rss_final_mb": float(rss_final),
-            "rss_trend": "stable" if abs(rss_max - rss_final) < 200 else "growing",
-            "n_alerts": len(self.alerts),
-            "alerts": self.alerts[:20],
-            "kohonen_final": final["kohonen"],
-            "hypothesis_final": final["hypothesis"],
-        }
-
-
-# ============================================================================
-# 5. Streaming dataset loader com fallback sintético
-# ============================================================================
-def load_streaming_samples(
-    dataset_name: str,
-    n_samples: int,
-    hf_token: Optional[str] = None,
-    timeout_s: int = 60,
-) -> List[str]:
-    """Carrega até n_samples de um dataset via streaming_datasets.
-
-    Fallback: se streaming falhar ou timeout, gera samples sintéticos.
-    """
-    samples: List[str] = []
-    t_start = time.time()
-    try:
-        from bigru_t.data.streaming_datasets import stream_dataset
-        for sample in stream_dataset(dataset_name, max_samples=n_samples, hf_token=hf_token):
-            if time.time() - t_start > timeout_s:
-                logger.warning(
-                    f"[V6.3] Streaming {dataset_name} timeout ({timeout_s}s) "
-                    f"after {len(samples)} samples"
-                )
-                break
-            if sample.raw_text and len(sample.raw_text.strip()) > 0:
-                samples.append(sample.raw_text.strip()[:200])
-            if len(samples) >= n_samples:
-                break
-    except Exception as e:
-        logger.warning(f"[V6.3] Streaming {dataset_name} failed: {e}")
-
-    # Fallback sintético se não conseguiu amostras suficientes
-    if len(samples) < n_samples:
-        templates = SYNTH_TEMPLATES.get(dataset_name, ["exemplo genérico"])
-        needed = n_samples - len(samples)
-        logger.info(
-            f"[V6.3] Fallback sintético: gerando {needed} samples para {dataset_name}"
-        )
-        for i in range(needed):
-            base = templates[i % len(templates)]
-            # Variação para diversidade
-            samples.append(f"{base} (var {i})")
-    return samples[:n_samples]
-
-
-# ============================================================================
-# 6. Labels binárias (placeholder para classificação)
-# ============================================================================
-def make_label(text: str) -> int:
-    """Gera label binário determinístico baseado no texto.
-
-    Heurística: 0 se contém palavra "gato", "gato mia", "dorme", "brinca";
-                1 caso contrário (cachorro, corre, joga, etc.)
-    """
-    text_lower = text.lower()
-    if any(w in text_lower for w in ["gato", "mia", "dorme", "brinca", "menina", "boneca"]):
-        return 0
-    return 1
-
-
-# ============================================================================
-# 7. Função principal de treino
-# ============================================================================
-def main() -> int:
-    print("\n" + "=" * 72)
-    print("V6.3 — REFACTORED KOHONEN LEARNING SYSTEM")
-    print("=" * 72)
-    print(f"  BATCH_SIZE           : {BATCH_SIZE}")
-    print(f"  Datasets             : {N_DATASETS}")
-    print(f"  Samples/dataset      : {SAMPLES_PER_DATASET}")
-    print(f"  Total samples        : {TOTAL_SAMPLES}")
-    print(f"  Epochs               : {EPOCHS}")
-    print(f"  SOM grid             : {SOM_GRID} ({SOM_GRID[0]*SOM_GRID[1]*SOM_GRID[2]*SOM_GRID[3]} neurons)")
-    print(f"  Hidden dim           : {HIDDEN_DIM}")
-    print(f"  Max seq len          : {MAX_SEQ_LEN}")
-    print(f"  Xeon cores           : {N_CORES}")
-    print(f"  Xeon AVX512          : {XEON_STATUS['avx512']['desc']}")
-    print(f"  Xeon AMX             : {XEON_STATUS['amx']['desc']}")
-    print(f"  Xeon IPEX            : {XEON_STATUS['ipex_available']}")
-    print(f"  FP16 best TFLOPS     : {FP16_BENCH.get('best_tflops', 0.0):.3f}")
-    print("=" * 72 + "\n")
-
-    # Inicializa KohonenLearningSystem
-    kls = KohonenLearningSystem(
-        vocab_size=8192,
-        hidden_dim=HIDDEN_DIM,
-        seq_len=MAX_SEQ_LEN,
-        som_grid=SOM_GRID,
-        alpha0=0.1,
-        sigma0=1.5,
-        lambda_ewc=0.02,
-        N_start=10,
-        dim_choice="y",
-        hypothesis_hidden=[512, 256, 128, 64, 32, 16, 8],
-    )
-    logger.info(
-        f"[V6.3] KohonenLearningSystem initialized. "
-        f"Vocab=8192, Hidden={HIDDEN_DIM}, Grid={SOM_GRID}"
-    )
-
-    # Treina tokenizer com corpus sintético básico (PT-BR comum)
-    corpus_inicial = []
-    for templates in SYNTH_TEMPLATES.values():
-        corpus_inicial.extend(templates)
-    kls.tokenizer.fit(corpus_inicial)
-    logger.info(f"[V6.3] Tokenizer fitted with {len(corpus_inicial)} corpus words")
-
-    # Monitor
-    monitor = MetricsMonitorV63()
-
-    # HF_TOKEN (será apagado ao final)
-    hf_token = os.environ.get("HF_TOKEN")
-
-    # Loop de treino
-    step = 0
-    t_train_start = time.time()
-
-    for epoch in range(EPOCHS):
-        logger.info(f"\n[V6.3] === Epoch {epoch + 1}/{EPOCHS} ===")
-        for ds_idx, dataset_name in enumerate(V63_DATASETS):
-            samples = load_streaming_samples(
-                dataset_name, SAMPLES_PER_DATASET, hf_token=hf_token, timeout_s=60
-            )
-            labels = [make_label(s) for s in samples]
-            # Processa em batches de BATCH_SIZE
-            for batch_start in range(0, len(samples), BATCH_SIZE):
-                batch_sents = samples[batch_start : batch_start + BATCH_SIZE]
-                batch_labels = labels[batch_start : batch_start + BATCH_SIZE]
-                # Processa batch
-                try:
-                    stop_requested = kls.process_batch(batch_sents, batch_labels)
-                except Exception as e:
-                    logger.error(f"[V6.3] process_batch error: {e}")
-                    traceback.print_exc()
-                    continue
-
-                # Métricas de loss/acc
-                acc = kls.evaluate_classification()
-                # Loss proxy: -log(acc + eps) — menor acc = maior loss
-                loss = -max(0.01, acc) ** 0.5 if acc > 0 else 5.0
-                # RSS
-                try:
-                    import resource
-                    rss_kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
-                    rss_mb = rss_kb / 1024.0
-                except (AttributeError, OSError):
-                    rss_mb = 0.0
-
-                monitor.record_step(
-                    step=step,
-                    epoch=epoch,
-                    dataset_name=dataset_name,
-                    batch_loss=float(loss),
-                    batch_acc=float(acc),
-                    kls=kls,
-                    rss_mb=float(rss_mb),
-                )
-                step += 1
-                if step % 5 == 0 or step == 1:
-                    som_s = kls.som.get_status()
-                    logger.info(
-                        f"[V6.3] step={step:3d} | ds={ds_idx+1}/{N_DATASETS} | "
-                        f"loss={loss:.3f} acc={acc:.3f} | "
-                        f"σ={som_s['sigma_t']:.3f} α={som_s['alpha_t']:.3f} | "
-                        f"punish={kls.punishment_count} success={kls.success_count} | "
-                        f"buff={len(kls.buffer_4d)} | "
-                        f"hyp={'Y' if kls.classifier_trained else 'N'} | "
-                        f"ewc={'Y' if som_s['has_ewc_reference'] else 'N'} | "
-                        f"RSS={rss_mb:.0f}MB"
-                    )
-                if stop_requested:
-                    logger.warning(
-                        f"[V6.3] stop_requested at step={step}. "
-                        f"Required new samples: {kls.required_new_samples}"
-                    )
-
-    t_train_end = time.time()
-    train_duration = t_train_end - t_train_start
-    logger.info(f"\n[V6.3] Treino concluído em {train_duration:.1f}s ({step} steps)")
-
-    # Finaliza e gera relatório
-    summary = monitor.summary()
-    som_final = kls.som.get_status()
-    kls_status = kls.get_status()
-
-    report = {
-        "version": "V6.3",
-        "timestamp": datetime.now().isoformat(),
-        "config": {
-            "BATCH_SIZE": BATCH_SIZE,
-            "N_DATASETS": N_DATASETS,
-            "SAMPLES_PER_DATASET": SAMPLES_PER_DATASET,
-            "TOTAL_SAMPLES": TOTAL_SAMPLES,
-            "EPOCHS": EPOCHS,
-            "SOM_GRID": list(SOM_GRID),
-            "HIDDEN_DIM": HIDDEN_DIM,
-            "MAX_SEQ_LEN": MAX_SEQ_LEN,
-            "N_start": 10,
-            "dim_choice": "y",
-        },
-        "xeon_status": XEON_STATUS,
-        "fp16_benchmark": FP16_BENCH,
-        "training": {
-            "duration_s": float(train_duration),
-            "n_steps": int(step),
-            "n_epochs": EPOCHS,
-        },
-        "summary": summary,
-        "kohonen_final": som_final,
-        "kls_status": kls_status,
-        "math_analysis": {
-            "text_to_4d": "SVD: M @ V[:3].T -> centroid 3D + sigmoid(norm) -> w",
-            "bmu_distance": "||W - x||^2 (L2 squared in R^4)",
-            "neighborhood": "Lambda(d, sigma) = exp(-d^2 / (2*sigma^2)), d^2 = di^2+dj^2+dk^2+dl^2",
-            "weight_update": "dW = alpha * Lambda * (x - W)",
-            "sigma_decay": "sigma_t = sigma0 * exp(-t/1000)",
-            "alpha_decay": "alpha_t = alpha0 * exp(-t/2000)",
-            "ewc_only_dim4": "penalty = lambda * F * (W_w - W*_w), F = mean((x_w - W_w)^2)",
-            "hypothesis_classifier": "8 layers FC: 512->256->128->64->32->16->8->1",
-            "bug_fixed_find_bmu": "removed premature return with Ellipsis (dead code)",
-        },
-        "datasets_used": V63_DATASETS,
-    }
-
-    # Salva relatórios
-    REPORT_PATH.write_text(json.dumps(report, indent=2, ensure_ascii=False))
-    logger.info(f"[V6.3] Report saved: {REPORT_PATH}")
-
-    metrics_full = {
-        "version": "V6.3",
-        "steps": monitor.steps,
-        "summary": summary,
-        "alerts": monitor.alerts,
-    }
-    METRICS_PATH.write_text(json.dumps(metrics_full, indent=2, ensure_ascii=False))
-    logger.info(f"[V6.3] Metrics saved: {METRICS_PATH}")
-
-    # Print final summary
-    print("\n" + "=" * 72)
-    print("V6.3 — TREINO CONCLUÍDO")
-    print("=" * 72)
-    print(f"  Steps              : {step}")
-    print(f"  Duration           : {train_duration:.1f}s")
-    print(f"  Final loss         : {summary.get('final_loss', 0):.3f}")
-    print(f"  Mean loss          : {summary.get('mean_loss', 0):.3f}")
-    print(f"  Final acc          : {summary.get('final_acc', 0):.3f}")
-    print(f"  Mean acc           : {summary.get('mean_acc', 0):.3f}")
-    print(f"  Sigma (start→end)  : {summary.get('sigma_start', 0):.3f} → {summary.get('sigma_end', 0):.3f}")
-    print(f"  Alpha (start→end)  : {summary.get('alpha_start', 0):.3f} → {summary.get('alpha_end', 0):.3f}")
-    print(f"  RSS max            : {summary.get('rss_max_mb', 0):.0f}MB")
-    print(f"  RSS trend          : {summary.get('rss_trend', '?')}")
-    print(f"  Alerts             : {summary.get('n_alerts', 0)}")
-    print(f"  Hypothesis trained : {kls.classifier_trained}")
-    print(f"  EWC reference set  : {som_final['has_ewc_reference']}")
-    print(f"  Fisher w mean      : {som_final['fisher_w_mean']:.6f}")
-    print(f"  Punishment count   : {kls.punishment_count}")
-    print(f"  Success count      : {kls.success_count}")
-    print("=" * 72)
-    print(f"\n  Report : {REPORT_PATH}")
-    print(f"  Metrics: {METRICS_PATH}\n")
-
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())