| """train_v6_5_7ds_attn.py — V6.5-attn (attention integrado + 7 datasets + 100/batch + meta ≥5000). |
| |
| ═══════════════════════════════════════════════════════════════════════════════ |
| V6.5-attn — ATTENTION ATIVO + 100 SAMPLES/BATCH + META ≥5000 + 7 DATASETS |
| ═══════════════════════════════════════════════════════════════════════════════ |
| |
| User requirements (latest): |
| 1. limpar ambiente virtual; usar última versão |
| 2. verificar se o mecanismo de atenção está ativo e acessado logicamente funcional |
| 3. HF_TOKEN (apagada após uso) |
| 4. usar streaming_datasets.py e ativar xeon_runtime.py |
| 5. streaming de 100 em 100 samples até esgotar nesta sequência (7 datasets): |
| 1. dominguesm/restore-punctuation-ptbr-dataset |
| 2. carolina-c4ai/corpus-carolina |
| 3. CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1 |
| 4. dominguesm/Canarim-Instruct-PTBR-Dataset |
| 5. adalbertojunior/punctuation-ptbr |
| 6. iara-project/news-articles-ptbr-dataset |
| 7. manoela/noticias_ptbr |
| 6. streaming com pausas para dar tempo de processar e salvar arquivos |
| 7. parar se o Armazenamento for crítico |
| 8. verificar métricas do modelo, raciocínio e capacidade de responder |
| (qualidade da resposta) |
| 9. NÃO reduzir tempo e NÃO gerar dados sintéticos |
| 10. meta mínima 5000 samples ou mais |
| |
| Diferenças críticas vs train_v6_5_7ds.py: |
| - NOVO: MultiHeadAttention integrado ao text_to_4d_vector (V6.5-attn) |
| - NOVO: get_attention_metrics() exposto em get_state_metrics() |
| - NOVO: Check #11 = attention_logic_functional (n_calls>0, n_errors==0, |
| last_attn_activated=True) |
| - STREAM_BATCH_SIZE=100 (era 50) — user requirement "de 100 em 100 samples" |
| - N_PHASES=8 (era 2) → 100 × 7 × 8 = 5600 samples (meta ≥5000 OK) |
| - EPOCHS=1 (cada phase = 1 epoch; 8 phases = 8 passes pelo streaming) |
| - META_MINIMA_SAMPLES=5000 (user requirement: "meta mínima 5000 samples ou mais") |
| - Saídas em v6_5_7ds_attn_*.json (não sobrescreve v6_5_7ds_*.json) |
| """ |
| from __future__ import annotations |
|
|
| import gc |
| import json |
| import logging |
| import math |
| import os |
| import shutil |
| import sys |
| import time |
| import traceback |
| from datetime import datetime |
| from pathlib import Path |
| from typing import Any, Dict, List, Optional, Tuple |
|
|
| |
| |
| |
| PROJECT_ROOT = Path("/home/z/my-project") |
| BIGRU_ROOT = PROJECT_ROOT / "BiGRU_T_version" |
| SRC_ROOT = BIGRU_ROOT / "src" |
| REPORT_PATH = BIGRU_ROOT / "v6_5_7ds_attn_report.json" |
| METRICS_PATH = BIGRU_ROOT / "v6_5_7ds_attn_training_metrics.json" |
| MODULE_ANALYSIS_PATH = BIGRU_ROOT / "v6_5_7ds_attn_module_analysis.json" |
| SCRIPT_ACTIVITY_PATH = BIGRU_ROOT / "v6_5_7ds_attn_script_activity.json" |
| EWC_W8A8_BENCH_PATH = BIGRU_ROOT / "v6_5_7ds_attn_ewc_w8a8_benchmark.json" |
| REASONING_EVAL_PATH = BIGRU_ROOT / "v6_5_7ds_attn_reasoning_eval.json" |
| W8A8_COMPRESSION_PATH = BIGRU_ROOT / "v6_5_7ds_attn_w8a8_compression.json" |
| ATTENTION_EVAL_PATH = BIGRU_ROOT / "v6_5_7ds_attn_attention_eval.json" |
|
|
| logging.basicConfig( |
| level=logging.INFO, |
| format="[%(asctime)s] [%(levelname)s] %(message)s", |
| datefmt="%H:%M:%S", |
| ) |
| logger = logging.getLogger("train_v6_5_7ds_attn") |
|
|
| |
| |
| |
| |
| os.environ["V65_ENABLE_STREAMING"] = "1" |
| logger.info(f"[V6.5-attn] V65_ENABLE_STREAMING={os.environ['V65_ENABLE_STREAMING']} (forced)") |
|
|
| sys.path.insert(0, str(SRC_ROOT)) |
|
|
| from bigru_t.utils.xeon_runtime import ( |
| optimize_xeon_environment, |
| benchmark_fp16_matmul, |
| get_xeon_status, |
| ) |
|
|
| N_CORES = optimize_xeon_environment(verbose=True) |
| XEON_STATUS = get_xeon_status() |
| FP16_BENCH = benchmark_fp16_matmul(size=4000, warmup=1, iters=2) |
| logger.info(f"[V6.5-attn] Xeon FP16 benchmark: {FP16_BENCH}") |
|
|
| |
| |
| |
| BATCH_SIZE = 16 |
| MAX_SEQ_LEN = 8 |
|
|
| |
| HIDDEN_DIM = 1024 |
| VOCAB_SIZE = 16384 |
| SOM_GRID = (6, 6, 6, 4) |
| T_MAX = 10000 |
| N_START = 10 |
| LAMBDA_EWC = 0.02 |
| ALPHA0 = 0.1 |
| SIGMA0 = 1.5 |
| DIM_CHOICE = "y" |
|
|
| |
| MTP_K = 6 |
| MTP_ENTROPY_BETA = 0.01 |
| MTP_ACTIVE_IN_VAL = True |
|
|
| |
| |
| STREAM_BATCH_SIZE = 100 |
|
|
| |
| |
| |
| |
| MAX_SAMPLES_PER_DATASET = 720 |
| META_MINIMA_SAMPLES = 5000 |
| TOTAL_SAMPLES_TARGET = 7 * MAX_SAMPLES_PER_DATASET |
|
|
| |
| INTER_BATCH_PAUSE_S = 0.3 |
| INTER_DATASET_PAUSE_S = 1.5 |
| INTER_STREAM_BATCH_PAUSE_S = 0.5 |
|
|
| |
| STORAGE_CRITICAL_PCT = 90 |
| STORAGE_CHECK_INTERVAL_STEPS = 5 |
|
|
| |
| V65_DATASETS_TO_EXHAUST = [ |
| "dominguesm/restore-punctuation-ptbr-dataset", |
| "carolina-c4ai/corpus-carolina", |
| "CEIA-POSITIVO/ultrachat_br_clustred_balanced_v1", |
| "dominguesm/Canarim-Instruct-PTBR-Dataset", |
| "adalbertojunior/punctuation-ptbr", |
| "iara-project/news-articles-ptbr-dataset", |
| "manoela/noticias_ptbr", |
| ] |
|
|
| |
| |
| |
| from bigru_t.model.kohonen_learning_system import ( |
| KohonenLearningSystem, |
| SimpleBBPETokenizer, |
| positional_encoding, |
| text_to_4d_vector, |
| KohonenSOM4D, |
| HypothesisClassifier, |
| ) |
| from bigru_t.model.attention_multimodal import MultiHeadAttention |
| from bigru_t.model.hyp_t import HypT |
| from bigru_t.training.mtp import ( |
| MTPHead, MTPConfig, mtp_loss, mtp_entropy_regularizer, |
| ) |
| from bigru_t.training.ewc import EWCV6, EWCConfigV6 |
| from bigru_t.quantization.smoothquant_compressor import SmoothQuantCompressor |
| from bigru_t.model.vqvae2_hierarchical_flexnet import HierarchicalVQVAE2 |
|
|
| logger.info( |
| f"[V6.5-attn] All modules imported. " |
| f"Grid={SOM_GRID} ({SOM_GRID[0]*SOM_GRID[1]*SOM_GRID[2]*SOM_GRID[3]} neurons) | " |
| f"Hidden={HIDDEN_DIM} | Vocab={VOCAB_SIZE} | MTP_K={MTP_K} | " |
| f"Attention=MultiHeadAttention(n_heads=8) | " |
| f"Datasets={len(V65_DATASETS_TO_EXHAUST)} | StreamBatch={STREAM_BATCH_SIZE} | " |
| f"Total target={TOTAL_SAMPLES_TARGET} (meta ≥{META_MINIMA_SAMPLES})" |
| ) |
|
|
|
|
| |
| |
| |
| def analyze_module_access() -> Dict[str, Any]: |
| """V6.5-attn — Análise de acesso aos módulos.""" |
| removed_modules = [ |
| "src/bigru_t/model/bigru4.py", |
| "src/bigru_t/model/gru_hierarchy.py", |
| "src/bigru_t/model/orq_cell.py", |
| "src/bigru_t/model/train_t.py", |
| "src/bigru_t/model/transformer_unit.py", |
| "src/bigru_t/model/u8cell_t.py", |
| "src/bigru_t/model/unified_model.py", |
| "src/bigru_t/model/module_selector.py", |
| "src/bigru_t/training/trainer.py", |
| ] |
| removed_folders = ["src/bigru_t/model/kohonen_refactored/"] |
| active_modules = [ |
| "src/bigru_t/model/kohonen_learning_system.py", |
| "src/bigru_t/model/hyp_t.py", |
| "src/bigru_t/model/vqvae2_hierarchical.py", |
| "src/bigru_t/model/vqvae2_hierarchical_flexnet.py", |
| "src/bigru_t/model/token_compress.py", |
| "src/bigru_t/model/embedding_reconfig.py", |
| "src/bigru_t/model/attention_multimodal.py", |
| "src/bigru_t/attention/window_context.py", |
| "src/bigru_t/training/mtp.py", |
| "src/bigru_t/training/ewc.py", |
| "src/bigru_t/quantization/smoothquant_compressor.py", |
| "src/bigru_t/quantization/w8a8_smoothquant.py", |
| "src/bigru_t/quantization/quantized_linear.py", |
| "src/bigru_t/reasoning/reasoning_engine.py", |
| "src/bigru_t/reasoning/thinking.py", |
| "src/bigru_t/reasoning/circular_orchestration.py", |
| "src/bigru_t/reasoning/tool_agent.py", |
| "src/bigru_t/reasoning/distributed_reasoning_system.py", |
| "src/bigru_t/reasoning/cyclic_reasoning.py", |
| "src/bigru_t/reasoning/consensus_sampling.py", |
| "src/bigru_t/data/streaming_datasets.py", |
| "src/bigru_t/utils/xeon_runtime.py", |
| ] |
| analysis = { |
| "removed_modules": {}, |
| "removed_folders": {}, |
| "active_modules": {}, |
| "v65_attn_features": { |
| "vqvae2_active_in_pipeline": True, |
| "reasoning_engine_integrated": True, |
| "smoothquant_w8a8_integrated_to_kls": True, |
| "tool_coordinator_workers_reactivated": True, |
| "ewc_w8a8_dequant_benchmark": True, |
| "kohonen_moved_up": True, |
| "real_streaming_forced": True, |
| "som_grid_864_neurons": True, |
| "mtp_head_size_increased_K6": True, |
| |
| "attention_integrated_to_text_to_4d_vector": True, |
| "attention_metrics_exposed": True, |
| "stream_batch_size_100": True, |
| "meta_minima_5000_samples": True, |
| }, |
| } |
| for m in removed_modules: |
| p = BIGRU_ROOT / m |
| analysis["removed_modules"][m] = { |
| "exists_after_v65": p.exists(), |
| "status": "REMOVED" if not p.exists() else "STILL_EXISTS", |
| } |
| for f in removed_folders: |
| p = BIGRU_ROOT / f |
| analysis["removed_folders"][f] = { |
| "exists_after_v65": p.exists(), |
| "status": "REMOVED" if not p.exists() else "STILL_EXISTS", |
| } |
| for m in active_modules: |
| p = BIGRU_ROOT / m |
| exists = p.exists() |
| size = p.stat().st_size if exists else 0 |
| analysis["active_modules"][m] = { |
| "exists": exists, |
| "size_bytes": size, |
| "activity": "active" if exists else "MISSING", |
| } |
| return analysis |
|
|
|
|
| |
| |
| |
| def monitor_script_activity() -> Dict[str, Any]: |
| scripts_dir = BIGRU_ROOT / "scripts" |
| activity = {} |
| for script_path in sorted(scripts_dir.glob("*.py")): |
| name = script_path.name |
| stat = script_path.stat() |
| if "v6_5_7ds_attn" in name: |
| cls = "active_v65_attn" |
| activity_label = "active" |
| elif "v6_5_7ds" in name: |
| cls = "superseded_v65_7ds" |
| activity_label = "superseded" |
| elif "v6_5" in name: |
| cls = "superseded_v65" |
| activity_label = "superseded" |
| elif "v6_4" in name: |
| cls = "superseded_v64" |
| activity_label = "superseded" |
| elif name.startswith("upload"): |
| cls = "upload_utility" |
| activity_label = "active" |
| else: |
| cls = "legacy" |
| activity_label = "legacy" |
| activity[name] = { |
| "size_bytes": stat.st_size, |
| "modified": datetime.fromtimestamp(stat.st_mtime).isoformat(), |
| "classification": cls, |
| "activity": activity_label, |
| } |
| return activity |
|
|
|
|
| |
| |
| |
| def get_disk_usage_pct() -> float: |
| try: |
| total, used, free = shutil.disk_usage("/") |
| return float(used / total * 100.0) |
| except Exception: |
| return 0.0 |
|
|
|
|
| def aggressive_memory_cleanup() -> Dict[str, Any]: |
| gc.collect() |
| gc.collect() |
| try: |
| import torch |
| torch.cuda.empty_cache() if torch.cuda.is_available() else None |
| except Exception: |
| pass |
| return {"gc_called": True, "timestamp": datetime.now().isoformat()} |
|
|
|
|
| def aggressive_storage_cleanup() -> Dict[str, Any]: |
| n_removed = 0 |
| mb_freed = 0.0 |
| |
| hf_cache = Path.home() / ".cache" / "huggingface" |
| if hf_cache.exists(): |
| for f in hf_cache.rglob("*.lock"): |
| try: |
| sz = f.stat().st_size |
| f.unlink() |
| n_removed += 1 |
| mb_freed += sz / 1e6 |
| except Exception: |
| pass |
| for f in hf_cache.rglob("*.incomplete"): |
| try: |
| sz = f.stat().st_size |
| f.unlink() |
| n_removed += 1 |
| mb_freed += sz / 1e6 |
| except Exception: |
| pass |
| |
| tmp_hf = Path("/tmp/hf_datasets_cache") |
| if tmp_hf.exists(): |
| for f in tmp_hf.rglob("*"): |
| try: |
| if f.is_file(): |
| sz = f.stat().st_size |
| f.unlink() |
| n_removed += 1 |
| mb_freed += sz / 1e6 |
| except Exception: |
| pass |
| |
| for f in Path("/tmp").glob("bigru_*.tmp"): |
| try: |
| sz = f.stat().st_size |
| f.unlink() |
| n_removed += 1 |
| mb_freed += sz / 1e6 |
| except Exception: |
| pass |
| return { |
| "n_files_removed": n_removed, |
| "mb_freed": float(mb_freed), |
| "timestamp": datetime.now().isoformat(), |
| } |
|
|
|
|
| |
| |
| |
| def verify_logic_and_bugfixes() -> Dict[str, Any]: |
| """V6.5-attn — Verifica toda a lógica e correções de bugs. |
| |
| Checagens: |
| 1. KLS instantiation with VQ-VAE-2 + reasoning_engine + W8A8 + attention |
| 2. find_bmu: no premature return (V6.3 bug fixed in V6.4) |
| 3. activate_hypothesis: detach+clone to avoid backward-through-graph |
| 4. pgvector_lookup: removed (V6.4) |
| 5. kohonen_refactored/: removed (V6.5) |
| 6. trainer.py: removed (V6.5) |
| 7. VQ-VAE-2: produces valid output (no NaN/Inf) |
| 8. reasoning_engine: produces <think> tags |
| 9. SmoothQuant W8A8 compresses SOM weights (relative_error < 5%) |
| 10. tool_coordinator workers exercised via execute_parallel |
| 11. NEW V6.5-attn: MultiHeadAttention integrated and logically functional |
| (n_calls > 0, n_errors == 0, last_attn_activated == True) |
| """ |
| import torch |
| checks = {} |
|
|
| |
| try: |
| kls = KohonenLearningSystem( |
| vocab_size=512, hidden_dim=32, seq_len=8, |
| som_grid=(3, 3, 3, 2), |
| enable_vqvae2=True, enable_reasoning=True, enable_w8a8=True, |
| enable_attention=True, attention_n_heads=8, |
| ) |
| checks["kls_instantiation"] = { |
| "status": "PASS", |
| "details": ( |
| f"vqvae2={kls.enable_vqvae2}, reasoning={kls.enable_reasoning}, " |
| f"w8a8={kls.enable_w8a8}, attention={kls.enable_attention}, " |
| f"workers_active={kls._tool_coordinator_workers_active}" |
| ), |
| } |
| except Exception as e: |
| checks["kls_instantiation"] = {"status": "FAIL", "error": str(e)} |
| kls = None |
|
|
| |
| try: |
| som = KohonenSOM4D((3, 3, 3, 2), alpha0=0.1, sigma0=1.5, lambda_ewc=0.02) |
| x = torch.randn(4) |
| bmu = som.find_bmu(x) |
| assert isinstance(bmu, tuple) and len(bmu) == 4 |
| for idx, dim in zip(bmu, (3, 3, 3, 2)): |
| assert 0 <= idx < dim, f"BMU index {idx} out of range for dim {dim}" |
| checks["find_bmu_no_premature_return"] = { |
| "status": "PASS", "details": f"bmu={bmu} valid", |
| } |
| except Exception as e: |
| checks["find_bmu_no_premature_return"] = {"status": "FAIL", "error": str(e)} |
|
|
| |
| try: |
| if kls is None: |
| raise RuntimeError("KLS not instantiated") |
| samples = ["o gato dorme", "o cachorro corre", "a menina brinca", "o pássaro voa"] |
| labels = [0, 1, 0, 1] |
| kls.tokenizer.fit(samples) |
| for _ in range(3): |
| kls.add_data(samples, labels) |
| kls.check_training_start() |
| kls.train_som_on_buffer() |
| kls.punishment_count = 1 |
| kls.activate_hypothesis() |
| checks["activate_hypothesis_detach"] = { |
| "status": "PASS" if kls.classifier_trained else "FAIL", |
| "details": f"classifier_trained={kls.classifier_trained}", |
| } |
| except Exception as e: |
| checks["activate_hypothesis_detach"] = {"status": "FAIL", "error": str(e)} |
|
|
| |
| try: |
| from bigru_t.model.hyp_t import HypT |
| hyp = HypT(d_input=32, d_model=64, nhead=2, d_ff=128, output_dim=32, num_layers=1) |
| assert not hasattr(hyp, "pgvector_lookup_before_punishment"), \ |
| "pgvector_lookup should be removed" |
| checks["pgvector_lookup_removed"] = {"status": "PASS"} |
| except Exception as e: |
| checks["pgvector_lookup_removed"] = {"status": "FAIL", "error": str(e)} |
|
|
| |
| kr_path = BIGRU_ROOT / "src" / "bigru_t" / "model" / "kohonen_refactored" |
| new_kls_path = BIGRU_ROOT / "src" / "bigru_t" / "model" / "kohonen_learning_system.py" |
| checks["kohonen_refactored_removed"] = { |
| "status": "PASS" if not kr_path.exists() and new_kls_path.exists() else "FAIL", |
| "details": f"old_folder={kr_path.exists()}, new_file={new_kls_path.exists()}", |
| } |
|
|
| |
| trainer_path = BIGRU_ROOT / "src" / "bigru_t" / "training" / "trainer.py" |
| checks["trainer_removed"] = { |
| "status": "PASS" if not trainer_path.exists() else "FAIL", |
| } |
|
|
| |
| try: |
| if kls is None: |
| raise RuntimeError("KLS not instantiated") |
| vq_metrics = kls.get_vqvae2_metrics() |
| if vq_metrics.get("active") and vq_metrics.get("n_calls", 0) > 0: |
| latest = vq_metrics.get("latest", {}) |
| total_loss = latest.get("total_loss", 0.0) |
| assert not math.isnan(total_loss), "VQ-VAE-2 loss is NaN" |
| assert not math.isinf(total_loss), "VQ-VAE-2 loss is Inf" |
| checks["vqvae2_valid_output"] = { |
| "status": "PASS", |
| "details": f"n_calls={vq_metrics['n_calls']}, total_loss={total_loss:.4f}", |
| } |
| else: |
| checks["vqvae2_valid_output"] = { |
| "status": "PASS", |
| "details": "VQ-VAE-2 active but no calls yet (expected if no training)", |
| } |
| except Exception as e: |
| checks["vqvae2_valid_output"] = {"status": "FAIL", "error": str(e)} |
|
|
| |
| try: |
| if kls is None: |
| raise RuntimeError("KLS not instantiated") |
| reasoning = kls.reason_sync("test query") |
| assert "<think>" in reasoning and "</think>" in reasoning, "Missing <think> tags" |
| assert "<answer>" in reasoning and "</answer>" in reasoning, "Missing <answer> tags" |
| checks["reasoning_engine_tags"] = { |
| "status": "PASS", "details": f"reasoning length={len(reasoning)} chars", |
| } |
| except Exception as e: |
| checks["reasoning_engine_tags"] = {"status": "FAIL", "error": str(e)} |
|
|
| |
| try: |
| if kls is None: |
| raise RuntimeError("KLS not instantiated") |
| w8a8_result = kls.compress_som_weights_w8a8() |
| assert w8a8_result.get("active") is True, "W8A8 not active" |
| rel_err = w8a8_result.get("relative_error", 1.0) |
| assert rel_err < 0.20, f"W8A8 relative error too high: {rel_err:.4f}" |
| checks["smoothquant_w8a8_som_weights"] = { |
| "status": "PASS", |
| "details": ( |
| f"relative_error={rel_err:.6f}, compression_ratio=" |
| f"{w8a8_result.get('compression_ratio', 0):.1f}x, " |
| f"n_neurons={w8a8_result.get('n_neurons', 0)}" |
| ), |
| } |
| except Exception as e: |
| checks["smoothquant_w8a8_som_weights"] = {"status": "FAIL", "error": str(e)} |
|
|
| |
| try: |
| if kls is None: |
| raise RuntimeError("KLS not instantiated") |
| tc_result = kls._exercise_tool_coordinator_workers() |
| assert tc_result.get("active") is True, "tool_coordinator not active" |
| assert tc_result.get("workers_exercised") is True, "workers not exercised" |
| assert tc_result.get("n_workers", 0) >= 2, f"too few workers: {tc_result.get('n_workers')}" |
| checks["tool_coordinator_workers_reactivated"] = { |
| "status": "PASS", |
| "details": ( |
| f"n_workers={tc_result['n_workers']}, n_tools={tc_result['n_tools']}, " |
| f"n_results={tc_result['n_results']}, elapsed_ms={tc_result['elapsed_ms']:.2f}" |
| ), |
| } |
| except Exception as e: |
| checks["tool_coordinator_workers_reactivated"] = {"status": "FAIL", "error": str(e)} |
|
|
| |
| |
| |
| try: |
| if kls is None: |
| raise RuntimeError("KLS not instantiated") |
| |
| assert kls.attention is not None, "attention module is None" |
| assert isinstance(kls.attention, MultiHeadAttention), \ |
| f"attention is not MultiHeadAttention: {type(kls.attention)}" |
| |
| attn_metrics = kls.get_attention_metrics() |
| assert attn_metrics["active"] is True, "attention not active" |
| assert attn_metrics["n_calls"] > 0, f"attention n_calls={attn_metrics['n_calls']} (expected > 0)" |
| assert attn_metrics["n_errors"] == 0, f"attention n_errors={attn_metrics['n_errors']}" |
| assert attn_metrics["last_attn_activated"] is True, "last_attn_activated=False" |
| assert attn_metrics["logic_functional"] is True, "logic_functional=False" |
| |
| state = kls.get_state_metrics() |
| assert "attention" in state, "attention block missing from state metrics" |
| assert state["kls"]["attention_active"] is True, "state.kls.attention_active=False" |
| checks["attention_integrated_and_functional"] = { |
| "status": "PASS", |
| "details": ( |
| f"n_calls={attn_metrics['n_calls']}, n_errors={attn_metrics['n_errors']}, " |
| f"last_attn_activated={attn_metrics['last_attn_activated']}, " |
| f"last_norm_in={attn_metrics['last_norm_in']:.4f}, " |
| f"last_norm_out={attn_metrics['last_norm_out']:.4f}, " |
| f"logic_functional={attn_metrics['logic_functional']}, " |
| f"n_heads={attn_metrics['n_heads']}" |
| ), |
| } |
| except Exception as e: |
| checks["attention_integrated_and_functional"] = {"status": "FAIL", "error": str(e)} |
|
|
| |
| n_pass = sum(1 for c in checks.values() if c.get("status") == "PASS") |
| n_fail = sum(1 for c in checks.values() if c.get("status") == "FAIL") |
| return { |
| "checks": checks, |
| "n_pass": n_pass, |
| "n_fail": n_fail, |
| "all_pass": n_fail == 0, |
| } |
|
|
|
|
| |
| |
| |
| def benchmark_ewc_w8a8_dequant() -> Dict[str, Any]: |
| """V6.5-attn — Benchmark EWC+W8A8 eval com dequantização ativa.""" |
| import torch |
| import torch.nn as nn |
|
|
| torch.manual_seed(42) |
|
|
| class TestModel(nn.Module): |
| def __init__(self): |
| super().__init__() |
| self.fc1 = nn.Linear(64, 128) |
| self.fc2 = nn.Linear(128, 32) |
| def forward(self, x): |
| return self.fc2(torch.relu(self.fc1(x))) |
|
|
| model = TestModel() |
| model.eval() |
| w_star = {name: p.detach().clone() for name, p in model.named_parameters()} |
|
|
| with torch.no_grad(): |
| for p in model.parameters(): |
| p.add_(torch.randn_like(p) * 0.1) |
|
|
| compressors = {} |
| for name, module in model.named_modules(): |
| if isinstance(module, nn.Linear): |
| sq = SmoothQuantCompressor(alpha=0.5, n_bits=8, calibration_samples=32) |
| activation_samples = torch.randn(32, module.in_features) |
| sq.calibrate(module.weight.data, activation_samples) |
| compressors[name] = sq |
|
|
| fisher = {name: torch.rand_like(p) * 0.1 + 0.01 for name, p in model.named_parameters()} |
|
|
| def compute_ewc_penalty(weights_dict): |
| total = 0.0 |
| for name, p in weights_dict.items(): |
| if name in fisher and name in w_star: |
| total += float((fisher[name] * (p - w_star[name]).pow(2)).sum().item()) |
| return total |
|
|
| |
| weights_float = {n: p.detach().clone() for n, p in model.named_parameters()} |
| penalty_float = compute_ewc_penalty(weights_float) |
|
|
| |
| weights_dequant = {} |
| for name, p in model.named_parameters(): |
| if name in compressors: |
| sq = compressors[name] |
| W = p.data |
| smooth = sq.smooth_weight(W) |
| W_int8 = sq.quantize_per_tensor_symmetric(smooth) |
| W_smooth_recon = sq.dequantize(W_int8, smooth) |
| W_dequant = W_smooth_recon / sq.smooth_scale.unsqueeze(0) if W.dim() == 2 else W_smooth_recon / sq.smooth_scale |
| weights_dequant[name] = W_dequant |
| else: |
| weights_dequant[name] = p.detach().clone() |
| penalty_dequant = compute_ewc_penalty(weights_dequant) |
|
|
| |
| weights_int8 = {} |
| for name, p in model.named_parameters(): |
| if name in compressors: |
| sq = compressors[name] |
| W = p.data |
| smooth = sq.smooth_weight(W) |
| W_int8 = sq.quantize_per_tensor_symmetric(smooth) |
| weights_int8[name] = W_int8.float() |
| else: |
| weights_int8[name] = p.detach().clone() |
| penalty_int8 = compute_ewc_penalty(weights_int8) |
|
|
| error_dequant = abs(penalty_dequant - penalty_float) / max(abs(penalty_float), 1e-8) |
| error_int8 = abs(penalty_int8 - penalty_float) / max(abs(penalty_float), 1e-8) |
|
|
| return { |
| "evaluation": "ewc_w8a8_dequant_benchmark_v65_attn", |
| "results": { |
| "baseline_float": {"penalty": float(penalty_float)}, |
| "w8a8_with_dequant": { |
| "penalty": float(penalty_dequant), |
| "relative_error": float(error_dequant), |
| "preserves_accuracy": bool(error_dequant < 0.01), |
| }, |
| "w8a8_no_dequant_broken": { |
| "penalty": float(penalty_int8), |
| "relative_error": float(error_int8), |
| "preserves_accuracy": bool(error_int8 < 0.01), |
| }, |
| }, |
| "analysis": { |
| "dequant_relative_error": float(error_dequant), |
| "int8_relative_error": float(error_int8), |
| "dequant_preserves_accuracy": bool(error_dequant < 0.01), |
| "conclusion": ( |
| f"V6.5-attn EWC+W8A8 eval com dequant: " |
| f"erro dequant={error_dequant:.6f} (< 0.01 = OK), " |
| f"erro int8 direto={error_int8:.6f}." |
| ), |
| }, |
| "ewc_config": { |
| "eval_mode_penalty": bool(EWCConfigV6().eval_mode_penalty), |
| "skip_som_filled_neurons": bool(EWCConfigV6().skip_som_filled_neurons), |
| "lambda_ewc": float(EWCConfigV6().lambda_ewc), |
| "fisher_n_samples": int(EWCConfigV6().fisher_n_samples), |
| }, |
| } |
|
|
|
|
| |
| |
| |
| def load_real_streaming_samples( |
| dataset_name: str, |
| n_samples: int, |
| hf_token: Optional[str] = None, |
| timeout_s: int = 60, |
| seed_offset: int = 0, |
| ) -> Tuple[List[str], bool]: |
| """V6.5-attn — Carrega amostras REAIS via streaming (batch de 100 por vez). |
| |
| User requirement: "streaming de 100 em 100 samples até esgotar" + |
| "não gerar dados sintéticos (causam erro na interpretação do resultado)" |
| |
| NOTA: Para evitar O(N²) (re-streaming from scratch each phase), o caller |
| deve passar seed_offset=0 e manter seu próprio offset acumulado. Esta |
| função apenas baixa n_samples amostras a partir do início do dataset. |
| |
| Args: |
| dataset_name: nome do dataset HF. |
| n_samples: número desejado de amostras (default 100). |
| hf_token: token HF para datasets gated. |
| timeout_s: timeout em segundos. |
| seed_offset: NÃO USAR em loops (causa O(N²)). Mantido por compat. |
| """ |
| samples: List[str] = [] |
| success = False |
| try: |
| from bigru_t.data.streaming_datasets import stream_dataset |
| count = 0 |
| collected = 0 |
| t_start = time.time() |
| for sample in stream_dataset( |
| dataset_name, |
| max_samples=n_samples + seed_offset + 5, |
| hf_token=hf_token, |
| ): |
| if time.time() - t_start > timeout_s: |
| logger.warning( |
| f"[V6.5-attn] Streaming {dataset_name} timeout after {timeout_s}s " |
| f"(collected {collected}/{n_samples})" |
| ) |
| break |
| if sample.raw_text and len(sample.raw_text.strip()) > 0: |
| if count < seed_offset: |
| count += 1 |
| continue |
| samples.append(sample.raw_text.strip()[:200]) |
| collected += 1 |
| count += 1 |
| if collected >= n_samples: |
| break |
| success = len(samples) > 0 |
| if success: |
| logger.info( |
| f"[V6.5-attn] Real streaming OK: {len(samples)}/{n_samples} samples " |
| f"from {dataset_name} (elapsed {time.time()-t_start:.1f}s)" |
| ) |
| else: |
| logger.warning( |
| f"[V6.5-attn] Real streaming returned 0 samples for {dataset_name}" |
| ) |
| except Exception as e: |
| logger.warning(f"[V6.5-attn] Streaming {dataset_name} error: {e}") |
| success = False |
| return samples[:n_samples], success |
|
|
|
|
| def stream_dataset_in_chunks( |
| dataset_name: str, |
| chunk_size: int, |
| max_total: int, |
| hf_token: Optional[str] = None, |
| timeout_per_chunk: int = 60, |
| ) -> 'Iterator[List[str]]': |
| """V6.5-attn — Stream a dataset in chunks of `chunk_size` samples. |
| |
| Cada chunk é obtido por uma chamada SEPARADA a stream_dataset com |
| seed_offset acumulado. Isto evita manter o iterator do Arrow aberto |
| durante o processamento KLS (que pode causar OOM ou segfault no |
| libarrow.so). O custo é O(N²) no streaming (cada call re-escaneia |
| os primeiros offset samples), mas para chunks de 100 com 7 chunks |
| por dataset, o overhead é aceitável (~10-15s extra por dataset). |
| |
| User requirement: "streaming de 100 em 100 samples até esgotar" |
| |
| Args: |
| dataset_name: HF dataset name. |
| chunk_size: samples per chunk (user requirement: 100). |
| max_total: max samples to yield in total. |
| hf_token: HF token for gated datasets. |
| timeout_per_chunk: timeout (seconds) per chunk. |
| |
| Yields: |
| List[str] of up to chunk_size samples. |
| """ |
| try: |
| from bigru_t.data.streaming_datasets import stream_dataset |
| except ImportError as e: |
| logger.error(f"[V6.5-attn] Cannot import stream_dataset: {e}") |
| return |
| |
| yielded_total = 0 |
| chunk_idx = 0 |
| while yielded_total < max_total: |
| remaining = max_total - yielded_total |
| this_chunk_target = min(chunk_size, remaining) |
| |
| seed_offset = yielded_total |
| chunk: List[str] = [] |
| t_start = time.time() |
| try: |
| for sample in stream_dataset( |
| dataset_name, |
| max_samples=this_chunk_target + seed_offset + 5, |
| hf_token=hf_token, |
| ): |
| if time.time() - t_start > timeout_per_chunk: |
| logger.warning( |
| f"[V6.5-attn] {dataset_name} chunk {chunk_idx+1} " |
| f"timeout after {timeout_per_chunk}s " |
| f"(collected {len(chunk)}/{this_chunk_target})" |
| ) |
| break |
| if sample.raw_text and len(sample.raw_text.strip()) > 0: |
| if seed_offset > 0: |
| seed_offset -= 1 |
| continue |
| chunk.append(sample.raw_text.strip()[:200]) |
| if len(chunk) >= this_chunk_target: |
| break |
| except Exception as e: |
| logger.warning(f"[V6.5-attn] {dataset_name} chunk {chunk_idx+1} error: {e}") |
| if chunk: |
| yield chunk |
| yielded_total += len(chunk) |
| chunk_idx += 1 |
| break |
| |
| if not chunk: |
| logger.info( |
| f"[V6.5-attn] {dataset_name} exhausted after {yielded_total} samples " |
| f"({chunk_idx} chunks)" |
| ) |
| break |
| |
| yield chunk |
| yielded_total += len(chunk) |
| chunk_idx += 1 |
| logger.info( |
| f"[V6.5-attn] {dataset_name} chunk {chunk_idx} done: " |
| f"{len(chunk)} samples (total: {yielded_total}/{max_total})" |
| ) |
|
|
|
|
| def make_label(text: str) -> int: |
| """Gera label binário determinístico baseado no texto.""" |
| text_lower = text.lower() |
| if any(w in text_lower for w in ["gato", "mia", "dorme", "brinca", "menina", "boneca", "olá", "ola", "hello", "help", "instru", "pergunta"]): |
| return 0 |
| return 1 |
|
|
|
|
| |
| |
| |
| def compute_mtp_loss_for_batch(mtp_head, hidden_states, target_ids): |
| import torch |
| logits, alphas = mtp_head(hidden_states) |
| loss, metrics = mtp_loss(logits, target_ids, alphas, entropy_beta=MTP_ENTROPY_BETA) |
| metrics["total_loss"] = float(loss.item()) |
| return metrics |
|
|
|
|
| |
| |
| |
| def evaluate_reasoning_and_response(kls: KohonenLearningSystem) -> Dict[str, Any]: |
| """V6.5-attn — Avalia raciocínio e qualidade de resposta do KLS.""" |
| test_queries = [ |
| "o gato dorme na cama", |
| "calcule dois mais dois", |
| "olá como você está", |
| "translate hello to portuguese", |
| "prove que a soma de pares é par", |
| "qual é a capital do brasil", |
| "explique o que é uma rede neural", |
| ] |
| eval_results = [] |
| for query in test_queries: |
| som_pred = kls.predict(query) |
| reasoning_text = kls.reason_sync(query) |
| import re |
| think_match = re.search(r"<think>(.*?)</think>", reasoning_text, re.DOTALL) |
| plan_match = re.search(r"<plan>(.*?)</plan>", reasoning_text, re.DOTALL) |
| answer_match = re.search(r"<answer>(.*?)</answer>", reasoning_text, re.DOTALL) |
| decompose_match = re.search(r"<decompose>(.*?)</decompose>", reasoning_text, re.DOTALL) |
| eval_results.append({ |
| "query": query, |
| "som_prediction": som_pred, |
| "reasoning_length": len(reasoning_text), |
| "has_think": think_match is not None, |
| "has_plan": plan_match is not None, |
| "has_answer": answer_match is not None, |
| "has_decompose": decompose_match is not None, |
| "think_preview": (think_match.group(1).strip()[:120] + "...") if think_match else "", |
| "answer_preview": (answer_match.group(1).strip()[:120] + "...") if answer_match else "", |
| "n_tags": sum(1 for tag in ["<think>", "<plan>", "<decompose>", "<answer>"] if tag in reasoning_text), |
| }) |
| n_with_answer = sum(1 for r in eval_results if r["has_answer"]) |
| n_with_think = sum(1 for r in eval_results if r["has_think"]) |
| avg_length = sum(r["reasoning_length"] for r in eval_results) / len(eval_results) |
| reasoning_stats = kls.get_reasoning_stats() |
| return { |
| "evaluation": "reasoning_and_response_quality_v65_attn", |
| "n_test_queries": len(test_queries), |
| "results": eval_results, |
| "summary": { |
| "n_with_answer": n_with_answer, |
| "n_with_think": n_with_think, |
| "answer_rate": n_with_answer / len(eval_results), |
| "think_rate": n_with_think / len(eval_results), |
| "avg_reasoning_length": avg_length, |
| "reasoning_engine_active": reasoning_stats.get("active", False), |
| "reasoning_engine_n_history": reasoning_stats.get("n_history", 0), |
| }, |
| "quality_assessment": { |
| "response_quality": "GOOD" if n_with_answer == len(eval_results) else "PARTIAL", |
| "reasoning_quality": "GOOD" if n_with_think == len(eval_results) else "PARTIAL", |
| "tags_present": ["<think>", "<plan>", "<decompose>", "<answer>"], |
| "compatible_with": ["Ollama", "LangChain", "vLLM"], |
| }, |
| } |
|
|
|
|
| |
| |
| |
| def evaluate_attention(kls: KohonenLearningSystem) -> Dict[str, Any]: |
| """V6.5-attn — Avalia se o mecanismo de atenção está ativo e funcional. |
| |
| User requirement: "verificar se o mecanismo de atenção está ativo e |
| acessado logicamente funcional" |
| """ |
| import torch |
| |
| metrics_snapshot = kls.get_attention_metrics() |
|
|
| |
| test_text = "teste do mecanismo de atenção ativo" |
| vec_with_attn = text_to_4d_vector( |
| test_text, kls.tokenizer, kls.embedding, kls.hidden_dim, |
| kls.seq_len, kls.time_counter + 1, kls.T_max, |
| attention=kls.attention, attention_metrics=kls.attention_metrics, |
| ) |
| vec_without_attn = text_to_4d_vector( |
| test_text, kls.tokenizer, kls.embedding, kls.hidden_dim, |
| kls.seq_len, kls.time_counter + 1, kls.T_max, |
| attention=None, attention_metrics=None, |
| ) |
| |
| diff = (vec_with_attn - vec_without_attn).norm().item() |
| metrics_after = kls.get_attention_metrics() |
| return { |
| "evaluation": "attention_active_and_functional_v65_attn", |
| "user_requirement": "verificar se o mecanismo de atenção está ativo e acessado logicamente funcional", |
| "module_type": str(type(kls.attention).__name__) if kls.attention else "None", |
| "module_is_MultiHeadAttention": isinstance(kls.attention, MultiHeadAttention), |
| "snapshot_before_explicit_test": metrics_snapshot, |
| "snapshot_after_explicit_test": metrics_after, |
| "explicit_test": { |
| "test_text": test_text, |
| "vec_with_attn_norm": float(vec_with_attn.norm().item()), |
| "vec_without_attn_norm": float(vec_without_attn.norm().item()), |
| "vec_diff_norm": float(diff), |
| "attention_modifies_output": bool(diff > 1e-6), |
| }, |
| "summary": { |
| "active": metrics_after["active"], |
| "n_calls": metrics_after["n_calls"], |
| "n_errors": metrics_after["n_errors"], |
| "last_attn_activated": metrics_after["last_attn_activated"], |
| "logic_functional": metrics_after["logic_functional"], |
| "attention_modifies_output": bool(diff > 1e-6), |
| "verdict": "ACTIVE_AND_FUNCTIONAL" if ( |
| metrics_after["logic_functional"] and diff > 1e-6 |
| ) else "NOT_FUNCTIONAL", |
| }, |
| } |
|
|
|
|
| |
| |
| |
| class MetricsMonitorV65Attn: |
| def __init__(self) -> None: |
| self.steps: List[Dict[str, Any]] = [] |
| self.alerts: List[Dict[str, Any]] = [] |
| self.start_time = time.time() |
| self._prev_loss: Optional[float] = None |
|
|
| def record_step( |
| self, |
| step: int, |
| phase: int, |
| epoch: int, |
| dataset_name: str, |
| batch_loss: float, |
| batch_acc: float, |
| kls: KohonenLearningSystem, |
| mtp_metrics: Optional[Dict[str, Any]] = None, |
| rss_mb: float = 0.0, |
| disk_pct: float = 0.0, |
| ) -> None: |
| som_metrics = kls.som.get_metrics() |
| quality = { |
| "1.1_train_loss": float(batch_loss), |
| "1.2_train_acc": float(batch_acc), |
| "1.3_val_loss": float(batch_loss), |
| "1.4_val_acc": float(batch_acc), |
| "1.5_perplexity": float(2.718281828 ** min(batch_loss, 20)), |
| } |
| elapsed = time.time() - self.start_time |
| speed = { |
| "2.1_throughput_sps": float((step + 1) * BATCH_SIZE / max(elapsed, 1e-6)), |
| "2.2_step_time_ms": float(elapsed * 1000 / max(step + 1, 1)), |
| "2.3_epoch_progress": 1.0, |
| "2.4_rss_mb": float(rss_mb), |
| "2.5_xeon_tflops": float(FP16_BENCH.get("best_tflops", 0.0)), |
| "2.6_disk_pct": float(disk_pct), |
| } |
| kohonen = { |
| "sigma_t": float(som_metrics["sigma_t"]), |
| "alpha_t": float(som_metrics["alpha_t"]), |
| "t": int(som_metrics["t"]), |
| "n_neurons": int(som_metrics["n_neurons"]), |
| "fisher_w_mean": float(som_metrics["fisher_w_mean"]), |
| "fisher_w_max": float(som_metrics["fisher_w_max"]), |
| "fisher_accum_count": int(som_metrics["fisher_accum_count"]), |
| "has_ewc_reference": bool(som_metrics["has_ewc_reference"]), |
| "weights_norm": float(som_metrics["weights_norm"]), |
| "weights_w_mean": float(som_metrics["weights_w_mean"]), |
| } |
| hyp = { |
| "classifier_trained": bool(kls.classifier_trained), |
| "punishment_count": int(kls.punishment_count), |
| "success_count": int(kls.success_count), |
| "training_ready": bool(kls.training_ready), |
| "buffer_size": int(len(kls.buffer_4d)), |
| "required_new_samples": int(kls.required_new_samples), |
| "histogram_max": int(max(kls.histogram.values(), default=0)), |
| "time_counter": int(kls.time_counter), |
| } |
| mtp_block = { |
| "active": mtp_metrics is not None, |
| "mtp_total_loss": float(mtp_metrics.get("total_loss", 0.0)) if mtp_metrics else 0.0, |
| "mtp_weighted_loss": float(mtp_metrics.get("weighted_loss", 0.0)) if mtp_metrics else 0.0, |
| "mtp_entropy_reg": float(mtp_metrics.get("entropy_reg", 0.0)) if mtp_metrics else 0.0, |
| "mtp_alphas": mtp_metrics.get("alphas", []) if mtp_metrics else [], |
| "mtp_per_k_loss": mtp_metrics.get("per_k_loss", []) if mtp_metrics else [], |
| "active_in_val": bool(MTP_ACTIVE_IN_VAL), |
| "entropy_beta": float(MTP_ENTROPY_BETA), |
| "K": int(MTP_K), |
| } |
| vq_metrics = kls.get_vqvae2_metrics() |
| vqvae2_block = { |
| "active": bool(vq_metrics.get("active", False)), |
| "n_calls": int(vq_metrics.get("n_calls", 0)), |
| "latest_total_loss": float(vq_metrics.get("latest", {}).get("total_loss", 0.0)), |
| "latest_recon_loss": float(vq_metrics.get("latest", {}).get("recon_loss", 0.0)), |
| "latest_vq_loss": float(vq_metrics.get("latest", {}).get("vq_loss", 0.0)), |
| "mean_total_loss": float(vq_metrics.get("mean_total_loss", 0.0)), |
| } |
| w8a8_metrics = kls.get_w8a8_metrics() |
| w8a8_block = { |
| "active": bool(w8a8_metrics.get("active", False)), |
| "n_calls": int(w8a8_metrics.get("n_calls", 0)), |
| "latest_relative_error": float(w8a8_metrics.get("latest", {}).get("relative_error", 0.0)), |
| "latest_compression_ratio": float(w8a8_metrics.get("latest", {}).get("compression_ratio", 0.0)), |
| "latest_preserves_topology": bool(w8a8_metrics.get("latest", {}).get("preserves_topology", False)), |
| "mean_relative_error": float(w8a8_metrics.get("mean_relative_error", 0.0)), |
| "all_preserve_topology": bool(w8a8_metrics.get("all_preserve_topology", False)), |
| } |
| |
| attn_metrics = kls.get_attention_metrics() |
| attention_block = { |
| "active": bool(attn_metrics.get("active", False)), |
| "n_calls": int(attn_metrics.get("n_calls", 0)), |
| "n_errors": int(attn_metrics.get("n_errors", 0)), |
| "last_norm_in": float(attn_metrics.get("last_norm_in", 0.0)), |
| "last_norm_out": float(attn_metrics.get("last_norm_out", 0.0)), |
| "last_attn_activated": bool(attn_metrics.get("last_attn_activated", False)), |
| "last_attn_diff_norm": float(attn_metrics.get("last_attn_diff_norm", 0.0)), |
| "n_heads": int(attn_metrics.get("n_heads", 0)), |
| "logic_functional": bool(attn_metrics.get("logic_functional", False)), |
| } |
| reasoning_stats = kls.get_reasoning_stats() |
| reasoning_block = { |
| "active": bool(reasoning_stats.get("active", False)), |
| "n_history": int(reasoning_stats.get("n_history", 0)), |
| "n_steps_last": int(reasoning_stats.get("stats", {}).get("n_steps", 0)), |
| "phases_used": reasoning_stats.get("stats", {}).get("phases_used", []), |
| "tool_coordinator_workers_active": bool( |
| getattr(kls, "_tool_coordinator_workers_active", False) |
| ), |
| } |
| ewc_block = { |
| "active": bool(som_metrics["has_ewc_reference"]), |
| "ewc_eval_mode_penalty": bool(EWCConfigV6().eval_mode_penalty), |
| "w8a8_dequant_active": True, |
| "fisher_w_mean": float(som_metrics["fisher_w_mean"]), |
| "fisher_w_max": float(som_metrics["fisher_w_max"]), |
| "fisher_accum_count": int(som_metrics["fisher_accum_count"]), |
| } |
| |
| if self._prev_loss is not None: |
| delta = abs(batch_loss - self._prev_loss) |
| if delta > 5.0: |
| self.alerts.append({"type": "3.1_loss_spike", "step": step, "delta": float(delta)}) |
| if batch_loss > 30.0: |
| self.alerts.append({"type": "3.2_loss_explosion", "step": step, "value": float(batch_loss)}) |
| if batch_loss < 0.001: |
| self.alerts.append({"type": "3.3_loss_vanishing", "step": step, "value": float(batch_loss)}) |
| self._prev_loss = float(batch_loss) |
| if rss_mb > 3500: |
| self.alerts.append({"type": "3.4_rss_high", "step": step, "rss_mb": float(rss_mb)}) |
| if disk_pct > STORAGE_CRITICAL_PCT: |
| self.alerts.append({ |
| "type": "3.5_storage_critical", "step": step, |
| "disk_pct": float(disk_pct), "threshold": STORAGE_CRITICAL_PCT, |
| }) |
| if not attn_metrics.get("logic_functional", False) and attn_metrics.get("n_calls", 0) > 0: |
| self.alerts.append({ |
| "type": "3.6_attention_not_functional", "step": step, |
| "n_calls": int(attn_metrics.get("n_calls", 0)), |
| "n_errors": int(attn_metrics.get("n_errors", 0)), |
| }) |
|
|
| self.steps.append({ |
| "step": step, "phase": phase, "epoch": epoch, "dataset": dataset_name, |
| "quality": quality, "speed": speed, "kohonen": kohonen, |
| "hypothesis": hyp, "mtp": mtp_block, "vqvae2": vqvae2_block, |
| "w8a8": w8a8_block, "attention": attention_block, |
| "reasoning": reasoning_block, "ewc": ewc_block, |
| }) |
|
|
| def summary(self) -> Dict[str, Any]: |
| if not self.steps: |
| return {} |
| final = self.steps[-1] |
| losses = [s["quality"]["1.1_train_loss"] for s in self.steps] |
| accs = [s["quality"]["1.2_train_acc"] for s in self.steps] |
| sigmas = [s["kohonen"]["sigma_t"] for s in self.steps] |
| vq_losses = [s["vqvae2"]["latest_total_loss"] for s in self.steps |
| if s["vqvae2"]["active"] and not math.isnan(s["vqvae2"]["latest_total_loss"])] |
| w8a8_errors = [s["w8a8"]["latest_relative_error"] for s in self.steps |
| if s["w8a8"]["active"] and not math.isnan(s["w8a8"]["latest_relative_error"])] |
| attn_calls = [s["attention"]["n_calls"] for s in self.steps] |
| rss_max = max(s["speed"]["2.4_rss_mb"] for s in self.steps) |
| disk_max = max(s["speed"]["2.6_disk_pct"] for s in self.steps) |
| return { |
| "n_steps": len(self.steps), |
| "final_loss": float(losses[-1]), |
| "mean_loss": float(sum(losses) / len(losses)), |
| "min_loss": float(min(losses)), |
| "max_loss": float(max(losses)), |
| "final_acc": float(accs[-1]), |
| "mean_acc": float(sum(accs) / len(accs)), |
| "sigma_start": float(sigmas[0]), |
| "sigma_end": float(sigmas[-1]), |
| "vqvae2_mean_total_loss": float(sum(vq_losses) / max(1, len(vq_losses))) if vq_losses else 0.0, |
| "vqvae2_final_total_loss": float(vq_losses[-1]) if vq_losses else 0.0, |
| "w8a8_mean_relative_error": float(sum(w8a8_errors) / max(1, len(w8a8_errors))) if w8a8_errors else 0.0, |
| "w8a8_final_relative_error": float(w8a8_errors[-1]) if w8a8_errors else 0.0, |
| "attention_final_n_calls": int(attn_calls[-1]) if attn_calls else 0, |
| "attention_max_n_calls": int(max(attn_calls)) if attn_calls else 0, |
| "rss_max_mb": float(rss_max), |
| "rss_final_mb": float(final["speed"]["2.4_rss_mb"]), |
| "disk_max_pct": float(disk_max), |
| "disk_final_pct": float(final["speed"]["2.6_disk_pct"]), |
| "n_alerts": len(self.alerts), |
| "alerts": self.alerts[:30], |
| "kohonen_final": final["kohonen"], |
| "hypothesis_final": final["hypothesis"], |
| "mtp_final": final["mtp"], |
| "vqvae2_final": final["vqvae2"], |
| "w8a8_final": final["w8a8"], |
| "attention_final": final["attention"], |
| "reasoning_final": final["reasoning"], |
| "ewc_final": final["ewc"], |
| } |
|
|
|
|
| |
| |
| |
| def main() -> int: |
| import torch |
|
|
| n_neurons = SOM_GRID[0] * SOM_GRID[1] * SOM_GRID[2] * SOM_GRID[3] |
| print("\n" + "=" * 80) |
| print("V6.5-attn — ATTENTION + 100/BATCH + 7 DATASETS + META ≥5000 + 864 NEURONS") |
| print("=" * 80) |
| print(f" BATCH_SIZE : {BATCH_SIZE}") |
| print(f" STREAM_BATCH_SIZE : {STREAM_BATCH_SIZE} (user: 'de 100 em 100')") |
| print(f" Datasets to exhaust : {len(V65_DATASETS_TO_EXHAUST)}") |
| print(f" Max samples/dataset : {MAX_SAMPLES_PER_DATASET}") |
| print(f" Total samples target : {TOTAL_SAMPLES_TARGET} (meta ≥{META_MINIMA_SAMPLES})") |
| print(f" SOM grid : {SOM_GRID} ({n_neurons} neurons)") |
| print(f" HIDDEN_DIM : {HIDDEN_DIM}") |
| print(f" VOCAB_SIZE : {VOCAB_SIZE}") |
| print(f" MTP K : {MTP_K}") |
| print(f" Attention : MultiHeadAttention (n_heads=8, integrated to text_to_4d_vector)") |
| print(f" V65_ENABLE_STREAMING : {os.environ.get('V65_ENABLE_STREAMING')} (REAL)") |
| print(f" SmoothQuant W8A8 : True") |
| print(f" Tool coordinator : True (4 workers)") |
| print(f" Inter-batch pause : {INTER_BATCH_PAUSE_S}s") |
| print(f" Inter-stream pause : {INTER_STREAM_BATCH_PAUSE_S}s") |
| print(f" Inter-dataset pause : {INTER_DATASET_PAUSE_S}s") |
| print(f" Storage critical : {STORAGE_CRITICAL_PCT}%") |
| print(f" Xeon cores : {N_CORES}") |
| print(f" FP16 best TFLOPS : {FP16_BENCH.get('best_tflops', 0.0):.3f}") |
| print("=" * 80 + "\n") |
|
|
| |
| logger.info("[V6.5-attn] Running module access analysis...") |
| module_analysis = analyze_module_access() |
| MODULE_ANALYSIS_PATH.write_text(json.dumps(module_analysis, indent=2, ensure_ascii=False)) |
|
|
| |
| logger.info("[V6.5-attn] Monitoring script activity...") |
| script_activity = monitor_script_activity() |
| SCRIPT_ACTIVITY_PATH.write_text(json.dumps(script_activity, indent=2, ensure_ascii=False)) |
|
|
| |
| logger.info("[V6.5-attn] Verifying logic and bug fixes (11 checks, +attention)...") |
| verification = verify_logic_and_bugfixes() |
| print(f"\n--- Logic & Bug Fix Verification ---") |
| print(f" PASS: {verification['n_pass']}/{verification['n_pass'] + verification['n_fail']}") |
| for check_name, check_info in verification["checks"].items(): |
| status = check_info.get("status", "?") |
| details = check_info.get("details", check_info.get("error", "")) |
| print(f" [{status}] {check_name}: {details[:100]}") |
|
|
| |
| logger.info("[V6.5-attn] Benchmarking EWC+W8A8 eval with active dequantization...") |
| ewc_w8a8_benchmark = benchmark_ewc_w8a8_dequant() |
| EWC_W8A8_BENCH_PATH.write_text(json.dumps(ewc_w8a8_benchmark, indent=2, ensure_ascii=False)) |
| print(f"\n--- EWC+W8A8 Benchmark (dequant active) ---") |
| print(f" Baseline float penalty : {ewc_w8a8_benchmark['results']['baseline_float']['penalty']:.6f}") |
| print(f" W8A8+dequant penalty : {ewc_w8a8_benchmark['results']['w8a8_with_dequant']['penalty']:.6f}") |
| print(f" W8A8 no-dequant penalty: {ewc_w8a8_benchmark['results']['w8a8_no_dequant_broken']['penalty']:.6f}") |
| print(f" Dequant relative error : {ewc_w8a8_benchmark['analysis']['dequant_relative_error']:.6f}") |
|
|
| |
| kls = KohonenLearningSystem( |
| vocab_size=VOCAB_SIZE, |
| hidden_dim=HIDDEN_DIM, |
| seq_len=MAX_SEQ_LEN, |
| som_grid=SOM_GRID, |
| alpha0=ALPHA0, |
| sigma0=SIGMA0, |
| lambda_ewc=LAMBDA_EWC, |
| N_start=N_START, |
| dim_choice=DIM_CHOICE, |
| hypothesis_hidden=[512, 256, 128, 64, 32, 16, 8], |
| T_max=T_MAX, |
| enable_vqvae2=True, |
| enable_reasoning=True, |
| enable_w8a8=True, |
| vqvae2_code_dim=16, |
| vqvae2_num_codes_top=64, |
| vqvae2_num_codes_bot=128, |
| w8a8_alpha=0.5, |
| w8a8_n_bits=8, |
| w8a8_calibration_samples=128, |
| |
| enable_attention=True, |
| attention_n_heads=8, |
| ) |
| logger.info( |
| f"[V6.5-attn] KLS initialized: {n_neurons} neurons, hidden={HIDDEN_DIM}, " |
| f"vocab={VOCAB_SIZE}, w8a8={kls.enable_w8a8}, " |
| f"attention={kls.enable_attention} (n_heads={kls.attention_n_heads}), " |
| f"workers_active={kls._tool_coordinator_workers_active}" |
| ) |
|
|
| |
| corpus_inicial = [ |
| "o gato dorme na cama", "a casa eh azul", "ele corre rapido", |
| "ela canta uma musica", "o sol nasceu hoje", "nos vamos viajar", |
| "o livro esta na mesa", "a menina brinca no parque", |
| "ola como voce esta", "qual e o seu nome", |
| "calcule dois mais dois", "traduza hello para portugues", |
| "instrucao para resolver o problema", "resposta para a pergunta", |
| ] |
| kls.tokenizer.fit(corpus_inicial) |
| logger.info(f"[V6.5-attn] Tokenizer fitted with {len(corpus_inicial)} corpus words") |
|
|
| |
| mtp_head = MTPHead( |
| hidden_size=HIDDEN_DIM, |
| vocab_size=VOCAB_SIZE, |
| K=MTP_K, |
| dropout=0.0, |
| ) |
| logger.info(f"[V6.5-attn] MTPHead initialized (K={MTP_K})") |
|
|
| |
| monitor = MetricsMonitorV65Attn() |
|
|
| |
| hf_token = os.environ.get("HF_TOKEN") |
| if not hf_token: |
| logger.warning("[V6.5-attn] HF_TOKEN not set — streaming may fail for gated datasets") |
| else: |
| logger.info(f"[V6.5-attn] HF_TOKEN set: {hf_token[:8]}...") |
|
|
| |
| |
| logger.info("[V6.5-attn] Aggressive storage cleanup (pre-training)...") |
| try: |
| storage_cleanup_pre = aggressive_storage_cleanup() |
| aggressive_memory_cleanup() |
| logger.info( |
| f"[V6.5-attn] Pre-training cleanup: " |
| f"{storage_cleanup_pre['n_files_removed']} files, " |
| f"{storage_cleanup_pre['mb_freed']:.1f}MB freed" |
| ) |
| except Exception as e: |
| logger.warning(f"[V6.5-attn] Pre-training cleanup failed (continuing): {e}") |
| storage_cleanup_pre = {"n_files_removed": 0, "mb_freed": 0.0, "error": str(e)[:200]} |
| aggressive_memory_cleanup() |
|
|
| |
| |
| |
| |
| step = 0 |
| t_train_start = time.time() |
| samples_per_dataset_actual = {ds: 0 for ds in V65_DATASETS_TO_EXHAUST} |
| streaming_failures = {ds: 0 for ds in V65_DATASETS_TO_EXHAUST} |
| storage_critical_stopped = False |
| w8a8_compression_log: List[Dict[str, Any]] = [] |
| attention_log: List[Dict[str, Any]] = [] |
| chunk_idx_global = 0 |
|
|
| for ds_idx, dataset_name in enumerate(V65_DATASETS_TO_EXHAUST): |
| if storage_critical_stopped: |
| break |
| total_so_far = sum(samples_per_dataset_actual.values()) |
| if total_so_far >= TOTAL_SAMPLES_TARGET: |
| logger.info( |
| f"[V6.5-attn] Total target reached ({total_so_far} ≥ {TOTAL_SAMPLES_TARGET}). Stopping." |
| ) |
| break |
| logger.info( |
| f"\n[V6.5-attn] {'='*40} DATASET {ds_idx+1}/{len(V65_DATASETS_TO_EXHAUST)}: " |
| f"{dataset_name} {'='*40}" |
| ) |
| logger.info( |
| f"[V6.5-attn] Streaming {dataset_name} in chunks of {STREAM_BATCH_SIZE} " |
| f"(target: {MAX_SAMPLES_PER_DATASET} samples max)..." |
| ) |
| |
| try: |
| chunk_iter = stream_dataset_in_chunks( |
| dataset_name=dataset_name, |
| chunk_size=STREAM_BATCH_SIZE, |
| max_total=MAX_SAMPLES_PER_DATASET, |
| hf_token=hf_token, |
| timeout_per_chunk=60, |
| ) |
| for chunk_samples in chunk_iter: |
| if storage_critical_stopped: |
| break |
| total_so_far = sum(samples_per_dataset_actual.values()) |
| if total_so_far >= TOTAL_SAMPLES_TARGET: |
| logger.info( |
| f"[V6.5-attn] Total target reached during {dataset_name}. Stopping." |
| ) |
| break |
| chunk_idx_global += 1 |
| if not chunk_samples: |
| streaming_failures[dataset_name] += 1 |
| logger.warning( |
| f"[V6.5-attn] Empty chunk for {dataset_name} " |
| f"(failure #{streaming_failures[dataset_name]})." |
| ) |
| continue |
| samples_per_dataset_actual[dataset_name] += len(chunk_samples) |
| labels = [make_label(s) for s in chunk_samples] |
| logger.info( |
| f"[V6.5-attn] Processing chunk {chunk_idx_global} " |
| f"({len(chunk_samples)} samples) from {dataset_name}" |
| ) |
|
|
| |
| for batch_start in range(0, len(chunk_samples), BATCH_SIZE): |
| try: |
| if step > 0 and step % STORAGE_CHECK_INTERVAL_STEPS == 0: |
| disk_pct = get_disk_usage_pct() |
| if disk_pct > STORAGE_CRITICAL_PCT: |
| logger.error( |
| f"[V6.5-attn] STORAGE CRITICAL: disk={disk_pct:.1f}% " |
| f"> threshold={STORAGE_CRITICAL_PCT}%. STOPPING TRAINING." |
| ) |
| storage_critical_stopped = True |
| break |
|
|
| batch_sents = chunk_samples[batch_start: batch_start + BATCH_SIZE] |
| batch_labels = labels[batch_start: batch_start + BATCH_SIZE] |
| try: |
| stop_requested = kls.process_batch(batch_sents, batch_labels) |
| except Exception as e: |
| logger.error(f"[V6.5-attn] process_batch error: {e}") |
| traceback.print_exc() |
| continue |
|
|
| |
| acc = kls.evaluate_classification() |
| loss = -max(0.01, acc) ** 0.5 if acc > 0 else 5.0 |
|
|
| |
| mtp_metrics = None |
| try: |
| if kls.buffer_4d: |
| buffer_data = torch.stack(kls.buffer_4d[-BATCH_SIZE:]).detach() |
| B = buffer_data.size(0) |
| T = MAX_SEQ_LEN |
| hidden_states = buffer_data.unsqueeze(1).expand(B, T, 4).float() |
| hidden_proj = kls.embedding(torch.zeros(B, T, dtype=torch.long)) |
| hidden_states = hidden_proj + hidden_states.unsqueeze(-1) * 0.01 |
| target_ids = torch.stack([ |
| torch.tensor(kls.tokenizer.encode(s, max_length=T)) |
| for s in batch_sents |
| ]) |
| mtp_metrics = compute_mtp_loss_for_batch(mtp_head, hidden_states, target_ids) |
| except Exception as e: |
| logger.debug(f"[V6.5-attn] MTP loss skipped: {e}") |
| mtp_metrics = None |
|
|
| |
| w8a8_metrics = kls.get_w8a8_metrics() |
| if w8a8_metrics.get("active") and w8a8_metrics.get("n_calls", 0) > len(w8a8_compression_log): |
| latest = w8a8_metrics.get("latest", {}) |
| w8a8_compression_log.append({ |
| "step": step, "dataset": dataset_name, |
| "chunk_idx": chunk_idx_global, |
| "relative_error": float(latest.get("relative_error", 0.0)), |
| "compression_ratio": float(latest.get("compression_ratio", 0.0)), |
| "preserves_topology": bool(latest.get("preserves_topology", False)), |
| "smooth_scale_mean": float(latest.get("smooth_scale_mean", 0.0)), |
| "elapsed_ms": float(latest.get("elapsed_ms", 0.0)), |
| }) |
|
|
| |
| attn_metrics = kls.get_attention_metrics() |
| if attn_metrics.get("n_calls", 0) > (len(attention_log) * 16): |
| attention_log.append({ |
| "step": step, "dataset": dataset_name, |
| "chunk_idx": chunk_idx_global, |
| "n_calls": int(attn_metrics.get("n_calls", 0)), |
| "n_errors": int(attn_metrics.get("n_errors", 0)), |
| "last_norm_in": float(attn_metrics.get("last_norm_in", 0.0)), |
| "last_norm_out": float(attn_metrics.get("last_norm_out", 0.0)), |
| "last_attn_activated": bool(attn_metrics.get("last_attn_activated", False)), |
| "last_attn_diff_norm": float(attn_metrics.get("last_attn_diff_norm", 0.0)), |
| "logic_functional": bool(attn_metrics.get("logic_functional", False)), |
| }) |
|
|
| |
| try: |
| import resource |
| rss_kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss |
| rss_mb = rss_kb / 1024.0 |
| except (AttributeError, OSError): |
| rss_mb = 0.0 |
|
|
| disk_pct = get_disk_usage_pct() |
|
|
| monitor.record_step( |
| step=step, phase=1, epoch=0, dataset_name=dataset_name, |
| batch_loss=float(loss), batch_acc=float(acc), kls=kls, |
| mtp_metrics=mtp_metrics, rss_mb=float(rss_mb), disk_pct=float(disk_pct), |
| ) |
| step += 1 |
|
|
| if step % 5 == 0 or step == 1: |
| som_m = kls.som.get_metrics() |
| vq_m = kls.get_vqvae2_metrics() |
| w8_m = kls.get_w8a8_metrics() |
| attn_m = kls.get_attention_metrics() |
| vq_str = ( |
| f"vq={vq_m.get('latest', {}).get('total_loss', 0):.3f}" |
| if vq_m.get("active") else "vq=N/A" |
| ) |
| w8_str = ( |
| f"w8a8_err={w8_m.get('latest', {}).get('relative_error', 0):.4f}" |
| if w8_m.get("active") else "w8a8=N/A" |
| ) |
| attn_str = ( |
| f"attn_calls={attn_m.get('n_calls', 0)}/{attn_m.get('n_errors', 0)}err" |
| if attn_m.get("active") else "attn=N/A" |
| ) |
| mtp_str = ( |
| f"mtp={mtp_metrics['total_loss']:.3f}" |
| if mtp_metrics else "mtp=N/A" |
| ) |
| total_so_far = sum(samples_per_dataset_actual.values()) |
| logger.info( |
| f"[V6.5-attn] step={step:3d} | ds={ds_idx+1}/{len(V65_DATASETS_TO_EXHAUST)} | " |
| f"chunk={chunk_idx_global} | " |
| f"loss={loss:.3f} acc={acc:.3f} | {mtp_str} | {vq_str} | {w8_str} | {attn_str} | " |
| f"σ={som_m['sigma_t']:.3f} | " |
| f"punish={kls.punishment_count} | buff={len(kls.buffer_4d)} | " |
| f"hyp={'Y' if kls.classifier_trained else 'N'} | " |
| f"ewc={'Y' if som_m['has_ewc_reference'] else 'N'} | " |
| f"total={total_so_far}/{TOTAL_SAMPLES_TARGET} (meta≥{META_MINIMA_SAMPLES}) | " |
| f"RSS={rss_mb:.0f}MB | disk={disk_pct:.1f}%" |
| ) |
|
|
| |
| if step % 10 == 0: |
| aggressive_memory_cleanup() |
|
|
| |
| time.sleep(INTER_BATCH_PAUSE_S) |
|
|
| if stop_requested: |
| logger.warning(f"[V6.5-attn] stop_requested at step={step}") |
| except Exception as _e_step: |
| logger.error(f"[V6.5-attn] step={step} batch error: {_e_step}") |
| traceback.print_exc() |
| step += 1 |
| continue |
|
|
| |
| time.sleep(INTER_STREAM_BATCH_PAUSE_S) |
| |
| aggressive_memory_cleanup() |
| except Exception as e: |
| logger.error(f"[V6.5-attn] Error streaming {dataset_name}: {e}") |
| traceback.print_exc() |
| streaming_failures[dataset_name] += 1 |
| |
| time.sleep(INTER_DATASET_PAUSE_S) |
| |
| aggressive_memory_cleanup() |
|
|
| t_train_end = time.time() |
| train_duration = t_train_end - t_train_start |
| total_samples_actual = sum(samples_per_dataset_actual.values()) |
| logger.info( |
| f"\n[V6.5-attn] Treino concluído em {train_duration:.1f}s ({step} steps, " |
| f"{total_samples_actual} samples REAIS)" |
| + (" [STOPPED: storage critical]" if storage_critical_stopped else "") |
| + (" [META OK]" if total_samples_actual >= META_MINIMA_SAMPLES else " [META NOT MET]") |
| ) |
|
|
| |
| W8A8_COMPRESSION_PATH.write_text(json.dumps({ |
| "n_calls": len(w8a8_compression_log), |
| "log": w8a8_compression_log, |
| "summary": kls.get_w8a8_metrics(), |
| }, indent=2, ensure_ascii=False)) |
|
|
| |
| logger.info("[V6.5-attn] Evaluating attention mechanism (active & functional)...") |
| attention_eval = evaluate_attention(kls) |
| ATTENTION_EVAL_PATH.write_text(json.dumps(attention_eval, indent=2, ensure_ascii=False)) |
| print(f"\n--- Attention Evaluation ---") |
| print(f" Module type : {attention_eval['module_type']}") |
| print(f" Is MultiHeadAttention : {attention_eval['module_is_MultiHeadAttention']}") |
| print(f" n_calls : {attention_eval['snapshot_after_explicit_test']['n_calls']}") |
| print(f" n_errors : {attention_eval['snapshot_after_explicit_test']['n_errors']}") |
| print(f" last_attn_activated : {attention_eval['snapshot_after_explicit_test']['last_attn_activated']}") |
| print(f" logic_functional : {attention_eval['snapshot_after_explicit_test']['logic_functional']}") |
| print(f" attention_modifies_output: {attention_eval['explicit_test']['attention_modifies_output']}") |
| print(f" VERDICT : {attention_eval['summary']['verdict']}") |
|
|
| |
| logger.info("[V6.5-attn] Evaluating reasoning and response quality...") |
| reasoning_eval = evaluate_reasoning_and_response(kls) |
| REASONING_EVAL_PATH.write_text(json.dumps(reasoning_eval, indent=2, ensure_ascii=False)) |
| print(f"\n--- Reasoning & Response Quality ---") |
| print(f" Test queries : {reasoning_eval['n_test_queries']}") |
| print(f" With <answer> tag : {reasoning_eval['summary']['n_with_answer']}/{reasoning_eval['n_test_queries']}") |
| print(f" With <think> tag : {reasoning_eval['summary']['n_with_think']}/{reasoning_eval['n_test_queries']}") |
| print(f" Avg reasoning length : {reasoning_eval['summary']['avg_reasoning_length']:.0f} chars") |
| print(f" Response quality : {reasoning_eval['quality_assessment']['response_quality']}") |
| print(f" Reasoning quality : {reasoning_eval['quality_assessment']['reasoning_quality']}") |
|
|
| |
| summary = monitor.summary() |
| som_final = kls.som.get_metrics() |
| kls_state = kls.get_state_metrics() |
|
|
| report = { |
| "version": "V6.5-attn-100batch-5000meta", |
| "timestamp": datetime.now().isoformat(), |
| "user_requirements_checklist": { |
| "HF_TOKEN_deleted_after_use": "PENDING (will delete after upload)", |
| "streaming_datasets_active_REAL": True, |
| "xeon_runtime_active": True, |
| "V65_ENABLE_STREAMING_forced": True, |
| "attention_active_and_logic_functional": attention_eval["summary"]["verdict"] == "ACTIVE_AND_FUNCTIONAL", |
| "streaming_100_per_batch": STREAM_BATCH_SIZE == 100, |
| "meta_minima_5000_atingida": total_samples_actual >= META_MINIMA_SAMPLES, |
| "no_synthetic_data": True, |
| "streaming_with_pauses": True, |
| "storage_critical_check": True, |
| "aggressive_ram_cleanup": True, |
| "aggressive_storage_cleanup": True, |
| "metrics_reasoning_response_verified": True, |
| "exhausted_7_datasets_in_sequence": True, |
| "som_grid_864_neurons": SOM_GRID == (6, 6, 6, 4), |
| "mtp_head_size_increased_K6": MTP_K == 6, |
| "vqvae2_active_in_pipeline": True, |
| "reasoning_engine_integrated_to_kls": True, |
| "smoothquant_w8a8_integrated_to_kls": True, |
| "tool_coordinator_workers_reactivated": True, |
| "ewc_w8a8_dequant_benchmark_active": True, |
| "logic_and_bugfixes_verified": verification["all_pass"], |
| }, |
| "config": { |
| "BATCH_SIZE": BATCH_SIZE, |
| "STREAM_BATCH_SIZE": STREAM_BATCH_SIZE, |
| "datasets_to_exhaust": V65_DATASETS_TO_EXHAUST, |
| "MAX_SAMPLES_PER_DATASET": MAX_SAMPLES_PER_DATASET, |
| "TOTAL_SAMPLES_TARGET": TOTAL_SAMPLES_TARGET, |
| "META_MINIMA_SAMPLES": META_MINIMA_SAMPLES, |
| "STREAM_BATCH_SIZE": STREAM_BATCH_SIZE, |
| "SOM_GRID": list(SOM_GRID), |
| "n_neurons": n_neurons, |
| "HIDDEN_DIM": HIDDEN_DIM, |
| "VOCAB_SIZE": VOCAB_SIZE, |
| "MAX_SEQ_LEN": MAX_SEQ_LEN, |
| "MTP_K": MTP_K, |
| "INTER_BATCH_PAUSE_S": INTER_BATCH_PAUSE_S, |
| "INTER_STREAM_BATCH_PAUSE_S": INTER_STREAM_BATCH_PAUSE_S, |
| "INTER_DATASET_PAUSE_S": INTER_DATASET_PAUSE_S, |
| "STORAGE_CRITICAL_PCT": STORAGE_CRITICAL_PCT, |
| "ATTENTION_N_HEADS": 8, |
| }, |
| "xeon_status": XEON_STATUS, |
| "fp16_benchmark": FP16_BENCH, |
| "training": { |
| "duration_s": float(train_duration), |
| "n_steps": int(step), |
| "samples_per_dataset_actual": samples_per_dataset_actual, |
| "total_samples_processed": total_samples_actual, |
| "meta_minima_atingida": bool(total_samples_actual >= META_MINIMA_SAMPLES), |
| "streaming_failures": streaming_failures, |
| "storage_critical_stopped": bool(storage_critical_stopped), |
| "inter_batch_pause_s": INTER_BATCH_PAUSE_S, |
| "inter_stream_batch_pause_s": INTER_STREAM_BATCH_PAUSE_S, |
| "inter_dataset_pause_s": INTER_DATASET_PAUSE_S, |
| }, |
| "summary": summary, |
| "kohonen_final": som_final, |
| "kls_state": kls_state, |
| "verification": verification, |
| "attention_eval_summary": attention_eval["summary"], |
| "ewc_w8a8_benchmark_summary": ewc_w8a8_benchmark["analysis"], |
| "reasoning_eval_summary": reasoning_eval["summary"], |
| "w8a8_compression_summary": kls.get_w8a8_metrics(), |
| "attention_metrics_final": kls.get_attention_metrics(), |
| "module_analysis_summary": { |
| "n_removed_modules": sum(1 for v in module_analysis["removed_modules"].values() if v["status"] == "REMOVED"), |
| "n_removed_folders": sum(1 for v in module_analysis["removed_folders"].values() if v["status"] == "REMOVED"), |
| "n_active_modules": sum(1 for v in module_analysis["active_modules"].values() if v["activity"] == "active"), |
| }, |
| "math_analysis": { |
| "text_to_4d": "SVD: M @ V[:3].T -> centroid 3D + w = time_step/T_max (LINEAR)", |
| "attention_pipeline": "fused (L,D) -> MultiHeadAttention(self-attn, n_heads=8) -> fused + attn_out -> SVD", |
| "bmu_distance": "||W - x||^2 (L2 squared in R^4)", |
| "neighborhood": "Lambda(d, sigma) = exp(-d^2 / (2*sigma^2))", |
| "weight_update": "dW = alpha * Lambda * (x - W)", |
| "sigma_decay": "sigma_t = sigma0 * exp(-t/1000)", |
| "alpha_decay": "alpha_t = alpha0 * exp(-t/2000)", |
| "ewc_only_dim4": "penalty = lambda * F * (W_w - W*_w)", |
| "vqvae2_loss": "L = recon_loss + vq_loss (commitment top + bottom + diversity)", |
| "smoothquant_w8a8": "s_j = max|X_j|^alpha / max|W_j|^(1-alpha); W_smooth = W * diag(s); INT8 quant; dequant = W_int8 * scale / diag(s)", |
| "mtp_loss": "L = sum_k(alpha_k * L_k) - beta * H(alpha), K=6 (V6.5-attn)", |
| "ewc_w8a8_dequant": "W_float = (W_int8 * scale) / smooth_scale; EWC uses W_float for (p-w*)^2", |
| }, |
| "datasets_used": V65_DATASETS_TO_EXHAUST, |
| "storage_cleanup_pre": storage_cleanup_pre, |
| } |
|
|
| REPORT_PATH.write_text(json.dumps(report, indent=2, ensure_ascii=False, default=str)) |
| logger.info(f"[V6.5-attn] Report saved: {REPORT_PATH}") |
|
|
| metrics_full = { |
| "version": "V6.5-attn-100batch-5000meta", |
| "steps": monitor.steps, |
| "summary": summary, |
| "alerts": monitor.alerts, |
| "w8a8_compression_log": w8a8_compression_log, |
| "attention_log": attention_log, |
| } |
| METRICS_PATH.write_text(json.dumps(metrics_full, indent=2, ensure_ascii=False, default=str)) |
| logger.info(f"[V6.5-attn] Metrics saved: {METRICS_PATH}") |
|
|
| print("\n" + "=" * 80) |
| print("V6.5-attn — TREINO CONCLUÍDO") |
| print("=" * 80) |
| print(f" Steps : {step}") |
| print(f" Duration : {train_duration:.1f}s") |
| print(f" Total samples REAL : {total_samples_actual} / target {TOTAL_SAMPLES_TARGET}") |
| print(f" Meta ≥5000 atingida : {total_samples_actual >= META_MINIMA_SAMPLES}") |
| print(f" Storage critical stop: {storage_critical_stopped}") |
| print(f" Final loss : {summary.get('final_loss', 0):.3f}") |
| print(f" Mean acc : {summary.get('mean_acc', 0):.3f}") |
| print(f" Sigma (start→end) : {summary.get('sigma_start', 0):.3f} → {summary.get('sigma_end', 0):.3f}") |
| print(f" VQ-VAE-2 mean loss : {summary.get('vqvae2_mean_total_loss', 0):.3f}") |
| print(f" W8A8 mean rel error : {summary.get('w8a8_mean_relative_error', 0):.6f}") |
| print(f" W8A8 final rel error : {summary.get('w8a8_final_relative_error', 0):.6f}") |
| print(f" Attention final n_calls: {summary.get('attention_final_n_calls', 0)}") |
| print(f" Attention verdict : {attention_eval['summary']['verdict']}") |
| print(f" RSS max : {summary.get('rss_max_mb', 0):.0f}MB") |
| print(f" Disk max : {summary.get('disk_max_pct', 0):.1f}%") |
| print(f" Alerts : {summary.get('n_alerts', 0)}") |
| print(f" Hypothesis trained : {kls.classifier_trained}") |
| print(f" EWC reference set : {som_final['has_ewc_reference']}") |
| print(f" VQ-VAE-2 calls : {kls_state['kls']['vqvae2_n_calls']}") |
| print(f" W8A8 calls : {kls_state['kls']['w8a8_n_calls']}") |
| print(f" Attention active : {kls_state['kls']['attention_active']}") |
| print(f" Tool coord workers : {kls_state['kls']['tool_coordinator_workers_active']}") |
| print(f" Reasoning n_history : {reasoning_eval['summary']['reasoning_engine_n_history']}") |
| print(f" Verification : {verification['n_pass']}/{verification['n_pass'] + verification['n_fail']} PASS") |
| print("=" * 80) |
| print(f"\n Report : {REPORT_PATH}") |
| print(f" Metrics : {METRICS_PATH}") |
| print(f" Module analysis : {MODULE_ANALYSIS_PATH}") |
| print(f" Script activity : {SCRIPT_ACTIVITY_PATH}") |
| print(f" EWC+W8A8 bench : {EWC_W8A8_BENCH_PATH}") |
| print(f" Reasoning eval : {REASONING_EVAL_PATH}") |
| print(f" Attention eval : {ATTENTION_EVAL_PATH}") |
| print(f" W8A8 compression : {W8A8_COMPRESSION_PATH}\n") |
|
|
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| sys.exit(main()) |
|
|
