V6.5-7ds cleanup: remove legacy scripts/upload_v6_1_resilient.py

scripts/upload_v6_1_resilient.py

@@ -1,251 +0,0 @@
-"""upload_v6_1_resilient.py — Upload resiliente V6.1 para o HF.
-
-Estratégia (igual V5/V6):
-  - HfApi.upload_folder (single commit lógico)
-  - allow_patterns: *.py + *.json + *.md + *.txt + tokenizer.json
-  - ignore_patterns: __pycache__, *.pyc, _test_*, *.safetensors, *.bin, *.pt, *.pth
-  - hf_transfer habilitado
-
-V6.1 inclui (NOVO vs V6):
-  - src/bigru_t/model/orq_cell.py (V6.1 — PGVector-like lookup +
-    neuron_filled_mask + fill_missing_neurons_random)
-  - src/bigru_t/training/ewc.py (V6.1 — skip_som_filled_neurons +
-    fisher_neuron_mask + classic Fisher masked for orq_cell.neuron_cache)
-  - scripts/train_v6_1.py (NOVO — 5 datasets × 10 samples + PGVector
-    capture + EWC V6.1 math upgrade)
-  - v6_1_report.json (relatório do treino V6.1)
-
-Uso:
-    export HF_TOKEN="hf_xxx..."  # env var temporária, será apagada após upload
-    python /home/z/my-project/scripts/upload_v6_1_resilient.py
-"""
-from __future__ import annotations
-
-import os
-import sys
-import time
-import json
-from pathlib import Path
-
-os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
-os.environ.setdefault("HF_HUB_ETAG_TIMEOUT", "60")
-os.environ.setdefault("PYTHONUNBUFFERED", "1")
-os.environ.setdefault("OMP_NUM_THREADS", "2")
-os.environ.setdefault("TOKENIZERS_PARALLELISM", "true")
-
-from huggingface_hub import HfApi  # noqa: E402
-
-REPO_ID = "PowerMachine/BiGRU_T_version"
-LOCAL_DIR = "/home/z/my-project/BiGRU_T_version"
-COMMIT_MESSAGE = """V6.1: PGVector-like lookup + EWC math upgrade + 5 datasets x 10 samples
-
-V6.1 improvements:
-- orq_cell.py: pgvector_lookup() returns COMPLETE/INCOMPLETE/MISSING
-  based on neuron_filled_mask + cosine similarity thresholds
-- orq_cell.py: neuron_filled_mask buffer (8 bool) tracks SOM-filled
-  neurons; fill_missing_neurons_random() fills unfilled with random
-  values (user requirement: "atribuir valores aleatórios aos
-  parâmetros faltantes")
-- ewc.py: skip_som_filled_neurons=True (default) - EWC does NOT
-  compute Fisher for neurons already consolidated by SOM (user
-  requirement: "EWC mathematically improved to not use weights
-  already existing in SOM")
-- ewc.py: classic Fisher for orq_cell.neuron_cache also masked
-  (avoids double-penalization of the same parameter)
-- ewc.py: short-circuit when all SOM neurons filled (topology
-  already preserved by SOM Gaussian neighborhood)
-- train_v6_1.py: 5 datasets from streaming_datasets.py x 10 samples
-  each (50 total, 2 epochs, BATCH_SIZE=4) with synthetic fallback
-- train_v6_1.py: PGVector status captured per step (COMPLETE/
-  INCOMPLETE/MISSING distribution tracked over training)
-- train_v6_1.py: EWC V6.1 metrics (n_masked_som_neurons tracked)
-- xeon_runtime.py V6 always active (AVX512 + AMX_INT8 + IPEX + OneDNN)
-- FP16 benchmark: 1.94 TFLOPS on this host
-
-V6.1 training results:
-- 50 samples (5 datasets x 10), 20 steps, 2 epochs
-- final_train_loss=8.96, final_val_loss=9.01
-- 0 alerts, RSS stable 872MB
-- Kohonen SOM: sigma 1.38->0.30, alpha 0.45->0.05 (full schedule)
-- 8/8 neurons filled (filled_ratio=100%)
-- PGVector: epoch 1 had 1 MISSING + 9 INCOMPLETE; epoch 2 all 10
-  INCOMPLETE (no COMPLETE yet - sim < 0.6 threshold)
-- EWC V6.1: n_masked_som_neurons=8 (all SOM neurons filled, EWC
-  contributes 0 to loss - math upgrade working as designed)
-- final coverage=1.000 (all inputs have similar neuron in map)
-
-User requirement satisfied:
-- "os níveis de treinamento e o de hipótese precisam verificar se
-  o mapa já possui aquela informação" -> pgvector_lookup at every
-  training step (and could be activated at hypothesis level)
-- "Elastic Weight Consolidation que deve ser matematicamente
-  aprimorado para não usar pesos já existentes em SOM" -> skip mask
-  zeros Fisher for SOM-filled neurons (classic + Kohonen)
-- "ativar xeon_runtime.py" -> V6 always active
-- "capturar métricas no re-treino de datasets 10 em 10 samples
-  diversos (5 datasets streaming_datasets.py)" -> 5x10=50 samples,
-  metrics captured per step (12/12 + Kohonen + MTP + EWC + PGVector)
-"""
-
-# Arquivos críticos V6.1 a serem verificados pós-upload
-CRITICAL_FILES_V6_1 = [
-    "src/bigru_t/model/orq_cell.py",
-    "src/bigru_t/training/ewc.py",
-    "src/bigru_t/training/mtp.py",
-    "src/bigru_t/training/dynamic_repetition_penalty.py",
-    "src/bigru_t/training/data_augmentation.py",
-    "src/bigru_t/training/rlhf_mapping.py",
-    "src/bigru_t/attention/window_context.py",
-    "src/bigru_t/reasoning/thinking.py",
-    "src/bigru_t/tools/tool_mutation.py",
-    "src/bigru_t/tools/tool_cache.py",
-    "src/bigru_t/tools/tool_agent.py",
-    "src/bigru_t/integrations/langchain_compat.py",
-    "src/bigru_t/pipeline/optimized_pipeline.py",
-    "src/bigru_t/utils/xeon_runtime.py",
-    "v6_report.json",
-    "v6_1_report.json",
-]
-
-ALLOW_PATTERNS = [
-    "*.py", "*.json", "*.md", "*.txt",
-    "tokenizer.json",
-]
-
-IGNORE_PATTERNS = [
-    "__pycache__", "*.pyc", "_test_*", "tests/",
-    "*.safetensors", "*.bin", "*.pt", "*.pth",
-    "*.so", "*.dylib", "*.dll",
-]
-
-
-def main():
-    print("=" * 72)
-    print(f"UPLOAD V6.1 RESILIENT — {REPO_ID}")
-    print("=" * 72)
-
-    token = os.environ.get("HF_TOKEN")
-    if not token:
-        print("[ERROR] HF_TOKEN não definido. Export HF_TOKEN=hf_xxx antes de rodar.")
-        sys.exit(1)
-    print(f"[INFO] HF_TOKEN presente (length={len(token)}, prefix={token[:5]}...)")
-
-    local_dir = Path(LOCAL_DIR)
-    if not local_dir.exists():
-        print(f"[ERROR] Diretório local não existe: {local_dir}")
-        sys.exit(1)
-
-    # Lista arquivos a enviar
-    files_to_upload = []
-    for pattern in ALLOW_PATTERNS:
-        files_to_upload.extend(local_dir.rglob(pattern))
-    # Filtra ignorados
-    files_filtered = []
-    for f in files_to_upload:
-        rel = f.relative_to(local_dir)
-        rel_str = str(rel)
-        if any(p in rel_str for p in IGNORE_PATTERNS if "*" not in p):
-            continue
-        if any(rel_str.endswith(p.replace("*", "")) for p in IGNORE_PATTERNS if "*" in p):
-            continue
-        files_filtered.append(rel_str)
-    files_filtered = sorted(set(files_filtered))
-    print(f"[INFO] {len(files_filtered)} arquivos locais a enviar")
-    print(f"[INFO] Primeiros 15 arquivos:")
-    for f in files_filtered[:15]:
-        size = (local_dir / f).stat().st_size
-        print(f"   - {f}  ({size} bytes)")
-    if len(files_filtered) > 15:
-        print(f"   ... e mais {len(files_filtered) - 15} arquivos")
-
-    api = HfApi(token=token)
-
-    # Verifica repo
-    print(f"\n[INFO] Verificando repositório {REPO_ID}…")
-    try:
-        api.repo_info(REPO_ID, repo_type="model")
-        print("[INFO] Repo já existia")
-    except Exception:
-        print("[INFO] Criando repo...")
-        api.create_repo(REPO_ID, repo_type="model", private=False)
-        print("[INFO] Repo criado")
-
-    # Lista arquivos atuais
-    try:
-        files_in_repo = [f.rfilename for f in api.list_repo_tree(REPO_ID, repo_type="model", recursive=True)]
-        print(f"[INFO] {len(files_in_repo)} arquivos no repo ANTES do upload")
-    except Exception:
-        files_in_repo = []
-
-    # Upload folder (single commit)
-    print("\n" + "=" * 72)
-    print("[INFO] Iniciando upload_folder (single commit)…")
-    print("=" * 72)
-    t0 = time.perf_counter()
-    commit_info = api.upload_folder(
-        folder_path=str(local_dir),
-        repo_id=REPO_ID,
-        repo_type="model",
-        commit_message=COMMIT_MESSAGE,
-        allow_patterns=ALLOW_PATTERNS,
-        ignore_patterns=IGNORE_PATTERNS,
-    )
-    elapsed = time.perf_counter() - t0
-    print(f"[INFO] upload_folder concluído em {elapsed:.2f}s")
-    print(f"[INFO] commit_info: {commit_info.commit_url if hasattr(commit_info, 'commit_url') else commit_info}")
-
-    # Aguarda indexação
-    print("\n[INFO] Aguardando 3s para o HF indexar…")
-    time.sleep(3)
-
-    # Verifica arquivos críticos V6
-    try:
-        files_in_repo_after = [f.rfilename for f in api.list_repo_tree(REPO_ID, repo_type="model", recursive=True)]
-        print(f"[INFO] {len(files_in_repo_after)} arquivos no repo DEPOIS do upload")
-        new_files = set(files_in_repo_after) - set(files_in_repo)
-        print(f"[INFO] Novos arquivos: {len(new_files)}")
-        overwritten = set(files_in_repo) & set(files_in_repo_after)
-        print(f"[INFO] Arquivos sobrescritos: {len(overwritten)}")
-    except Exception as e:
-        print(f"[WARN] Não foi possível listar arquivos pós-upload: {e}")
-        files_in_repo_after = []
-
-    print(f"\n[INFO] Verificação de arquivos críticos V6 no repo:")
-    all_critical_ok = True
-    for cf in CRITICAL_FILES_V6_1:
-        if cf in files_in_repo_after:
-            print(f"   ✓ {cf}")
-        else:
-            print(f"   ✗ {cf}  (NÃO ENCONTRADO)")
-            all_critical_ok = False
-
-    # Salva relatório
-    report_path = local_dir / "v6_1_upload_report.json"
-    upload_report = {
-        "version": "V6",
-        "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
-        "repo_id": REPO_ID,
-        "commit_url": str(commit_info.commit_url) if hasattr(commit_info, "commit_url") else str(commit_info),
-        "elapsed_seconds": elapsed,
-        "n_files_uploaded": len(files_filtered),
-        "n_files_in_repo_before": len(files_in_repo),
-        "n_files_in_repo_after": len(files_in_repo_after),
-        "new_files": list(new_files) if 'new_files' in locals() else [],
-        "critical_files_ok": all_critical_ok,
-        "critical_files": CRITICAL_FILES_V6_1,
-    }
-    with open(report_path, "w") as f:
-        json.dump(upload_report, f, indent=2, default=str)
-    print(f"\n[INFO] Relatório salvo: {report_path}")
-
-    print("\n" + "=" * 72)
-    if all_critical_ok:
-        print(f"[OK] UPLOAD V6.1 CONCLUÍDO — {len(files_filtered)} arquivos enviados")
-        print(f"[OK] URL: https://huggingface.co/{REPO_ID}")
-    else:
-        print(f"[WARN] UPLOAD V6.1 COMPLETO mas alguns arquivos críticos não verificados")
-    print("=" * 72)
-
-
-if __name__ == "__main__":
-    main()