V6.5-7ds cleanup: remove legacy scripts/upload_v6_2_resilient.py

scripts/upload_v6_2_resilient.py

@@ -1,248 +0,0 @@
-"""upload_v6_2_resilient.py — Upload resiliente V6.2 para o HF.
-
-Estratégia (igual V6/V6.1):
-  - HfApi.upload_folder (single commit lógico)
-  - allow_patterns: *.py + *.json + *.md + *.txt + tokenizer.json
-  - ignore_patterns: __pycache__, *.pyc, _test_*, *.safetensors, *.bin, *.pt, *.pth
-  - hf_transfer habilitado
-
-V6.2 inclui (NOVO vs V6.1):
-  - src/bigru_t/model/hyp_t.py (V6.2 — pgvector_lookup_before_punishment()
-    consults SOM before applying punishment; COMPLETE → skip delta,
-    INCOMPLETE/MISSING → apply delta; tracks n_complete_skip, n_incomplete_apply,
-    n_missing_apply, skip_rate)
-  - scripts/train_v6_2.py (NOVO — 5 datasets × 50 samples + BATCH_SIZE=16 +
-    hyp pgvector capture)
-  - v6_2_report.json (relatório do treino V6.2)
-
-Uso:
-    export HF_TOKEN="hf_xxx..."  # env var temporária, será apagada após upload
-    python /home/z/my-project/scripts/upload_v6_2_resilient.py
-"""
-from __future__ import annotations
-
-import os
-import sys
-import time
-import json
-from pathlib import Path
-
-os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
-os.environ.setdefault("HF_HUB_ETAG_TIMEOUT", "60")
-os.environ.setdefault("PYTHONUNBUFFERED", "1")
-os.environ.setdefault("OMP_NUM_THREADS", "2")
-os.environ.setdefault("TOKENIZERS_PARALLELISM", "true")
-
-from huggingface_hub import HfApi  # noqa: E402
-
-REPO_ID = "PowerMachine/BiGRU_T_version"
-LOCAL_DIR = "/home/z/my-project/BiGRU_T_version"
-COMMIT_MESSAGE = """V6.2: HypT pgvector_lookup before punishment + BATCH_SIZE=16 + 5 datasets x 50
-
-V6.2 improvements (vs V6.1):
-- hyp_t.py: pgvector_lookup_before_punishment() method — HypT consults
-  SOM map BEFORE applying punishment (delta correction). If status ==
-  COMPLETE, punishment is SKIPPED (model trusts y_hat). If INCOMPLETE/
-  MISSING, punishment is applied normally.
-  (User requirement: "ative pgvector_lookup no nível de hipótese (hyp_t.py)
-   para que ele também consulte o mapa antes de aplicar punição")
-- hyp_t.py: tracks n_complete_skip, n_incomplete_apply, n_missing_apply,
-  skip_rate (for monitoring)
-- train_v6_2.py: BATCH_SIZE=16 (was 4) — user requirement
-- train_v6_2.py: 5 datasets from streaming_datasets.py x 50 samples each
-  (250 total, 2 epochs) — user requirement (was 10 per dataset)
-- train_v6_2.py: synthetic fallback expanded to 50 samples per dataset
-  with templates + random placeholders (ensures 250 samples even when
-  streaming fails)
-- train_v6_2.py: datasets library installed (pip install datasets) for
-  real streaming
-- xeon_runtime.py V6 always active (AVX512 + AMX_INT8 + IPEX + OneDNN)
-
-V6.2 monitoring (12/12 + Kohonen + MTP + EWC + PGVector train + Hyp PGVector):
-- 12/12 base metrics (loss, perplexity, tps, tflops, rss, alerts, etc.)
-- Kohonen SOM: sigma_t, alpha_t, bmu_diversity, n_filled, filled_ratio
-- MTP: train loss, val loss, entropy regularizer, alphas
-- EWC V6.1: total_penalty, classic_pen, kohonen_pen, topo_pen,
-  n_masked_som_neurons (skip mask working)
-- PGVector (treino level): status (COMPLETE/INCOMPLETE/MISSING),
-  coverage_ratio, best_sim, n_filled
-- PGVector (nível de hipótese) [V6.2 NEW]: status, apply_punishment,
-  skipped_by_pgvector, n_complete_skip, n_incomplete_apply,
-  n_missing_apply, skip_rate
-
-User requirement satisfied:
-- "ative pgvector_lookup no nível de hipótese (hyp_t.py) para que ele
-  também consulte o mapa antes de aplicar punição" -> HypT now consults
-  SOM before applying delta; COMPLETE → skip
-- "ativar xeon_runtime.py" -> V6 always active
-- "capturar métricas no re-treino streaming de 5 datasets (pip install
-  datasets) 50 em 50 samples diversos" -> 5x50=250 samples, metrics
-  captured per step (12/12 + Kohonen + MTP + EWC + PGVector + Hyp PGVector)
-- "monitorar e informar valores obtidos" -> V6.2 report includes all
-  monitored values + evolution
-- "BATCH_SIZE = 16" -> applied
-"""
-
-# Arquivos críticos V6.2 a serem verificados pós-upload
-CRITICAL_FILES_V6_2 = [
-    "src/bigru_t/model/hyp_t.py",          # V6.2 NOVO
-    "src/bigru_t/model/orq_cell.py",
-    "src/bigru_t/training/ewc.py",
-    "src/bigru_t/training/mtp.py",
-    "src/bigru_t/training/dynamic_repetition_penalty.py",
-    "src/bigru_t/training/data_augmentation.py",
-    "src/bigru_t/training/rlhf_mapping.py",
-    "src/bigru_t/attention/window_context.py",
-    "src/bigru_t/reasoning/thinking.py",
-    "src/bigru_t/tools/tool_mutation.py",
-    "src/bigru_t/tools/tool_cache.py",
-    "src/bigru_t/tools/tool_agent.py",
-    "src/bigru_t/integrations/langchain_compat.py",
-    "src/bigru_t/pipeline/optimized_pipeline.py",
-    "src/bigru_t/utils/xeon_runtime.py",
-    "v6_1_report.json",
-    "v6_2_report.json",                    # V6.2 NOVO
-]
-
-ALLOW_PATTERNS = [
-    "*.py", "*.json", "*.md", "*.txt",
-    "tokenizer.json",
-]
-
-IGNORE_PATTERNS = [
-    "__pycache__", "*.pyc", "_test_*", "tests/",
-    "*.safetensors", "*.bin", "*.pt", "*.pth",
-    "*.so", "*.dylib", "*.dll",
-]
-
-
-def main():
-    print("=" * 72)
-    print(f"UPLOAD V6.2 RESILIENT — {REPO_ID}")
-    print("=" * 72)
-
-    token = os.environ.get("HF_TOKEN")
-    if not token:
-        print("[ERROR] HF_TOKEN não definido. Export HF_TOKEN=hf_xxx antes de rodar.")
-        sys.exit(1)
-    print(f"[INFO] HF_TOKEN presente (length={len(token)}, prefix={token[:5]}...)")
-
-    local_dir = Path(LOCAL_DIR)
-    if not local_dir.exists():
-        print(f"[ERROR] Diretório local não existe: {local_dir}")
-        sys.exit(1)
-
-    # Lista arquivos a enviar
-    files_to_upload = []
-    for pattern in ALLOW_PATTERNS:
-        files_to_upload.extend(local_dir.rglob(pattern))
-    # Filtra ignorados
-    files_filtered = []
-    for f in files_to_upload:
-        rel = f.relative_to(local_dir)
-        rel_str = str(rel)
-        if any(p in rel_str for p in IGNORE_PATTERNS if "*" not in p):
-            continue
-        if any(rel_str.endswith(p.replace("*", "")) for p in IGNORE_PATTERNS if "*" in p):
-            continue
-        files_filtered.append(rel_str)
-    files_filtered = sorted(set(files_filtered))
-    print(f"[INFO] {len(files_filtered)} arquivos locais a enviar")
-    print(f"[INFO] Primeiros 15 arquivos:")
-    for f in files_filtered[:15]:
-        size = (local_dir / f).stat().st_size
-        print(f"   - {f}  ({size} bytes)")
-    if len(files_filtered) > 15:
-        print(f"   ... e mais {len(files_filtered) - 15} arquivos")
-
-    api = HfApi(token=token)
-
-    # Verifica repo
-    print(f"\n[INFO] Verificando repositório {REPO_ID}…")
-    try:
-        api.repo_info(REPO_ID, repo_type="model")
-        print("[INFO] Repo já existia")
-    except Exception:
-        print("[INFO] Criando repo...")
-        api.create_repo(REPO_ID, repo_type="model", private=False)
-        print("[INFO] Repo criado")
-
-    # Lista arquivos atuais
-    try:
-        files_in_repo = [f.rfilename for f in api.list_repo_tree(REPO_ID, repo_type="model", recursive=True)]
-        print(f"[INFO] {len(files_in_repo)} arquivos no repo ANTES do upload")
-    except Exception:
-        files_in_repo = []
-
-    # Upload folder (single commit)
-    print("\n" + "=" * 72)
-    print("[INFO] Iniciando upload_folder (single commit)…")
-    print("=" * 72)
-    t0 = time.perf_counter()
-    commit_info = api.upload_folder(
-        folder_path=str(local_dir),
-        repo_id=REPO_ID,
-        repo_type="model",
-        commit_message=COMMIT_MESSAGE,
-        allow_patterns=ALLOW_PATTERNS,
-        ignore_patterns=IGNORE_PATTERNS,
-    )
-    elapsed = time.perf_counter() - t0
-    print(f"[INFO] upload_folder concluído em {elapsed:.2f}s")
-    print(f"[INFO] commit_info: {commit_info.commit_url if hasattr(commit_info, 'commit_url') else commit_info}")
-
-    # Aguarda indexação
-    print("\n[INFO] Aguardando 3s para o HF indexar…")
-    time.sleep(3)
-
-    # Verifica arquivos críticos V6.2
-    try:
-        files_in_repo_after = [f.rfilename for f in api.list_repo_tree(REPO_ID, repo_type="model", recursive=True)]
-        print(f"[INFO] {len(files_in_repo_after)} arquivos no repo DEPOIS do upload")
-        new_files = set(files_in_repo_after) - set(files_in_repo)
-        print(f"[INFO] Novos arquivos: {len(new_files)}")
-        overwritten = set(files_in_repo) & set(files_in_repo_after)
-        print(f"[INFO] Arquivos sobrescritos: {len(overwritten)}")
-    except Exception as e:
-        print(f"[WARN] Não foi possível listar arquivos pós-upload: {e}")
-        files_in_repo_after = []
-
-    print(f"\n[INFO] Verificação de arquivos críticos V6.2 no repo:")
-    all_critical_ok = True
-    for cf in CRITICAL_FILES_V6_2:
-        if cf in files_in_repo_after:
-            print(f"   ✓ {cf}")
-        else:
-            print(f"   ✗ {cf}  (NÃO ENCONTRADO)")
-            all_critical_ok = False
-
-    # Salva relatório
-    report_path = local_dir / "v6_2_upload_report.json"
-    upload_report = {
-        "version": "V6.2",
-        "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
-        "repo_id": REPO_ID,
-        "commit_url": str(commit_info.commit_url) if hasattr(commit_info, "commit_url") else str(commit_info),
-        "elapsed_seconds": elapsed,
-        "n_files_uploaded": len(files_filtered),
-        "n_files_in_repo_before": len(files_in_repo),
-        "n_files_in_repo_after": len(files_in_repo_after),
-        "new_files": list(new_files) if 'new_files' in locals() else [],
-        "critical_files_ok": all_critical_ok,
-        "critical_files": CRITICAL_FILES_V6_2,
-    }
-    with open(report_path, "w") as f:
-        json.dump(upload_report, f, indent=2, default=str)
-    print(f"\n[INFO] Relatório salvo: {report_path}")
-
-    print("\n" + "=" * 72)
-    if all_critical_ok:
-        print(f"[OK] UPLOAD V6.2 CONCLUÍDO — {len(files_filtered)} arquivos enviados")
-        print(f"[OK] URL: https://huggingface.co/{REPO_ID}")
-    else:
-        print(f"[WARN] UPLOAD V6.2 COMPLETO mas alguns arquivos críticos não verificados")
-    print("=" * 72)
-
-
-if __name__ == "__main__":
-    main()