Update app.py

app.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import math
+import json
 from typing import Optional, Dict, Any, List
 
 import numpy as np
@@ -9,16 +10,18 @@ from scipy.linalg import expm
 
 from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel, Field
+from pydantic import ConfigDict  # para evitar warning de protected_namespaces
 
 from sentence_transformers import SentenceTransformer
 from huggingface_hub import hf_hub_download
 import joblib
 
-from datasets import load_dataset  # para /v1/rrf_tutor
+import torch
+import torch.nn as nn
 
 
 # ============================
-# Configuración de modelos
+# Configuración general
 # ============================
 
 HF_TOKEN = os.environ.get("HF_TOKEN", "")
@@ -26,7 +29,29 @@ HF_TOKEN = os.environ.get("HF_TOKEN", "")
 ENCODER_MODEL_ID     = "antonypamo/RRFSAVANTMADE"
 META_LOGIT_REPO      = "antonypamo/RRFSavantMetaLogit"
 META_LOGIT_FILENAME  = "logreg_rrf_savant_15.joblib"
-RRF_TUTOR_DATASET_ID = "antonypamo/savant_rrf1"
+
+# Dataset central con TODOS los artefactos
+RRF_DATASET_REPO     = "antonypamo/savant_rrf1_curated"
+
+
+def hf_data_path(filename: str) -> str:
+    """
+    Descarga un archivo desde el dataset antonypamo/savant_rrf1_curated
+    y devuelve la ruta local en cache.
+    """
+    return hf_hub_download(
+        repo_id=RRF_DATASET_REPO,
+        filename=filename,
+        repo_type="dataset",
+        token=HF_TOKEN or None,
+    )
+
+
+print("===== Application Startup =====", flush=True)
+
+# ============================
+# Cargar encoder y meta-logit
+# ============================
 
 print("🔄 [Startup] Cargando encoder RRFSAVANTMADE...", flush=True)
 try:
@@ -41,7 +66,7 @@ try:
     meta_logit_path = hf_hub_download(
         repo_id=META_LOGIT_REPO,
         filename=META_LOGIT_FILENAME,
-        token=HF_TOKEN if HF_TOKEN else None,  # si es público, puede ser None
+        token=HF_TOKEN or None,
     )
     print(f"🔄 [Startup] Cargando modelo meta-logit '{META_LOGIT_FILENAME}'...", flush=True)
     meta_logit = joblib.load(meta_logit_path)
@@ -55,6 +80,138 @@ except Exception as e:
     raise
 
 
+# ============================
+# Rutas a artefactos desde el dataset central
+# ============================
+
+def safe_hf(path_name: str) -> Optional[str]:
+    try:
+        p = hf_data_path(path_name)
+        print(f"✅ [Dataset] Descargado {path_name}", flush=True)
+        return p
+    except Exception as e:
+        print(f"⚠️ [Dataset] No se pudo descargar {path_name}: {e}", file=sys.stderr, flush=True)
+        return None
+
+
+SAVANT_CNN_PATH          = safe_hf("savant_cnn.pt")
+RRF_NODES_PATH           = safe_hf("rrf_nodes.pt")
+RRF_TUTOR_JSONL_PATH     = safe_hf("rrf_tutor_curated.jsonl")
+RRF_SEMANTIC_CORPUS_PATH = safe_hf("RRF_SAVANT_SEMANTIC_CORPUS.jsonl")
+RRF_CORPUS_INDEX_PATH    = safe_hf("RRF_SAVANT_CORPUS.index")
+
+PHYS_RRF_RESONANCE_MATRIX = safe_hf("rrf_resonance_matrix.csv")
+PHYS_RRF_ENERGY_PROFILE   = safe_hf("rrf_energy_profile.csv")
+PHYS_RRF_EIGEN_SPECTRUM   = safe_hf("rrf_eigen_spectrum.csv")
+
+PHYS_RES_MATRIX_13        = safe_hf("resonance_matrix_13.csv")
+PHYS_NODES_13             = safe_hf("nodes_13.csv")
+PHYS_ENERGY_LOGPHI_13     = safe_hf("energy_logphi_13.csv")
+PHYS_DEGREE_13            = safe_hf("degree_13.csv")
+PHYS_ADJ_13               = safe_hf("adjacency_13.csv")
+
+
+# ============================
+# Savant CNN + nodos RRF (para futura integración)
+# ============================
+
+class SavantCNN(nn.Module):
+    """
+    CNN tal como fue entrenada originalmente:
+    - conv1: [1 -> 32]
+    - conv2: [32 -> 64]
+    - conv3: [64 -> 128]
+    - fc:   [512 -> 64]  (según checkpoint original)
+    """
+    def __init__(self, in_channels: int = 1, out_dim: int = 64):
+        super().__init__()
+        self.conv1 = nn.Conv1d(in_channels, 32, kernel_size=3, padding=1)
+        self.conv2 = nn.Conv1d(32, 64, kernel_size=3, padding=1)
+        self.conv3 = nn.Conv1d(64, 128, kernel_size=3, padding=1)
+        self.pool = nn.AdaptiveAvgPool1d(1)
+        self.fc   = nn.Linear(512, out_dim)  # mantiene compatibilidad con checkpoint
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # x: [batch, channels, length]
+        x = torch.relu(self.conv1(x))
+        x = torch.relu(self.conv2(x))
+        x = torch.relu(self.conv3(x))
+        x = self.pool(x).squeeze(-1)  # [batch, 128] en este diseño simplificado
+        x = self.fc(x)
+        return x
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+savant_cnn: Optional[SavantCNN] = None
+rrf_nodes: Optional[Any] = None
+
+if SAVANT_CNN_PATH is not None:
+    try:
+        state_dict = torch.load(SAVANT_CNN_PATH, map_location=device)
+        print("✅ Checkpoint keys:", list(state_dict.keys()))
+        print("ℹ️ conv3.weight shape en checkpoint:", state_dict["conv3.weight"].shape)
+        print("ℹ️ fc.weight shape en checkpoint:", state_dict["fc.weight"].shape)
+
+        savant_cnn = SavantCNN()
+        savant_cnn.load_state_dict(state_dict)
+        savant_cnn.to(device)
+        savant_cnn.eval()
+        print("✅ Loaded Savant CNN from", SAVANT_CNN_PATH)
+    except Exception as e:
+        print("⚠️ Error loading Savant CNN:", e, file=sys.stderr)
+        savant_cnn = None
+else:
+    print("⚠️ SAVANT_CNN_PATH is None, no se cargó CNN.", file=sys.stderr)
+
+
+if RRF_NODES_PATH is not None:
+    try:
+        rrf_nodes = torch.load(RRF_NODES_PATH, map_location=device)
+        print("✅ Loaded RRF nodes from", RRF_NODES_PATH)
+        print("Type of rrf_nodes:", type(rrf_nodes))
+        if isinstance(rrf_nodes, dict):
+            print("🔑 rrf_nodes keys:", list(rrf_nodes.keys())[:10])
+    except Exception as e:
+        print("⚠️ Error loading RRF nodes:", e, file=sys.stderr)
+        rrf_nodes = None
+else:
+    print("⚠️ RRF_NODES_PATH is None, no se cargaron nodos.", file=sys.stderr)
+
+
+def fuse_cnn_with_node(example_length: int = 64):
+    """
+    Utilidad interna: ejemplo de cómo fusionar la CNN con un nodo RRF.
+    No se expone aún como endpoint, pero sirve para demos técnicas.
+    """
+    if savant_cnn is None or rrf_nodes is None:
+        print("Fusion not available – missing CNN or RRF nodes snapshot.")
+        return None
+
+    x = torch.randn(1, 1, example_length, device=device)
+    cnn_emb = savant_cnn(x)  # [1, 64]
+
+    try:
+        # asumir que el primer nodo es algo como rrf_nodes["node_0"]
+        node0_key = list(rrf_nodes.keys())[0]
+        node0 = rrf_nodes[node0_key]
+        if isinstance(node0, dict) and "linguistic" in node0:
+            linguistic_vec = node0["linguistic"]
+            if isinstance(linguistic_vec, torch.Tensor):
+                linguistic_vec = linguistic_vec.detach().clone().to(device)
+            else:
+                linguistic_vec = torch.tensor(linguistic_vec, dtype=torch.float32, device=device)
+        else:
+            linguistic_vec = torch.randn(cnn_emb.shape[-1], device=device)
+    except Exception:
+        linguistic_vec = torch.randn(cnn_emb.shape[-1], device=device)
+
+    linguistic_vec = linguistic_vec.unsqueeze(0)  # [1, 64]
+    fused = torch.cat([cnn_emb, linguistic_vec], dim=-1)  # [1, 128]
+    print("Fused embedding shape (CNN + linguistic node):", fused.shape)
+    return fused
+
+
 # ============================
 # Geometría icosaédrica Φ12.0
 # ============================
@@ -92,6 +249,7 @@ def geodesic_kernel(nodes, sigma=0.618, alpha_log=0.10):
 
     if alpha_log > 0.0:
         corr = 1.0 + alpha_log * np.log1p(dist ** 2)
+        # mantener diagonal en 1 para evitar log(0)
         corr[range(N), range(N)] = 1.0
         W = W / corr
 
@@ -197,23 +355,6 @@ def evolve_dirac_shell(psi0, H, dt=0.05, steps=100, record_every=25):
     }
 
 
-# ============================
-# Hamiltoniano base (startup)
-# ============================
-
-print("🔄 [Startup] Construyendo Hamiltoniano base Φ12.0...", flush=True)
-H_BASE = build_dirac_hamiltonian(
-    m=0.25,
-    v=1.0,
-    sigma=0.618,
-    alpha_log=0.10,
-    q=1.0,
-    flux_vector=(0.0, 0.0, 0.0),
-    gauge_scale=0.0,
-)
-print("✅ Hamiltoniano base construido.", flush=True)
-
-
 # ============================
 # Core RRF: embeddings + features + scores
 # ============================
@@ -224,21 +365,25 @@ def get_embedding(text: str) -> np.ndarray:
 
 
 def compute_rrf_features(prompt: str, answer: str) -> Dict[str, float]:
-    # Embeddings
     e_p = get_embedding(prompt)
     e_a = get_embedding(answer)
 
     cosine_pa = float(np.dot(e_p, e_a))
     len_ratio = len(answer) / (len(prompt) + 1.0)
 
-    # Simulación Dirac shell determinista (semilla por prompt+answer)
     rng = np.random.default_rng(abs(hash(prompt + answer)) % (2 ** 32))
     vec = rng.normal(0, 1, (2 * N,)) + 1j * rng.normal(0, 1, (2 * N,))
     vec /= np.sqrt(np.vdot(vec, vec))
     psi0 = vec
 
-    # Usamos el Hamiltoniano base global
-    out = evolve_dirac_shell(psi0, H_BASE, dt=0.05, steps=100, record_every=25)
+    H = build_dirac_hamiltonian(
+        m=0.25, v=1.0, sigma=0.618,
+        alpha_log=0.10, q=1.0,
+        flux_vector=(0.0, 0.0, 0.0),
+        gauge_scale=0.0,
+    )
+
+    out = evolve_dirac_shell(psi0, H, dt=0.05, steps=100, record_every=25)
 
     entropy = out["entropy"]
     energy = out["energy"]
@@ -251,7 +396,6 @@ def compute_rrf_features(prompt: str, answer: str) -> Dict[str, float]:
     E_mean = float(np.mean(energy))
     E_std = float(np.std(energy))
 
-    # Núcleo de 7 features
     feats: Dict[str, float] = {
         "cosine_pa": cosine_pa,
         "len_ratio": len_ratio,
@@ -262,7 +406,6 @@ def compute_rrf_features(prompt: str, answer: str) -> Dict[str, float]:
         "dirac_energy_std": E_std,
     }
 
-    # Derivadas para llegar a 15 (igual que en el CSV)
     S_max = math.log(N)
     feats["entropy_norm"]      = feats["dirac_entropy_final"] / S_max
     feats["entropy_abs_delta"] = abs(feats["dirac_entropy_delta"])
@@ -343,10 +486,7 @@ ROLE_PROFILES: Dict[str, Dict[str, float]] = {
 }
 
 
-def apply_role_profile(
-    scores: Dict[str, float],
-    role_name: Optional[str],
-) -> Dict[str, Any]:
+def apply_role_profile(scores: Dict[str, float], role_name: Optional[str]) -> Dict[str, Any]:
     if not role_name:
         role_name = "default"
 
@@ -370,52 +510,93 @@ def apply_role_profile(
 
 
 # ============================
-# RRF Tutor: carga de dataset savant_rrf1
+# RRF Tutor (curated JSONL)
 # ============================
 
-print(f"🔄 [Startup] Cargando dataset para RRF Tutor: {RRF_TUTOR_DATASET_ID}...", flush=True)
-try:
-    ds_rrf = load_dataset(RRF_TUTOR_DATASET_ID, split="train")
-    ds_rrf = ds_rrf.filter(
-        lambda ex: ex.get("prompt") is not None and ex.get("completion") is not None
-    )
-    print(f"✅ Dataset RRF Tutor cargado. Ejemplos útiles: {len(ds_rrf)}", flush=True)
-except Exception as e:
-    print(f"❌ Error cargando dataset RRF Tutor: {e}", file=sys.stderr, flush=True)
-    ds_rrf = None
-
-if ds_rrf is not None:
-    print("🔄 [Startup] Construyendo textos y embeddings para RRF Tutor...", flush=True)
-    rrf_corpus_texts: List[str] = []
-    rrf_corpus_prompts: List[str] = []
-    rrf_corpus_completions: List[str] = []
-
-    for ex in ds_rrf:
-        p = ex["prompt"]
-        c = ex["completion"]
-        rrf_corpus_prompts.append(p)
-        rrf_corpus_completions.append(c)
-        rrf_corpus_texts.append(p + "\n\n" + c)
-
-    rrf_corpus_embeds = encoder.encode(
-        rrf_corpus_texts,
-        convert_to_numpy=True,
-        show_progress_bar=True,
-        normalize_embeddings=True,
-    )
-    print("✅ [RRF Tutor] Embeddings construidos.", flush=True)
+print("🔄 [Startup] Cargando dataset RRF Tutor (curated JSONL)...", flush=True)
+rrf_corpus_texts: List[str] = []
+rrf_corpus_prompts: List[str] = []
+rrf_corpus_completions: List[str] = []
+
+if RRF_TUTOR_JSONL_PATH is not None:
+    try:
+        with open(RRF_TUTOR_JSONL_PATH, "r", encoding="utf-8") as f:
+            for line in f:
+                if not line.strip():
+                    continue
+                ex = json.loads(line)
+                p = ex.get("prompt")
+                c = ex.get("completion")
+                if p and c:
+                    rrf_corpus_prompts.append(p)
+                    rrf_corpus_completions.append(c)
+                    rrf_corpus_texts.append(p + "\n\n" + c)
+
+        if rrf_corpus_texts:
+            print(f"✅ RRF Tutor: {len(rrf_corpus_texts)} ejemplos cargados.", flush=True)
+            rrf_corpus_embeds = encoder.encode(
+                rrf_corpus_texts,
+                convert_to_numpy=True,
+                show_progress_bar=True,
+                normalize_embeddings=True,
+            )
+            print("✅ [RRF Tutor] Embeddings construidos.", flush=True)
+        else:
+            print("⚠️ RRF Tutor JSONL no tiene ejemplos válidos.", file=sys.stderr, flush=True)
+            rrf_corpus_embeds = np.zeros((0, 384), dtype=np.float32)
+    except Exception as e:
+        print(f"❌ Error cargando/parsing RRF Tutor JSONL: {e}", file=sys.stderr, flush=True)
+        rrf_corpus_embeds = np.zeros((0, 384), dtype=np.float32)
 else:
-    # Dimensión consistente con el encoder
-    dim = encoder.get_sentence_embedding_dimension()
-    rrf_corpus_texts = []
-    rrf_corpus_prompts = []
-    rrf_corpus_completions = []
-    rrf_corpus_embeds = np.zeros((0, dim), dtype=np.float32)
-    print("⚠️ [RRF Tutor] Dataset no disponible, el endpoint devolverá error si se usa.", flush=True)
+    print("⚠️ No se encontró RRF_TUTOR_JSONL_PATH.", file=sys.stderr, flush=True)
+    rrf_corpus_embeds = np.zeros((0, 384), dtype=np.float32)
+
+
+def rrf_tutor_retrieve_examples(query: str, top_k: int = 3):
+    if rrf_corpus_embeds is None or len(rrf_corpus_embeds) == 0:
+        raise RuntimeError("Embeddings de RRF Tutor no están disponibles.")
+
+    q_emb = encoder.encode([query], convert_to_numpy=True, normalize_embeddings=True)[0]
+    sims = np.dot(rrf_corpus_embeds, q_emb)
+
+    top_k = min(top_k, len(rrf_corpus_embeds))
+    top_idx = np.argsort(-sims)[:top_k]
+
+    results = []
+    for idx in top_idx:
+        results.append(
+            {
+                "idx": int(idx),
+                "score": float(sims[idx]),
+                "prompt": rrf_corpus_prompts[idx],
+                "completion": rrf_corpus_completions[idx],
+            }
+        )
+    return results
+
+
+def rrf_tutor_build_answer(query: str, retrieved_examples):
+    if not retrieved_examples:
+        return (
+            "No encontré ejemplos relevantes en el dataset RRF Tutor para tu consulta. "
+            "Intenta reformular la pregunta o revisar la configuración del dataset."
+        )
+
+    best = retrieved_examples[0]
+    base_completion = best["completion"]
+
+    answer = (
+        "🔎 Respuesta basada en el ejemplo más cercano del corpus RRF:\n\n"
+        f"{base_completion}\n\n"
+        "💡 Nota: Esta es una versión mínima que reutiliza directamente la 'completion' "
+        "del ejemplo más similar en savant_rrf1_curated. En una versión extendida, aquí se "
+        "conectaría un LLM pequeño que use varios ejemplos como contexto."
+    )
+    return answer
 
 
 # ============================
-# FastAPI app & modelos Pydantic
+# FastAPI models
 # ============================
 
 class EvaluateRequest(BaseModel):
@@ -466,6 +647,9 @@ class RerankDocumentResult(BaseModel):
 
 
 class RerankResponse(BaseModel):
+    # evitar warning con 'model_id'
+    model_config = ConfigDict(protected_namespaces=())
+
     model_id: str
     alpha: float
     query_embedding_norm: bool
@@ -476,11 +660,11 @@ class RRFTutorRequest(BaseModel):
     query: str = Field(..., description="Pregunta o fragmento de ecuación/idea RRF.")
     max_examples: int = Field(
         3, ge=1, le=8,
-        description="Número de ejemplos de savant_rrf1 a recuperar (1-8)."
+        description="Número de ejemplos de savant_rrf1_curated a recuperar (1-8).",
     )
     include_raw_context: bool = Field(
         False,
-        description="Si es true, devuelve los ejemplos recuperados."
+        description="Si es true, devuelve los ejemplos recuperados.",
     )
 
 
@@ -495,10 +679,14 @@ class RRFTutorResponse(BaseModel):
     retrieved: Optional[List[RetrievedExample]] = None
 
 
+# ============================
+# FastAPI app
+# ============================
+
 app = FastAPI(
     title="Savant RRF Φ12.0 API",
-    description="Dirac-Resonant conceptual quality layer + reranking + RRF Tutor.",
-    version="1.1.0",
+    description="Dirac-Resonant conceptual quality layer + reranking + RRF Tutor (+ CNN/nodes listos).",
+    version="1.2.0",
 )
 
 
@@ -543,54 +731,6 @@ def _compute_rerank_scores(query: str, docs: List[str], alpha: float, norm_query
     return reranked
 
 
-# ============================
-# Utilidades /v1/rrf_tutor
-# ============================
-
-def rrf_tutor_retrieve_examples(query: str, top_k: int = 3):
-    if rrf_corpus_embeds is None or len(rrf_corpus_embeds) == 0:
-        raise RuntimeError("Embeddings de RRF Tutor no están disponibles.")
-
-    q_emb = encoder.encode([query], convert_to_numpy=True, normalize_embeddings=True)[0]
-    sims = np.dot(rrf_corpus_embeds, q_emb)
-
-    top_k = min(top_k, len(rrf_corpus_embeds))
-    top_idx = np.argsort(-sims)[:top_k]
-
-    results = []
-    for idx in top_idx:
-        results.append(
-            {
-                "idx": int(idx),
-                "score": float(sims[idx]),
-                "prompt": rrf_corpus_prompts[idx],
-                "completion": rrf_corpus_completions[idx],
-            }
-        )
-    return results
-
-
-def rrf_tutor_build_answer(query: str, retrieved_examples):
-    if not retrieved_examples:
-        return (
-            "No encontré ejemplos relevantes en el dataset RRF Tutor para tu consulta. "
-            "Intenta reformular la pregunta o revisar la configuración del dataset."
-        )
-
-    best = retrieved_examples[0]
-    base_completion = best["completion"]
-
-    answer = (
-        "🔎 Respuesta basada en el ejemplo más cercano del corpus RRF:\n\n"
-        f"{base_completion}\n\n"
-        "💡 Nota: Esta es una versión mínima que reutiliza directamente la 'completion' "
-        "del ejemplo más similar en savant_rrf1. En una versión extendida, aquí se "
-        "conectaría un LLM pequeño (TinyLlama, etc.) que use varios ejemplos como "
-        "contexto para generar una explicación personalizada a tu `query`."
-    )
-    return answer
-
-
 # ============================
 # Endpoints
 # ============================
@@ -607,6 +747,9 @@ def health():
         "encoder_model_id": ENCODER_MODEL_ID,
         "meta_logit_filename": META_LOGIT_FILENAME,
         "N_sites": N,
+        "rrf_tutor_examples": len(rrf_corpus_texts),
+        "cnn_loaded": savant_cnn is not None,
+        "rrf_nodes_loaded": rrf_nodes is not None,
     }
 
 
@@ -626,14 +769,19 @@ def evaluate(req: EvaluateRequest):
 
         role_profile = apply_role_profile(scores, req.model_label)
 
-        # Segunda simulación solo para resumen visual
+        H = build_dirac_hamiltonian(
+            m=0.25, v=1.0, sigma=0.618,
+            alpha_log=0.10, q=1.0,
+            flux_vector=(0.0, 0.0, 0.0),
+            gauge_scale=0.0,
+        )
         rng = np.random.default_rng(
             abs(hash(req.prompt + req.answer + "sim")) % (2 ** 32)
         )
         vec = rng.normal(0, 1, (2 * N,)) + 1j * rng.normal(0, 1, (2 * N,))
         vec /= np.sqrt(np.vdot(vec, vec))
         psi0 = vec
-        sim = evolve_dirac_shell(psi0, H_BASE, dt=0.05, steps=60, record_every=20)
+        sim = evolve_dirac_shell(psi0, H, dt=0.05, steps=60, record_every=20)
 
         sim_summary = {
             "entropy_initial": float(sim["entropy"][0]),
@@ -691,10 +839,10 @@ def rrf_tutor_endpoint(body: RRFTutorRequest):
     if not body.query or not body.query.strip():
         raise HTTPException(status_code=400, detail="El campo 'query' no puede estar vacío.")
 
-    if ds_rrf is None or rrf_corpus_embeds is None or len(rrf_corpus_embeds) == 0:
+    if rrf_corpus_embeds is None or len(rrf_corpus_embeds) == 0:
         raise HTTPException(
             status_code=500,
-            detail="El dataset/embeddings de RRF Tutor no están disponibles en este momento."
+            detail="El dataset/embeddings de RRF Tutor no están disponibles en este momento.",
         )
 
     try: