Upload 6 files

app.py

@@ -1,6 +1,6 @@
 # ==============================================================================
-#  API do AetherMap — VERSÃO 7.3 (KNOWLEDGE GRAPH EDITION)
-#  Backend com RAG Híbrido, CSV, Tavily, NER Entity Graph
+#  API do AetherMap — VERSÃO 7.4 (FAISS EDITION)
+#  Backend com RAG Híbrido, CSV, Tavily, NER Entity Graph, FAISS ANN
 # ==============================================================================
 
 import numpy as np
@@ -30,6 +30,7 @@ from sklearn.preprocessing import StandardScaler
 from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
 from scipy.stats import entropy
+import faiss
 
 # NER & Language Detection
 import spacy
@@ -641,9 +642,19 @@ async def process_api(
 
         df, embeddings = processar_pipeline(textos)
         
+        # Criar índice FAISS para busca rápida
+        embeddings_normalized = embeddings / np.linalg.norm(embeddings, axis=1, keepdims=True)
+        faiss_index = faiss.IndexFlatIP(embeddings_normalized.shape[1])  # Inner Product = Cosine sim para vetores normalizados
+        faiss_index.add(embeddings_normalized.astype('float32'))
+        
         job_id = str(uuid.uuid4())
-        cache[job_id] = {"embeddings": embeddings, "df": df}
-        logging.info(f"Job criado: {job_id}")
+        cache[job_id] = {
+            "embeddings": embeddings, 
+            "embeddings_normalized": embeddings_normalized,
+            "faiss_index": faiss_index,
+            "df": df
+        }
+        logging.info(f"Job criado: {job_id} (FAISS index com {faiss_index.ntotal} vetores)")
 
         metricas_globais = calcular_metricas(df["full_text"].tolist())
         analise_de_duplicados = encontrar_duplicados(df, embeddings)
@@ -685,20 +696,34 @@ async def search_api(query: str = Form(...), job_id: str = Form(...)):
         
         cached_data = cache[job_id]
         df = cached_data["df"]
-        corpus_embeddings = cached_data["embeddings"]
+        faiss_index = cached_data.get("faiss_index")
         
-        # FASE 1: Varredura Ampla
+        # FASE 1: Busca FAISS (O(log N) ao invés de O(N))
         query_embedding = model.encode([query], convert_to_numpy=True)
-        similarities = cosine_similarity(query_embedding, corpus_embeddings)[0]
+        query_normalized = query_embedding / np.linalg.norm(query_embedding, axis=1, keepdims=True)
         
-        top_k_retrieval = 50 
-        top_indices = np.argsort(similarities)[-top_k_retrieval:][::-1]
+        top_k_retrieval = min(50, faiss_index.ntotal) if faiss_index else 50
+        
+        if faiss_index:
+            # FAISS search - retorna (distances, indices)
+            scores, top_indices = faiss_index.search(query_normalized.astype('float32'), top_k_retrieval)
+            scores = scores[0]  # Flatten
+            top_indices = top_indices[0]
+            logging.info(f"FAISS search: top score = {scores[0]:.3f}")
+        else:
+            # Fallback para busca bruta se não tiver FAISS
+            corpus_embeddings = cached_data["embeddings"]
+            similarities = cosine_similarity(query_embedding, corpus_embeddings)[0]
+            top_indices = np.argsort(similarities)[-top_k_retrieval:][::-1]
+            scores = similarities[top_indices]
         
         candidate_docs = []
         candidate_indices = []
         
-        for idx in top_indices:
-            if similarities[idx] > 0.15: 
+        for i, idx in enumerate(top_indices):
+            if idx < 0:  # FAISS pode retornar -1 se não tiver resultados suficientes
+                continue
+            if scores[i] > 0.15: 
                 doc_text = df.iloc[int(idx)]["full_text"]
                 candidate_docs.append([query, doc_text])
                 candidate_indices.append(int(idx))
@@ -886,13 +911,24 @@ async def search_web_api(
         # Processar através do pipeline com parâmetros para datasets pequenos
         df, embeddings = processar_pipeline(textos, small_dataset=True)
         
+        # Criar índice FAISS para busca rápida
+        embeddings_normalized = embeddings / np.linalg.norm(embeddings, axis=1, keepdims=True)
+        faiss_index = faiss.IndexFlatIP(embeddings_normalized.shape[1])
+        faiss_index.add(embeddings_normalized.astype('float32'))
+        
         # Adicionar URL de origem a cada ponto
         df["source_url"] = [sources[i]["url"] if i < len(sources) else "" for i in range(len(df))]
         df["source_title"] = [sources[i]["title"] if i < len(sources) else "" for i in range(len(df))]
         
         # Criar job e cachear
         job_id = str(uuid.uuid4())
-        cache[job_id] = {"embeddings": embeddings, "df": df, "sources": sources}
+        cache[job_id] = {
+            "embeddings": embeddings, 
+            "embeddings_normalized": embeddings_normalized,
+            "faiss_index": faiss_index,
+            "df": df, 
+            "sources": sources
+        }
         logging.info(f"Tavily Job criado: {job_id}")
         
         # Calcular métricas e análises

requirements.txt

@@ -15,6 +15,7 @@ scikit-learn
 scipy
 umap-learn
 hdbscan
+faiss-cpu
 nltk
 spacy
 langdetect