Upload 6 files

app.py

@@ -202,25 +202,41 @@ def get_csv_columns(file_bytes: bytes) -> List[str]:
     return df.columns.tolist()
 
 
-
-def processar_pipeline(textos: List[str]) -> (pd.DataFrame, np.ndarray):
-    logging.info(f"Iniciando pipeline para {len(textos)} textos...")
+def processar_pipeline(textos: List[str], small_dataset: bool = False) -> (pd.DataFrame, np.ndarray):
+    """
+    Processa textos através do pipeline de embeddings, UMAP e HDBSCAN.
+    
+    Args:
+        textos: Lista de textos
+        small_dataset: Se True, usa parâmetros otimizados para datasets pequenos (Tavily)
+    """
+    logging.info(f"Iniciando pipeline para {len(textos)} textos (small_dataset={small_dataset})...")
     model = load_retriever()
     
     # 1. Embeddings
     embeddings = model.encode(textos, batch_size=BATCH_SIZE, show_progress_bar=False, convert_to_numpy=True)
     
-    # 2. UMAP
-    reducer = umap.UMAP(n_components=3, n_neighbors=UMAP_N_NEIGHBORS, min_dist=0.0, metric="cosine", random_state=42)
+    # 2. UMAP - ajustar n_neighbors para datasets pequenos
+    num_textos = len(textos)
+    n_neighbors = min(15, max(3, num_textos - 1)) if small_dataset else UMAP_N_NEIGHBORS
+    
+    reducer = umap.UMAP(n_components=3, n_neighbors=n_neighbors, min_dist=0.0, metric="cosine", random_state=42)
     emb_3d = reducer.fit_transform(embeddings)
     emb_3d = StandardScaler().fit_transform(emb_3d)
     
-    # 3. HDBSCAN
-    num_textos = len(textos)
-    min_size = max(10, int(num_textos * 0.02)) 
-    logging.info(f"HDBSCAN min_cluster_size: {min_size}")
+    # 3. HDBSCAN - parâmetros adaptativos
+    if small_dataset:
+        # Para Tavily (10-50 docs): clusters menores, mais agressivo
+        min_size = max(2, int(num_textos * 0.1))  # mínimo 2, 10% do dataset
+        min_samples = 1  # permite clusters mais esparsos
+    else:
+        # Para datasets grandes: comportamento padrão
+        min_size = max(10, int(num_textos * 0.02))
+        min_samples = None  # usa default do HDBSCAN
+    
+    logging.info(f"HDBSCAN: min_cluster_size={min_size}, min_samples={min_samples}")
     
-    clusterer = hdbscan.HDBSCAN(min_cluster_size=min_size)
+    clusterer = hdbscan.HDBSCAN(min_cluster_size=min_size, min_samples=min_samples)
     clusters = clusterer.fit_predict(emb_3d)
     
     # 4. DataFrame
@@ -591,8 +607,12 @@ async def search_web_api(
         if not textos:
             return {"error": "Resultados sem conteúdo válido.", "results_count": 0}
         
-        # Processar através do pipeline existente
-        df, embeddings = processar_pipeline(textos)
+        # Processar através do pipeline com parâmetros para datasets pequenos
+        df, embeddings = processar_pipeline(textos, small_dataset=True)
+        
+        # Adicionar URL de origem a cada ponto
+        df["source_url"] = [sources[i]["url"] if i < len(sources) else "" for i in range(len(df))]
+        df["source_title"] = [sources[i]["title"] if i < len(sources) else "" for i in range(len(df))]
         
         # Criar job e cachear
         job_id = str(uuid.uuid4())
@@ -617,8 +637,8 @@ async def search_web_api(
             },
             "metrics": metricas_globais,
             "cluster_analysis": analise_por_cluster_tfidf,
-            "plot_data": df[["x", "y", "z", "cluster", "full_text"]].to_dict("records"),
-            "sources": sources  # URLs originais
+            "plot_data": df[["x", "y", "z", "cluster", "full_text", "source_url", "source_title"]].to_dict("records"),
+            "sources": sources
         }
         
     except Exception as e: