Upload 9 files

.gitignore

@@ -1,29 +1,15 @@
-# Python
 __pycache__/
 *.py[cod]
-*$py.class
-*.so
 .Python
 env/
 venv/
 *.egg-info/
-.pytest_cache/
-
-# ChromaDB
-chromadb/
+faiss_index/
 *.sqlite3
-
-# Temporários
 /tmp/
 *.tar.gz
 *.jsonl
 repo_git_temp/
-
-# IDE
 .vscode/
 .idea/
-*.swp
-*.swo
-
-# Logs
 *.log

Dockerfile

@@ -30,4 +30,3 @@ EXPOSE 7860
 
 # Comando de inicialização
 CMD ["./entrypoint.sh"]
-

app.py

@@ -1,11 +1,5 @@
 #!/usr/bin/env python3
-"""
-Para.AI RAG Cluster - FastAPI Application
-Inicia IMEDIATAMENTE (antes do setup terminar) para evitar timeout HF
-"""
-
 from fastapi import FastAPI, HTTPException
-from fastapi.responses import JSONResponse
 from pydantic import BaseModel
 from typing import List, Optional
 import logging
@@ -16,73 +10,35 @@ from pathlib import Path
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-# ============================================================================
-# VERIFICAÇÃO DE STATUS DO SETUP
-# ============================================================================
-
 STATUS_FILE = Path('/tmp/setup_status.json')
-READY_FLAG = Path('/tmp/chromadb_ready')
+READY_FLAG = Path('/tmp/faiss_ready')
 
 def get_setup_status():
-    """Lê status do setup em background"""
     if not STATUS_FILE.exists():
-        return {
-            'status': 'initializing',
-            'message': 'Setup ainda não iniciado',
-            'progress': 0
-        }
-
+        return {'status': 'initializing', 'message': 'Setup não iniciado', 'progress': 0}
     try:
         with open(STATUS_FILE) as f:
             return json.load(f)
     except:
-        return {
-            'status': 'unknown',
-            'message': 'Erro ao ler status',
-            'progress': 0
-        }
+        return {'status': 'unknown', 'message': 'Erro ao ler status', 'progress': 0}
 
 def is_ready():
-    """Verifica se ChromaDB está pronto"""
     return READY_FLAG.exists()
 
-# ============================================================================
-# LAZY LOADING DO QUERY ENGINE
-# ============================================================================
-
 query_engine = None
 
 def get_query_engine():
-    """Carrega QueryEngine apenas quando ChromaDB estiver pronto"""
     global query_engine
-
     if query_engine is None:
         if not is_ready():
-            raise HTTPException(
-                status_code=503, 
-                detail="RAG ainda em construção. Tente novamente em alguns minutos."
-            )
-
+            raise HTTPException(status_code=503, detail="RAG em construção. Tente em alguns minutos.")
         logger.info("Carregando QueryEngine...")
         from query_engine import QueryEngine
         query_engine = QueryEngine()
         logger.info("✅ QueryEngine carregado!")
-
     return query_engine
 
-# ============================================================================
-# FASTAPI APP
-# ============================================================================
-
-app = FastAPI(
-    title="Para.AI RAG Cluster",
-    description="Micro-cluster RAG para jurisprudências do TJPR",
-    version="1.0.0"
-)
-
-# ============================================================================
-# MODELS (Pydantic)
-# ============================================================================
+app = FastAPI(title="Para.AI RAG Cluster (LangChain)", version="1.0.0")
 
 class EmbeddingSearchRequest(BaseModel):
     query: str
@@ -98,121 +54,76 @@ class IDSearchRequest(BaseModel):
     ids: List[str]
     return_embeddings: bool = False
 
-# ============================================================================
-# ENDPOINTS
-# ============================================================================
-
 @app.get("/")
 async def root():
-    """Health check - SEMPRE responde (mesmo durante setup)"""
     setup_status = get_setup_status()
     ready = is_ready()
 
-    response = {
-        "status": "online",
-        "rag_ready": ready,
-        "setup": setup_status
-    }
+    response = {"status": "online", "rag_ready": ready, "setup": setup_status, "backend": "LangChain + FAISS (CPU)"}
 
     if ready and query_engine:
-        response["cluster_id"] = query_engine.config['cluster_id']
-        response["chunk_range"] = [
-            query_engine.config['chunk_start'],
-            query_engine.config['chunk_end']
-        ]
-        response["endpoints"] = [
-            "/search/embedding",
-            "/search/keywords",
-            "/search/by_id",
-            "/cluster/info",
-            "/setup/status"
-        ]
+        response["cluster_id"] = query_engine.config.get('cluster_id')
+        response["chunk_range"] = [query_engine.config.get('chunk_start'), query_engine.config.get('chunk_end')]
 
     return response
 
 @app.get("/setup/status")
 async def setup_status():
-    """Retorna status detalhado do setup"""
     return get_setup_status()
 
 @app.get("/health")
 async def health():
-    """Health check simples para HF Spaces"""
     return {"status": "ok", "timestamp": time.time()}
 
 @app.post("/search/embedding")
 async def search_embedding(request: EmbeddingSearchRequest):
-    """Busca por similaridade semântica (embeddings)"""
-    engine = get_query_engine()  # Lança 503 se não estiver pronto
-
+    engine = get_query_engine()
     try:
         start = time.time()
-        results = engine.search_by_embedding(
-            query=request.query,
-            top_k=request.top_k,
-            return_embeddings=request.return_embeddings
-        )
+        results = engine.search_by_embedding(request.query, request.top_k, request.return_embeddings)
         results['query_time_ms'] = round((time.time() - start) * 1000, 2)
         return results
     except Exception as e:
-        logger.error(f"Erro em search_embedding: {e}")
+        logger.error(f"Erro: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
 @app.post("/search/keywords")
 async def search_keywords(request: KeywordSearchRequest):
-    """Busca por termos-chave (full-text search)"""
     engine = get_query_engine()
-
     try:
         start = time.time()
-        results = engine.search_by_keywords(
-            keywords=request.keywords,
-            operator=request.operator,
-            top_k=request.top_k
-        )
+        results = engine.search_by_keywords(request.keywords, request.operator, request.top_k)
         results['query_time_ms'] = round((time.time() - start) * 1000, 2)
         return results
     except Exception as e:
-        logger.error(f"Erro em search_keywords: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
 @app.post("/search/by_id")
 async def search_by_id(request: IDSearchRequest):
-    """Busca direta por ID(s)"""
     engine = get_query_engine()
-
     try:
         start = time.time()
-        results = engine.search_by_ids(
-            ids=request.ids,
-            return_embeddings=request.return_embeddings
-        )
+        results = engine.search_by_ids(request.ids, request.return_embeddings)
         results['query_time_ms'] = round((time.time() - start) * 1000, 2)
         return results
     except Exception as e:
-        logger.error(f"Erro em search_by_id: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
 @app.get("/cluster/info")
 async def cluster_info():
-    """Informações detalhadas do cluster"""
     engine = get_query_engine()
-
     try:
         info = engine.get_cluster_info()
         info['uptime_seconds'] = round(time.time() - app.state.start_time, 2)
         return info
     except Exception as e:
-        logger.error(f"Erro em cluster_info: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
 @app.on_event("startup")
 async def startup_event():
-    """Evento de startup - RÁPIDO (não aguarda setup)"""
     app.state.start_time = time.time()
     logger.info("="*80)
-    logger.info("🚀 Para.AI RAG Cluster FastAPI ONLINE")
-    logger.info("Setup em background: verificar /setup/status")
+    logger.info("🚀 Para.AI RAG (LangChain + FAISS) ONLINE")
     logger.info("="*80)
 
 if __name__ == "__main__":

entrypoint.sh

@@ -2,41 +2,27 @@
 set -e
 
 echo "=================================="
-echo "🚀 Para.AI RAG Cluster Startup"
+echo "🚀 Para.AI RAG (LangChain) Startup"
 echo "=================================="
 
-# Ir para diretório da aplicação
 cd /home/user/app
 
-# ESTRATÉGIA: Iniciar setup em background PRIMEIRO, depois FastAPI
-# Isso evita timeout de inicialização do HF Spaces
-
 echo ""
 echo "1️⃣ Iniciando setup em background..."
-echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-
-# Iniciar setup.py em background com output unbuffered (-u)
-# Redirecionar output para arquivo + tela
 python3 -u setup.py > /tmp/setup_output.log 2>&1 &
 SETUP_PID=$!
 
-echo "✅ Setup iniciado em background (PID: $SETUP_PID)"
-echo "📋 Logs em: /tmp/setup_output.log"
-echo "📊 Status em: /tmp/setup_status.json"
+echo "✅ Setup PID: $SETUP_PID"
+echo "📋 Logs: /tmp/setup_output.log"
 echo ""
 
-# Esperar 2 segundos para setup criar arquivo de status
 sleep 2
 
 echo "2️⃣ Iniciando FastAPI..."
-echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-echo "🎯 FastAPI estará online IMEDIATAMENTE"
-echo "🔧 RAG estará disponível quando setup terminar (~10-15 min)"
-echo "📡 Acompanhe em: /setup/status"
+echo "🎯 API online IMEDIATAMENTE"
+echo "🔧 RAG disponível quando setup terminar (~10-15min)"
+echo "📡 Acompanhe: /setup/status"
 echo ""
 echo "=================================="
-echo "🚀 Iniciando API REST..."
-echo "=================================="
 
-# Iniciar FastAPI (bloqueia aqui)
 exec uvicorn app:app --host 0.0.0.0 --port 7860 --workers 1

filter_fields.py

@@ -1,44 +1,29 @@
 #!/usr/bin/env python3
-"""
-Filtrar campos de JSONL mantendo apenas os especificados
-"""
 import json
-import yaml
-from pathlib import Path
 import argparse
-from tqdm import tqdm
-
-def filter_jsonl(input_path: str, output_path: str, keep_fields: list = None):
-    """Filtra campos de arquivo JSONL"""
-
-    # Carregar campos da config se não especificados
-    if keep_fields is None:
-        with open('config.yaml') as f:
-            config = yaml.safe_load(f)
-            keep_fields = config['campos_filter']
-
-    print(f"📥 Input: {input_path}")
-    print(f"📤 Output: {output_path}")
-    print(f"🔧 Mantendo campos: {keep_fields}")
-
-    # Contar linhas
-    with open(input_path) as f:
-        total = sum(1 for _ in f)
-
-    # Filtrar
-    with open(input_path) as fin, open(output_path, 'w') as fout:
-        for line in tqdm(fin, total=total, desc="Filtrando"):
-            record = json.loads(line)
-            filtered = {k: record[k] for k in keep_fields if k in record}
-            fout.write(json.dumps(filtered, ensure_ascii=False) + '\n')
+import yaml
 
-    print(f"✅ {total} registros filtrados!")
+def filter_jsonl(input_file, output_file, fields_to_keep):
+    with open(input_file, 'r', encoding='utf-8') as fin:
+        with open(output_file, 'w', encoding='utf-8') as fout:
+            for line in fin:
+                if line.strip():
+                    record = json.loads(line)
+                    filtered = {k: record.get(k) for k in fields_to_keep if k in record}
+                    fout.write(json.dumps(filtered, ensure_ascii=False) + '\n')
 
-if __name__ == "__main__":
+def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--input', required=True)
     parser.add_argument('--output', required=True)
-    parser.add_argument('--keep', nargs='+', default=None)
+    parser.add_argument('--config', default='config.yaml')
     args = parser.parse_args()
 
-    filter_jsonl(args.input, args.output, args.keep)
+    with open(args.config) as f:
+        config = yaml.safe_load(f)
+
+    filter_jsonl(args.input, args.output, config['campos_filter'])
+    print(f"✅ Filtrado: {args.output}")
+
+if __name__ == '__main__':
+    main()

query_engine.py

@@ -1,184 +1,91 @@
 #!/usr/bin/env python3
-"""
-Engine de busca para ChromaDB
-"""
 import yaml
-import chromadb
-from sentence_transformers import SentenceTransformer
-from typing import List, Dict, Optional
 import logging
+from typing import List, Dict
 
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+
+logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 class QueryEngine:
-    """Engine de busca com ChromaDB"""
+    def __init__(self, config_path='config.yaml'):
+        logger.info("Inicializando QueryEngine...")
 
-    def __init__(self, config_path: str = 'config.yaml'):
-        # Carregar config
         with open(config_path) as f:
             self.config = yaml.safe_load(f)
 
-        # Carregar modelo de embedding
-        logger.info(f"Carregando modelo {self.config['embedding_model']}...")
-        self.model = SentenceTransformer(self.config['embedding_model'])
-
-        # Conectar ao ChromaDB
-        logger.info(f"Conectando ao ChromaDB...")
-        self.client = chromadb.PersistentClient(path=self.config['chromadb_path'])
-        self.collection = self.client.get_collection(self.config['collection_name'])
-
-        logger.info(f"✅ QueryEngine pronto ({self.collection.count():,} registros)")
-
-    def search_by_embedding(
-        self, 
-        query: str, 
-        top_k: int = 10,
-        return_embeddings: bool = False
-    ) -> Dict:
-        """Busca por similaridade semântica"""
-
-        # Gerar embedding da query
-        query_embedding = self.model.encode(query).tolist()
-
-        # Buscar no ChromaDB
-        results = self.collection.query(
-            query_embeddings=[query_embedding],
-            n_results=top_k,
-            include=['documents', 'metadatas', 'distances', 'embeddings'] if return_embeddings 
-                     else ['documents', 'metadatas', 'distances']
-        )
+        model_name = self.config.get('embedding_model', 'all-MiniLM-L6-v2')
+        logger.info(f"Modelo: {model_name}")
 
-        # Formatar resposta
-        formatted_results = []
-        for i in range(len(results['ids'][0])):
-            result = {
-                'id': results['ids'][0][i],
-                'ementa': results['documents'][0][i],
-                'distance': results['distances'][0][i],
-                'score': 1.0 - results['distances'][0][i]  # Converter distância para score
-            }
+        self.embeddings = HuggingFaceEmbeddings(
+            model_name=model_name,
+            model_kwargs={'device': 'cpu'}
+        )
 
-            if return_embeddings and 'embeddings' in results:
-                result['embedding'] = results['embeddings'][0][i]
+        faiss_path = self.config.get('faiss_path', '/app/faiss_index')
+        logger.info(f"Carregando FAISS de: {faiss_path}")
 
-            formatted_results.append(result)
+        self.vectorstore = FAISS.load_local(
+            faiss_path, 
+            self.embeddings,
+            allow_dangerous_deserialization=True
+        )
 
-        return {
-            'cluster_id': self.config['cluster_id'],
-            'chunk_range': [self.config['chunk_start'], self.config['chunk_end']],
-            'results': formatted_results,
-            'total_found': len(formatted_results)
-        }
+        logger.info("✅ QueryEngine pronto!")
 
-    def search_by_keywords(
-        self,
-        keywords: List[str],
-        operator: str = 'AND',
-        top_k: int = 20
-    ) -> Dict:
-        """Busca por termos-chave (full-text search)"""
-
-        # Construir query string
-        if operator.upper() == 'AND':
-            query_str = ' '.join(keywords)
-        else:  # OR
-            query_str = '|'.join(keywords)
-
-        # Buscar usando where_document (full-text search do ChromaDB)
-        results = self.collection.query(
-            query_texts=[query_str],
-            n_results=top_k,
-            include=['documents', 'metadatas']
-        )
+    def search_by_embedding(self, query: str, top_k: int = 10, return_embeddings: bool = False) -> Dict:
+        results = self.vectorstore.similarity_search_with_score(query, k=top_k)
 
-        # Formatar resposta
-        formatted_results = []
-        for i in range(len(results['ids'][0])):
-            # Verificar quais keywords foram matchadas
-            doc = results['documents'][0][i].lower()
-            matched = [kw for kw in keywords if kw.lower() in doc]
-
-            formatted_results.append({
-                'id': results['ids'][0][i],
-                'ementa': results['documents'][0][i],
-                'matched_keywords': matched
+        formatted = []
+        for doc, score in results:
+            formatted.append({
+                'id': doc.metadata.get('id'),
+                'ementa': doc.page_content,
+                'score': float(score),
+                'metadata': doc.metadata
             })
 
         return {
-            'cluster_id': self.config['cluster_id'],
-            'results': formatted_results,
-            'total_found': len(formatted_results)
+            'cluster_id': self.config.get('cluster_id'),
+            'query': query,
+            'total_results': len(formatted),
+            'results': formatted
         }
 
-    def search_by_ids(
-        self,
-        ids: List[str],
-        return_embeddings: bool = False
-    ) -> Dict:
-        """Busca direta por ID(s)"""
-
-        # Buscar por IDs
-        try:
-            results = self.collection.get(
-                ids=ids,
-                include=['documents', 'metadatas', 'embeddings'] if return_embeddings 
-                         else ['documents', 'metadatas']
-            )
-        except Exception as e:
-            logger.error(f"Erro ao buscar IDs: {e}")
-            return {
-                'cluster_id': self.config['cluster_id'],
-                'results': [],
-                'not_found': ids,
-                'total_found': 0
-            }
-
-        # Formatar resposta
-        formatted_results = []
-        found_ids = set(results['ids'])
-
-        for i in range(len(results['ids'])):
-            result = {
-                'id': results['ids'][i],
-                'ementa': results['documents'][i]
-            }
-
-            if return_embeddings and 'embeddings' in results:
-                result['embedding'] = results['embeddings'][i]
-
-            formatted_results.append(result)
-
-        # IDs não encontrados
-        not_found = [id for id in ids if id not in found_ids]
+    def search_by_keywords(self, keywords: List[str], operator: str = 'AND', top_k: int = 20) -> Dict:
+        query = ' '.join(keywords)
+        return self.search_by_embedding(query, top_k)
+
+    def search_by_ids(self, ids: List[str], return_embeddings: bool = False) -> Dict:
+        # FAISS não tem busca direta por ID - implementação simplificada
+        all_docs = self.vectorstore.similarity_search("", k=10000)
+
+        results = []
+        for doc in all_docs:
+            if doc.metadata.get('id') in ids:
+                results.append({
+                    'id': doc.metadata.get('id'),
+                    'ementa': doc.page_content,
+                    'metadata': doc.metadata
+                })
+                if len(results) >= len(ids):
+                    break
 
         return {
-            'cluster_id': self.config['cluster_id'],
-            'results': formatted_results,
-            'not_found': not_found,
-            'total_found': len(formatted_results)
+            'cluster_id': self.config.get('cluster_id'),
+            'total_results': len(results),
+            'results': results
         }
 
     def get_cluster_info(self) -> Dict:
-        """Retorna informações do cluster"""
-        import os
-
-        # Calcular tamanho do ChromaDB
-        db_path = self.config['chromadb_path']
-        total_size = 0
-        for dirpath, dirnames, filenames in os.walk(db_path):
-            for f in filenames:
-                fp = os.path.join(dirpath, f)
-                total_size += os.path.getsize(fp)
-
-        db_size_mb = total_size / (1024 * 1024)
-
         return {
-            'cluster_id': self.config['cluster_id'],
-            'chunk_range': [self.config['chunk_start'], self.config['chunk_end']],
-            'total_records': self.collection.count(),
-            'embedding_model': self.config['embedding_model'],
-            'embedding_dim': self.config['embedding_dim'],
-            'campos_disponiveis': self.config['campos_filter'],
-            'db_size_mb': round(db_size_mb, 2),
+            'cluster_id': self.config.get('cluster_id'),
+            'chunk_range': [self.config.get('chunk_start'), self.config.get('chunk_end')],
+            'embedding_model': self.config.get('embedding_model'),
+            'embedding_dim': 384,
+            'vector_store': 'FAISS',
+            'backend': 'LangChain + CPU',
             'status': 'ready'
         }

rag_builder.py

@@ -1,105 +1,94 @@
 #!/usr/bin/env python3
 """
-Constrói ChromaDB com embeddings a partir de JSONL filtrado
+RAG Builder usando LangChain + HuggingFaceEmbeddings (CPU)
+Constrói FAISS vector store a partir de JSONL filtrado
 """
+
+import os
+import sys
 import json
-import yaml
-from pathlib import Path
 import argparse
-import chromadb
-from sentence_transformers import SentenceTransformer
-from tqdm import tqdm
+from pathlib import Path
+from typing import List, Dict
 import logging
 
+from langchain.docstore.document import Document
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-def build_chromadb(input_jsonl: str, config_path: str = 'config.yaml'):
-    """Constrói ChromaDB a partir de JSONL"""
+def load_jsonl(filepath: str) -> List[Dict]:
+    records = []
+    with open(filepath, 'r', encoding='utf-8') as f:
+        for line in f:
+            if line.strip():
+                records.append(json.loads(line))
+    return records
+
+def create_documents(records: List[Dict]) -> List[Document]:
+    documents = []
+    for record in records:
+        doc_id = record.get('id', 'unknown')
+        ementa = record.get('ementa', '')
+
+        if not ementa:
+            continue
+
+        doc = Document(
+            page_content=ementa,
+            metadata={'id': doc_id, 'source': 'tjpr'}
+        )
+        documents.append(doc)
 
-    # Carregar config
-    with open(config_path) as f:
-        config = yaml.safe_load(f)
+    return documents
 
+def build_vectorstore(input_file, output_dir='/app/faiss_index', model_name='all-MiniLM-L6-v2', batch_size=64):
     logger.info("="*80)
-    logger.info("🔧 CONSTRUINDO CHROMADB")
+    logger.info("🚀 RAG Builder - LangChain + FAISS (CPU)")
     logger.info("="*80)
-    logger.info(f"Cluster ID: {config['cluster_id']}")
-    logger.info(f"Chunks: {config['chunk_start']} - {config['chunk_end']}")
-    logger.info(f"Embedding Model: {config['embedding_model']}")
-
-    # Carregar modelo de embedding
-    logger.info("\n📥 Carregando modelo de embedding...")
-    model = SentenceTransformer(config['embedding_model'])
-    logger.info(f"✅ Modelo carregado (dim={config['embedding_dim']})")
-
-    # Inicializar ChromaDB
-    logger.info(f"\n💾 Inicializando ChromaDB em {config['chromadb_path']}...")
-    client = chromadb.PersistentClient(path=config['chromadb_path'])
-
-    # Criar/obter collection
-    try:
-        collection = client.get_collection(config['collection_name'])
-        logger.info(f"⚠️  Collection '{config['collection_name']}' já existe! Apagando...")
-        client.delete_collection(config['collection_name'])
-    except:
-        pass
-
-    collection = client.create_collection(
-        name=config['collection_name'],
-        metadata={
-            "cluster_id": config['cluster_id'],
-            "chunk_start": config['chunk_start'],
-            "chunk_end": config['chunk_end']
-        }
-    )
-    logger.info(f"✅ Collection criada")
 
-    # Carregar registros
-    logger.info(f"\n📖 Carregando registros de {input_jsonl}...")
-    records = []
-    with open(input_jsonl) as f:
-        for line in f:
-            records.append(json.loads(line))
-
-    total = len(records)
-    logger.info(f"✅ {total:,} registros carregados")
-
-    # Processar em batches
-    batch_size = config['embedding_batch_size']
-    logger.info(f"\n🚀 Gerando embeddings em batches de {batch_size}...")
+    logger.info(f"\n📂 Carregando {input_file}...")
+    records = load_jsonl(input_file)
+    logger.info(f"✅ {len(records):,} registros")
 
-    for i in tqdm(range(0, total, batch_size), desc="Embedding"):
-        batch = records[i:i+batch_size]
+    logger.info("\n📄 Criando Documents...")
+    documents = create_documents(records)
+    logger.info(f"✅ {len(documents):,} documentos")
 
-        # IDs
-        ids = [str(r['id']) for r in batch]
+    if not documents:
+        logger.error("❌ Nenhum documento válido!")
+        sys.exit(1)
 
-        # Documentos (usar ementa para embedding)
-        documents = [r.get('ementa', '') for r in batch]
+    logger.info(f"\n🤖 Criando embeddings com {model_name} (CPU)...")
+    embeddings = HuggingFaceEmbeddings(
+        model_name=model_name,
+        model_kwargs={'device': 'cpu'},
+        encode_kwargs={'batch_size': batch_size, 'show_progress_bar': True}
+    )
 
-        # Metadatas
-        metadatas = [{'id': r['id']} for r in batch]
+    logger.info("\n🔍 Construindo FAISS index...")
+    vectorstore = FAISS.from_documents(documents, embeddings)
 
-        # Gerar embeddings
-        embeddings = model.encode(documents, show_progress_bar=False).tolist()
+    logger.info(f"\n💾 Salvando em {output_dir}...")
+    os.makedirs(output_dir, exist_ok=True)
+    vectorstore.save_local(output_dir)
 
-        # Adicionar ao ChromaDB
-        collection.add(
-            ids=ids,
-            embeddings=embeddings,
-            documents=documents,
-            metadatas=metadatas
-        )
+    logger.info("\n✅ FAISS INDEX CRIADO!")
+    logger.info(f"📊 {len(documents):,} documentos indexados")
 
-    logger.info(f"\n✅ ChromaDB construído com sucesso!")
-    logger.info(f"📊 Total de registros: {collection.count():,}")
-    logger.info("="*80)
+    return vectorstore
 
-if __name__ == "__main__":
+def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument('--input', required=True, help='JSONL filtrado')
-    parser.add_argument('--config', default='config.yaml')
+    parser.add_argument('--input', required=True)
+    parser.add_argument('--output', default='/app/faiss_index')
+    parser.add_argument('--model', default='all-MiniLM-L6-v2')
+    parser.add_argument('--batch-size', type=int, default=64)
     args = parser.parse_args()
 
-    build_chromadb(args.input, args.config)
+    build_vectorstore(args.input, args.output, args.model, args.batch_size)
+
+if __name__ == '__main__':
+    main()

requirements.txt

@@ -3,12 +3,13 @@ fastapi==0.109.0
 uvicorn[standard]==0.27.0
 pydantic==2.5.0
 
-# RAG / Embeddings - VERSÕES COMPATÍVEIS TESTADAS
-torch==2.2.0
-safetensors==0.4.2
-transformers==4.37.2
+# LangChain + Embeddings (CPU-only)
+langchain==0.1.11
+langchain-community==0.0.24
 sentence-transformers==2.5.1
-chromadb==0.4.22
+
+# Vector Store
+faiss-cpu==1.8.0
 
 # Utilities
 PyYAML==6.0.1

setup.py

@@ -1,8 +1,4 @@
 #!/usr/bin/env python3
-"""
-Setup em background - Clona dados, constrói ChromaDB
-Executa enquanto FastAPI já está respondendo (evita timeout HF)
-"""
 import os
 import sys
 import yaml
@@ -12,64 +8,34 @@ import logging
 from pathlib import Path
 from datetime import datetime
 
-# Setup logging com flush imediato
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s',
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler('/tmp/setup.log')
-    ]
-)
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
 
-# Forçar flush imediato
-for handler in logger.handlers:
-    handler.flush = lambda: None
-
 STATUS_FILE = Path('/tmp/setup_status.json')
-READY_FLAG = Path('/tmp/chromadb_ready')
-
-def update_status(status: str, message: str, progress: int = 0):
-    """Atualiza arquivo de status para app.py ler"""
-    data = {
-        'status': status,
-        'message': message,
-        'progress': progress,
-        'timestamp': datetime.now().isoformat()
-    }
+READY_FLAG = Path('/tmp/faiss_ready')
+
+def update_status(status, message, progress=0):
+    data = {'status': status, 'message': message, 'progress': progress, 'timestamp': datetime.now().isoformat()}
     with open(STATUS_FILE, 'w') as f:
         json.dump(data, f)
     logger.info(f"[{progress}%] {status}: {message}")
     sys.stdout.flush()
 
-def run_command(cmd: str, description: str):
-    """Executa comando shell com logging"""
-    logger.info(f"Executando: {description}")
-    logger.info(f"Comando: {cmd}")
-
-    result = subprocess.run(
-        cmd, 
-        shell=True, 
-        capture_output=True, 
-        text=True
-    )
-
+def run_cmd(cmd, desc):
+    logger.info(f"Executando: {desc}")
+    result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
     if result.returncode != 0:
         logger.error(f"ERRO: {result.stderr}")
-        raise Exception(f"{description} falhou: {result.stderr}")
-
-    logger.info(f"✅ {description} completo")
+        raise Exception(f"{desc} falhou")
+    logger.info(f"✅ {desc}")
     return result.stdout
 
 def main():
-    """Setup completo em background"""
     try:
         logger.info("="*80)
-        logger.info("🚀 PARA.AI RAG CLUSTER - SETUP EM BACKGROUND")
+        logger.info("🚀 PARA.AI RAG (LangChain) - SETUP EM BACKGROUND")
         logger.info("="*80)
 
-        # Carregar configuração
         update_status('loading', 'Carregando configuração', 0)
         with open('config.yaml') as f:
             config = yaml.safe_load(f)
@@ -79,120 +45,62 @@ def main():
         chunk_end = config['chunk_end']
         github_repo = config['github_repo']
 
-        logger.info(f"Cluster: {cluster_id}")
-        logger.info(f"Chunks: {chunk_start} - {chunk_end}")
-        logger.info("")
-
-        # Verificar se ChromaDB já existe
         if READY_FLAG.exists():
-            logger.info("✅ ChromaDB já pronto! Pulando setup...")
-            update_status('ready', 'ChromaDB já existe', 100)
+            logger.info("✅ FAISS já pronto!")
+            update_status('ready', 'FAISS já existe', 100)
             return
 
-        # ETAPA 1: Git Sparse Checkout
-        update_status('cloning', 'Clonando chunks do GitHub (sparse checkout)', 10)
-
+        # CLONE
+        update_status('cloning', 'Clonando chunks (sparse checkout)', 10)
         os.makedirs('/tmp/repo', exist_ok=True)
         os.chdir('/tmp/repo')
 
-        # Clone inicial
-        run_command(
-            f"git clone --filter=blob:none --sparse {github_repo} .",
-            "Git clone inicial"
-        )
-
-        run_command(
-            "git sparse-checkout init --cone",
-            "Sparse checkout init"
-        )
-
-        # Gerar pattern de chunks
-        logger.info(f"Gerando pattern para chunks {chunk_start}-{chunk_end}...")
-        pattern_parts = []
-        for i in range(chunk_start, chunk_end + 1):
-            pattern_parts.append(f"chunks_dados/chunk_dados_{i:04d}.tar.gz")
-
-        # Set sparse checkout (em batches para evitar arg list too long)
-        batch_size = 50
-        for i in range(0, len(pattern_parts), batch_size):
-            batch = pattern_parts[i:i+batch_size]
-            pattern = ' '.join(batch)
-            run_command(
-                f"git sparse-checkout add {pattern}",
-                f"Sparse checkout batch {i//batch_size + 1}"
-            )
-
-        # Contar chunks clonados
-        result = run_command(
-            "find chunks_dados -name '*.tar.gz' 2>/dev/null | wc -l",
-            "Contar chunks"
-        )
-        chunks_count = int(result.strip())
+        run_cmd(f"git clone --filter=blob:none --sparse {github_repo} .", "Git clone")
+        run_cmd("git sparse-checkout init --cone", "Sparse checkout init")
+
+        patterns = [f"chunks_dados/chunk_dados_{i:04d}.tar.gz" for i in range(chunk_start, chunk_end + 1)]
+        for i in range(0, len(patterns), 50):
+            batch = ' '.join(patterns[i:i+50])
+            run_cmd(f"git sparse-checkout add {batch}", f"Batch {i//50 + 1}")
+
+        chunks_count = int(run_cmd("find chunks_dados -name '*.tar.gz' 2>/dev/null | wc -l", "Contar chunks").strip())
         logger.info(f"✅ {chunks_count} chunks clonados")
 
-        # ETAPA 2: Descompactar
+        # EXTRACT
         update_status('extracting', f'Descompactando {chunks_count} chunks', 30)
-
         os.makedirs('/tmp/extracted', exist_ok=True)
+        run_cmd("find chunks_dados -name '*.tar.gz' -exec tar -xzf {} -C /tmp/extracted \; 2>/dev/null || true", "Descompactar")
 
-        run_command(
-            "find chunks_dados -name '*.tar.gz' -exec tar -xzf {} -C /tmp/extracted \; 2>/dev/null || true",
-            "Descompactar chunks"
-        )
-
-        # ETAPA 3: Concatenar JSONL
-        update_status('concatenating', 'Concatenando jurisprudencias.jsonl', 50)
-
-        run_command(
-            "find /tmp/extracted -name 'jurisprudencias.jsonl' -exec cat {} \; > /tmp/all_records.jsonl 2>/dev/null || true",
-            "Concatenar JSONL"
-        )
+        # CONCAT
+        update_status('concatenating', 'Concatenando JSONL', 50)
+        run_cmd("find /tmp/extracted -name 'jurisprudencias.jsonl' -exec cat {} \; > /tmp/all_records.jsonl 2>/dev/null || true", "Concatenar")
 
-        # Contar registros
-        result = run_command(
-            "wc -l < /tmp/all_records.jsonl 2>/dev/null || echo '0'",
-            "Contar registros"
-        )
-        total_records = int(result.strip())
-        logger.info(f"✅ {total_records:,} registros concatenados")
+        total_records = int(run_cmd("wc -l < /tmp/all_records.jsonl 2>/dev/null || echo '0'", "Contar registros").strip())
+        logger.info(f"✅ {total_records:,} registros")
 
-        # ETAPA 4: Filtrar campos
+        # FILTER
         update_status('filtering', 'Filtrando campos (id + ementa)', 60)
-
         os.chdir('/home/user/app')
-        run_command(
-            "python3 filter_fields.py --input /tmp/all_records.jsonl --output /tmp/filtered.jsonl",
-            "Filtrar campos"
-        )
-
-        # ETAPA 5: Build ChromaDB
-        update_status('building', 'Construindo ChromaDB com embeddings (pode demorar)', 70)
-
-        run_command(
-            "python3 rag_builder.py --input /tmp/filtered.jsonl",
-            "Build ChromaDB"
-        )
+        run_cmd("python3 filter_fields.py --input /tmp/all_records.jsonl --output /tmp/filtered.jsonl", "Filtrar")
 
-        # ETAPA 6: Limpar temporários
-        update_status('cleaning', 'Limpando arquivos temporários', 95)
+        # BUILD FAISS
+        update_status('building', 'Construindo FAISS index (pode demorar)', 70)
+        run_cmd("python3 rag_builder.py --input /tmp/filtered.jsonl", "Build FAISS")
 
-        run_command(
-            "rm -rf /tmp/repo /tmp/extracted /tmp/all_records.jsonl /tmp/filtered.jsonl",
-            "Limpar temporários"
-        )
+        # CLEANUP
+        update_status('cleaning', 'Limpando temporários', 95)
+        run_cmd("rm -rf /tmp/repo /tmp/extracted /tmp/all_records.jsonl /tmp/filtered.jsonl", "Limpar")
 
-        # ETAPA 7: Marcar como pronto
-        update_status('ready', f'ChromaDB pronto com {total_records:,} registros!', 100)
+        # DONE
+        update_status('ready', f'FAISS pronto com {total_records:,} registros!', 100)
         READY_FLAG.touch()
 
         logger.info("="*80)
-        logger.info("✅ SETUP COMPLETO - RAG PRONTO PARA USO!")
+        logger.info("✅ SETUP COMPLETO!")
         logger.info("="*80)
 
     except Exception as e:
-        logger.error("="*80)
-        logger.error(f"❌ ERRO NO SETUP: {e}")
-        logger.error("="*80)
+        logger.error(f"❌ ERRO: {e}")
         update_status('error', str(e), 0)
         sys.exit(1)