Upload 8 files

Dockerfile

@@ -1,32 +1,23 @@
 FROM python:3.11-slim
 
-# Instalar Git
-RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y git curl && rm -rf /var/lib/apt/lists/*
 
-# Criar usuário não-root
 RUN useradd -m -u 1000 user
 USER user
 
-# Configurar environment
 ENV HOME=/home/user \
     PATH=/home/user/.local/bin:$PATH \
     PYTHONUNBUFFERED=1
 
 WORKDIR $HOME/app
 
-# Copiar requirements e instalar dependências
 COPY --chown=user requirements.txt .
 RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir -r requirements.txt
 
-# Copiar código da aplicação
 COPY --chown=user . .
-
-# Tornar entrypoint executável
 RUN chmod +x entrypoint.sh
 
-# Expor porta
 EXPOSE 7860
 
-# Comando de inicialização
 CMD ["./entrypoint.sh"]

README.md

@@ -1,5 +1,5 @@
 ---
-title: Para.AI RAG Cluster
+title: Para.AI RAG Cluster DEBUG
 emoji: ⚖️
 colorFrom: blue
 colorTo: purple
@@ -7,21 +7,12 @@ sdk: docker
 pinned: false
 ---
 
-# ⚖️ Para.AI RAG Cluster (LangChain + FAISS)
+# ⚖️ Para.AI RAG (DEBUG VERSION)
 
-Micro-cluster RAG para jurisprudências do TJPR.
+Versão com logs de depuração intensivos.
 
-## 🚀 Deploy
+## Endpoints adicionais
 
-1. Editar `config.yaml` (definir chunk_start/end)
-2. `git init && git add . && git commit -m "Initial"`
-3. `git push origin main`
-
-## 📡 Endpoints
-
-- `GET /` - Status
-- `GET /setup/status` - Progresso do setup
-- `POST /search/embedding` - Busca semântica
-- `GET /cluster/info` - Info do cluster
+- `GET /setup/logs` - Ver logs completos do setup
 
 ⚖️ **InJustiça não para o Paraná!** 🐝

app.py

@@ -31,14 +31,14 @@ def get_query_engine():
     global query_engine
     if query_engine is None:
         if not is_ready():
-            raise HTTPException(status_code=503, detail="RAG em construção. Tente em alguns minutos.")
+            raise HTTPException(status_code=503, detail="RAG em construção. Aguarde setup terminar.")
         logger.info("Carregando QueryEngine...")
         from query_engine import QueryEngine
         query_engine = QueryEngine()
         logger.info("✅ QueryEngine carregado!")
     return query_engine
 
-app = FastAPI(title="Para.AI RAG Cluster (LangChain)", version="1.0.0")
+app = FastAPI(title="Para.AI RAG Cluster", version="1.0.0")
 
 class EmbeddingSearchRequest(BaseModel):
     query: str
@@ -58,19 +58,25 @@ class IDSearchRequest(BaseModel):
 async def root():
     setup_status = get_setup_status()
     ready = is_ready()
-
     response = {"status": "online", "rag_ready": ready, "setup": setup_status, "backend": "LangChain + FAISS (CPU)"}
-
     if ready and query_engine:
         response["cluster_id"] = query_engine.config.get('cluster_id')
         response["chunk_range"] = [query_engine.config.get('chunk_start'), query_engine.config.get('chunk_end')]
-
     return response
 
 @app.get("/setup/status")
 async def setup_status():
     return get_setup_status()
 
+@app.get("/setup/logs")
+async def setup_logs():
+    try:
+        with open('/tmp/setup_debug.log', 'r') as f:
+            logs = f.read()
+        return {"logs": logs, "size": len(logs)}
+    except:
+        return {"logs": "Log file not available", "size": 0}
+
 @app.get("/health")
 async def health():
     return {"status": "ok", "timestamp": time.time()}
@@ -123,7 +129,7 @@ async def cluster_info():
 async def startup_event():
     app.state.start_time = time.time()
     logger.info("="*80)
-    logger.info("🚀 Para.AI RAG (LangChain + FAISS) ONLINE")
+    logger.info("🚀 Para.AI RAG ONLINE")
     logger.info("="*80)
 
 if __name__ == "__main__":

entrypoint.sh

@@ -1,22 +1,16 @@
 #!/bin/bash
 set -e
-
-echo "=================================="
+echo "=========================================="
 echo "🚀 Para.AI RAG Startup"
-echo "=================================="
-
+echo "=========================================="
 cd /home/user/app
-
 echo "1️⃣ Iniciando setup em background..."
 python3 -u setup.py > /tmp/setup_output.log 2>&1 &
-echo "✅ Setup iniciado"
-echo ""
-
+echo "✅ Setup PID: $!"
 sleep 2
-
 echo "2️⃣ Iniciando FastAPI..."
-echo "🎯 API online IMEDIATAMENTE"
-echo "🔧 RAG disponível quando setup terminar"
-echo "=================================="
-
+echo "🎯 API online"
+echo "📊 Status: /setup/status"
+echo "📋 Logs: /setup/logs"
+echo "=========================================="
 exec uvicorn app:app --host 0.0.0.0 --port 7860 --workers 1

filter_fields.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python3
-import json
-import argparse
-import yaml
+import json, argparse, yaml
 
 def filter_jsonl(input_file, output_file, fields_to_keep):
     with open(input_file, 'r', encoding='utf-8') as fin:
@@ -18,10 +16,8 @@ def main():
     parser.add_argument('--output', required=True)
     parser.add_argument('--config', default='config.yaml')
     args = parser.parse_args()
-
     with open(args.config) as f:
         config = yaml.safe_load(f)
-
     filter_jsonl(args.input, args.output, config['campos_filter'])
     print(f"✅ Filtrado: {args.output}")
 

query_engine.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python3
-import yaml
-import logging
+import yaml, logging
 from typing import List, Dict
-
 from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain_community.vectorstores import FAISS
 
@@ -12,79 +10,26 @@ logger = logging.getLogger(__name__)
 class QueryEngine:
     def __init__(self, config_path='config.yaml'):
         logger.info("Inicializando QueryEngine...")
-
         with open(config_path) as f:
             self.config = yaml.safe_load(f)
-
         model_name = self.config.get('embedding_model', 'sentence-transformers/all-MiniLM-L6-v2')
-        logger.info(f"Modelo: {model_name}")
-
-        self.embeddings = HuggingFaceEmbeddings(
-            model_name=model_name,
-            model_kwargs={'device': 'cpu'}
-        )
-
+        self.embeddings = HuggingFaceEmbeddings(model_name=model_name, model_kwargs={'device': 'cpu'})
         faiss_path = self.config.get('faiss_path', '/app/faiss_index')
-        logger.info(f"Carregando FAISS de: {faiss_path}")
-
-        self.vectorstore = FAISS.load_local(
-            faiss_path, 
-            self.embeddings,
-            allow_dangerous_deserialization=True
-        )
-
+        self.vectorstore = FAISS.load_local(faiss_path, self.embeddings, allow_dangerous_deserialization=True)
         logger.info("✅ QueryEngine pronto!")
 
     def search_by_embedding(self, query: str, top_k: int = 10, return_embeddings: bool = False) -> Dict:
         results = self.vectorstore.similarity_search_with_score(query, k=top_k)
-
-        formatted = []
-        for doc, score in results:
-            formatted.append({
-                'id': doc.metadata.get('id'),
-                'ementa': doc.page_content,
-                'score': float(score),
-                'metadata': doc.metadata
-            })
-
-        return {
-            'cluster_id': self.config.get('cluster_id'),
-            'query': query,
-            'total_results': len(formatted),
-            'results': formatted
-        }
+        formatted = [{'id': doc.metadata.get('id'), 'ementa': doc.page_content, 'score': float(score), 'metadata': doc.metadata} for doc, score in results]
+        return {'cluster_id': self.config.get('cluster_id'), 'query': query, 'total_results': len(formatted), 'results': formatted}
 
     def search_by_keywords(self, keywords: List[str], operator: str = 'AND', top_k: int = 20) -> Dict:
-        query = ' '.join(keywords)
-        return self.search_by_embedding(query, top_k)
+        return self.search_by_embedding(' '.join(keywords), top_k)
 
     def search_by_ids(self, ids: List[str], return_embeddings: bool = False) -> Dict:
         all_docs = self.vectorstore.similarity_search("", k=10000)
-
-        results = []
-        for doc in all_docs:
-            if doc.metadata.get('id') in ids:
-                results.append({
-                    'id': doc.metadata.get('id'),
-                    'ementa': doc.page_content,
-                    'metadata': doc.metadata
-                })
-                if len(results) >= len(ids):
-                    break
-
-        return {
-            'cluster_id': self.config.get('cluster_id'),
-            'total_results': len(results),
-            'results': results
-        }
+        results = [{'id': doc.metadata.get('id'), 'ementa': doc.page_content, 'metadata': doc.metadata} for doc in all_docs if doc.metadata.get('id') in ids][:len(ids)]
+        return {'cluster_id': self.config.get('cluster_id'), 'total_results': len(results), 'results': results}
 
     def get_cluster_info(self) -> Dict:
-        return {
-            'cluster_id': self.config.get('cluster_id'),
-            'chunk_range': [self.config.get('chunk_start'), self.config.get('chunk_end')],
-            'embedding_model': self.config.get('embedding_model'),
-            'embedding_dim': 384,
-            'vector_store': 'FAISS',
-            'backend': 'LangChain + CPU',
-            'status': 'ready'
-        }
+        return {'cluster_id': self.config.get('cluster_id'), 'chunk_range': [self.config.get('chunk_start'), self.config.get('chunk_end')], 'embedding_model': self.config.get('embedding_model'), 'embedding_dim': 384, 'vector_store': 'FAISS', 'backend': 'LangChain + CPU', 'status': 'ready'}

rag_builder.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-import os, sys, json, argparse, logging, traceback
+import os, sys, json, argparse, logging, traceback, time
 from pathlib import Path
 from typing import List, Dict
 from langchain.docstore.document import Document
@@ -11,31 +11,22 @@ logger = logging.getLogger(__name__)
 
 def load_jsonl(filepath: str) -> List[Dict]:
     records = []
-    try:
-        logger.info(f"📂 Abrindo: {filepath}")
-        if not os.path.exists(filepath):
-            raise FileNotFoundError(f"Arquivo não encontrado: {filepath}")
-        with open(filepath, 'r', encoding='utf-8') as f:
-            for i, line in enumerate(f, 1):
-                if line.strip():
-                    records.append(json.loads(line))
-                if i % 50000 == 0:
-                    logger.info(f"  {i:,} linhas...")
-        logger.info(f"✅ {len(records):,} registros")
-        return records
-    except Exception as e:
-        logger.error(f"❌ Erro: {e}")
-        raise
+    logger.info(f"📂 Carregando: {filepath}")
+    with open(filepath, 'r', encoding='utf-8') as f:
+        for i, line in enumerate(f, 1):
+            if line.strip():
+                records.append(json.loads(line))
+            if i % 50000 == 0:
+                logger.info(f"  {i:,} linhas...")
+    logger.info(f"✅ {len(records):,} registros")
+    return records
 
 def create_documents(records: List[Dict]) -> List[Document]:
     documents = []
     for i, record in enumerate(records, 1):
         ementa = record.get('ementa', '')
         if ementa:
-            documents.append(Document(
-                page_content=ementa,
-                metadata={'id': str(record.get('id', f'u{i}')), 'source': 'tjpr'}
-            ))
+            documents.append(Document(page_content=ementa, metadata={'id': str(record.get('id', f'u{i}')), 'source': 'tjpr'}))
         if i % 50000 == 0:
             logger.info(f"  {i:,}/{len(records):,}...")
     logger.info(f"✅ {len(documents):,} documentos")
@@ -43,48 +34,28 @@ def create_documents(records: List[Dict]) -> List[Document]:
 
 def build_vectorstore(input_file, output_dir='/app/faiss_index', model_name='sentence-transformers/all-MiniLM-L6-v2', batch_size=16):
     try:
-        import time
         logger.info("="*80)
-        logger.info("🚀 RAG Builder - LangChain + FAISS")
+        logger.info("🚀 RAG Builder")
         logger.info("="*80)
-
         logger.info("\nPASSO 1/5: Carregando JSONL")
         records = load_jsonl(input_file)
-        if not records:
-            raise ValueError("Nenhum registro!")
-
         logger.info("\nPASSO 2/5: Criando Documents")
         documents = create_documents(records)
-        if not documents:
-            raise ValueError("Nenhum documento!")
-
         logger.info(f"\nPASSO 3/5: Inicializando Embeddings ({model_name})")
-        embeddings = HuggingFaceEmbeddings(
-            model_name=model_name,
-            model_kwargs={'device': 'cpu'},
-            encode_kwargs={'batch_size': batch_size, 'show_progress_bar': True, 'normalize_embeddings': True}
-        )
+        embeddings = HuggingFaceEmbeddings(model_name=model_name, model_kwargs={'device': 'cpu'}, encode_kwargs={'batch_size': batch_size, 'show_progress_bar': True, 'normalize_embeddings': True})
         logger.info("✅ Embeddings OK")
-
         logger.info(f"\nPASSO 4/5: Construindo FAISS ({len(documents):,} docs)")
         start = time.time()
         vectorstore = FAISS.from_documents(documents, embeddings)
-        logger.info(f"✅ FAISS em {time.time()-start:.1f}s ({len(documents)/(time.time()-start):.0f} docs/s)")
-
+        logger.info(f"✅ FAISS em {time.time()-start:.1f}s")
         logger.info(f"\nPASSO 5/5: Salvando em {output_dir}")
         os.makedirs(output_dir, exist_ok=True)
         vectorstore.save_local(output_dir)
-        logger.info("✅ Salvo!")
-
-        logger.info("\n" + "="*80)
         logger.info("✅ BUILD COMPLETO!")
-        logger.info("="*80)
         return vectorstore
     except Exception as e:
-        logger.error("\n" + "="*80)
-        logger.error(f"❌ ERRO: {type(e).__name__}: {e}")
+        logger.error(f"\n❌ ERRO: {type(e).__name__}: {e}")
         logger.error(traceback.format_exc())
-        logger.error("="*80)
         raise
 
 def main():

requirements.txt

@@ -1,28 +1,28 @@
 # FastAPI
-fastapi
-uvicorn[standard]
-pydantic
+fastapi==0.109.0
+uvicorn[standard]==0.27.0
+pydantic==2.5.0
 
 # LangChain + Embeddings
-langchain
-langchain-community
+langchain==0.1.11
+langchain-community==0.0.24
 
-# Sentence Transformers (NECESSÁRIO!)
-sentence-transformers
-transformers
-torch
-tokenizers
-safetensors
+# Sentence Transformers
+sentence-transformers==2.5.1
+transformers==4.37.2
+torch==2.2.0
+tokenizers==0.15.2
+safetensors==0.4.2
 
 # FAISS
-faiss-cpu
+faiss-cpu==1.8.0
 
 # HuggingFace Hub
-huggingface-hub
+huggingface-hub==0.20.3
 
 # Utilities
-PyYAML
-GitPython
-pandas
-numpy
-tqdm
+PyYAML==6.0.1
+GitPython==3.1.41
+pandas==2.1.4
+numpy==1.26.3
+tqdm==4.66.1