Upload 8 files

Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.11-slim
 
-RUN apt-get update && apt-get install -y git curl && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
 
 RUN useradd -m -u 1000 user
 USER user

README.md

@@ -1,5 +1,5 @@
 ---
-title: Para.AI RAG Cluster DEBUG
+title: Para.AI RAG Cluster
 emoji: ⚖️
 colorFrom: blue
 colorTo: purple
@@ -7,12 +7,15 @@ sdk: docker
 pinned: false
 ---
 
-# ⚖️ Para.AI RAG (DEBUG VERSION)
+# ⚖️ Para.AI RAG v2.0 - Otimizado
 
-Versão com logs de depuração intensivos.
+Versão otimizada com download direto de chunks (sem git clone).
 
-## Endpoints adicionais
+## Melhorias
 
-- `GET /setup/logs` - Ver logs completos do setup
+- ✅ wget/curl direto (não clona repo inteiro)
+- ✅ Processa chunks um por vez (economiza espaço)
+- ✅ Filtro de campos corrigido (id minúsculo)
+- ✅ Cleanup automático de temporários
 
 ⚖️ **InJustiça não para o Paraná!** 🐝

app.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
-from typing import List, Optional
+from typing import List
 import logging
 import time
 import json
@@ -38,12 +38,11 @@ def get_query_engine():
         logger.info("✅ QueryEngine carregado!")
     return query_engine
 
-app = FastAPI(title="Para.AI RAG Cluster", version="1.0.0")
+app = FastAPI(title="Para.AI RAG Cluster", version="2.0.0")
 
 class EmbeddingSearchRequest(BaseModel):
     query: str
     top_k: int = 10
-    return_embeddings: bool = False
 
 class KeywordSearchRequest(BaseModel):
     keywords: List[str]
@@ -52,13 +51,18 @@ class KeywordSearchRequest(BaseModel):
 
 class IDSearchRequest(BaseModel):
     ids: List[str]
-    return_embeddings: bool = False
 
 @app.get("/")
 async def root():
     setup_status = get_setup_status()
     ready = is_ready()
-    response = {"status": "online", "rag_ready": ready, "setup": setup_status, "backend": "LangChain + FAISS (CPU)"}
+    response = {
+        "status": "online",
+        "rag_ready": ready,
+        "setup": setup_status,
+        "backend": "LangChain + FAISS (CPU)",
+        "version": "2.0.0 - Otimizado"
+    }
     if ready and query_engine:
         response["cluster_id"] = query_engine.config.get('cluster_id')
         response["chunk_range"] = [query_engine.config.get('chunk_start'), query_engine.config.get('chunk_end')]
@@ -86,7 +90,7 @@ async def search_embedding(request: EmbeddingSearchRequest):
     engine = get_query_engine()
     try:
         start = time.time()
-        results = engine.search_by_embedding(request.query, request.top_k, request.return_embeddings)
+        results = engine.search_by_embedding(request.query, request.top_k)
         results['query_time_ms'] = round((time.time() - start) * 1000, 2)
         return results
     except Exception as e:
@@ -109,7 +113,7 @@ async def search_by_id(request: IDSearchRequest):
     engine = get_query_engine()
     try:
         start = time.time()
-        results = engine.search_by_ids(request.ids, request.return_embeddings)
+        results = engine.search_by_ids(request.ids)
         results['query_time_ms'] = round((time.time() - start) * 1000, 2)
         return results
     except Exception as e:
@@ -129,7 +133,7 @@ async def cluster_info():
 async def startup_event():
     app.state.start_time = time.time()
     logger.info("="*80)
-    logger.info("🚀 Para.AI RAG ONLINE")
+    logger.info("🚀 Para.AI RAG v2.0 ONLINE")
     logger.info("="*80)
 
 if __name__ == "__main__":

entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 echo "=========================================="
-echo "🚀 Para.AI RAG Startup"
+echo "🚀 Para.AI RAG v2.0 Startup"
 echo "=========================================="
 cd /home/user/app
 echo "1️⃣ Iniciando setup em background..."
@@ -9,8 +9,8 @@ python3 -u setup.py > /tmp/setup_output.log 2>&1 &
 echo "✅ Setup PID: $!"
 sleep 2
 echo "2️⃣ Iniciando FastAPI..."
-echo "🎯 API online"
-echo "📊 Status: /setup/status"
-echo "📋 Logs: /setup/logs"
+echo "🎯 API online em http://0.0.0.0:7860"
+echo "📊 Status: GET /setup/status"
+echo "📋 Logs: GET /setup/logs"
 echo "=========================================="
 exec uvicorn app:app --host 0.0.0.0 --port 7860 --workers 1

query_engine.py

@@ -12,24 +12,64 @@ class QueryEngine:
         logger.info("Inicializando QueryEngine...")
         with open(config_path) as f:
             self.config = yaml.safe_load(f)
-        model_name = self.config.get('embedding_model', 'sentence-transformers/all-MiniLM-L6-v2')
-        self.embeddings = HuggingFaceEmbeddings(model_name=model_name, model_kwargs={'device': 'cpu'})
+        model_name = self.config.get('embedding_model')
+        self.embeddings = HuggingFaceEmbeddings(
+            model_name=model_name,
+            model_kwargs={'device': 'cpu'}
+        )
         faiss_path = self.config.get('faiss_path', '/app/faiss_index')
-        self.vectorstore = FAISS.load_local(faiss_path, self.embeddings, allow_dangerous_deserialization=True)
+        self.vectorstore = FAISS.load_local(
+            faiss_path,
+            self.embeddings,
+            allow_dangerous_deserialization=True
+        )
         logger.info("✅ QueryEngine pronto!")
 
-    def search_by_embedding(self, query: str, top_k: int = 10, return_embeddings: bool = False) -> Dict:
+    def search_by_embedding(self, query: str, top_k: int = 10) -> Dict:
         results = self.vectorstore.similarity_search_with_score(query, k=top_k)
-        formatted = [{'id': doc.metadata.get('id'), 'ementa': doc.page_content, 'score': float(score), 'metadata': doc.metadata} for doc, score in results]
-        return {'cluster_id': self.config.get('cluster_id'), 'query': query, 'total_results': len(formatted), 'results': formatted}
+        formatted = [
+            {
+                'id': doc.metadata.get('id'),
+                'ementa': doc.page_content,
+                'score': float(score),
+                'metadata': doc.metadata
+            }
+            for doc, score in results
+        ]
+        return {
+            'cluster_id': self.config.get('cluster_id'),
+            'query': query,
+            'total_results': len(formatted),
+            'results': formatted
+        }
 
     def search_by_keywords(self, keywords: List[str], operator: str = 'AND', top_k: int = 20) -> Dict:
         return self.search_by_embedding(' '.join(keywords), top_k)
 
-    def search_by_ids(self, ids: List[str], return_embeddings: bool = False) -> Dict:
+    def search_by_ids(self, ids: List[str]) -> Dict:
         all_docs = self.vectorstore.similarity_search("", k=10000)
-        results = [{'id': doc.metadata.get('id'), 'ementa': doc.page_content, 'metadata': doc.metadata} for doc in all_docs if doc.metadata.get('id') in ids][:len(ids)]
-        return {'cluster_id': self.config.get('cluster_id'), 'total_results': len(results), 'results': results}
+        results = [
+            {
+                'id': doc.metadata.get('id'),
+                'ementa': doc.page_content,
+                'metadata': doc.metadata
+            }
+            for doc in all_docs
+            if doc.metadata.get('id') in ids
+        ][:len(ids)]
+        return {
+            'cluster_id': self.config.get('cluster_id'),
+            'total_results': len(results),
+            'results': results
+        }
 
     def get_cluster_info(self) -> Dict:
-        return {'cluster_id': self.config.get('cluster_id'), 'chunk_range': [self.config.get('chunk_start'), self.config.get('chunk_end')], 'embedding_model': self.config.get('embedding_model'), 'embedding_dim': 384, 'vector_store': 'FAISS', 'backend': 'LangChain + CPU', 'status': 'ready'}
+        return {
+            'cluster_id': self.config.get('cluster_id'),
+            'chunk_range': [self.config.get('chunk_start'), self.config.get('chunk_end')],
+            'embedding_model': self.config.get('embedding_model'),
+            'embedding_dim': 384,
+            'vector_store': 'FAISS',
+            'backend': 'LangChain + CPU',
+            'status': 'ready'
+        }

rag_builder.py

@@ -1,12 +1,16 @@
 #!/usr/bin/env python3
-import os, sys, json, argparse, logging, traceback, time
+import os, sys, json, argparse, logging, time
 from pathlib import Path
 from typing import List, Dict
 from langchain.docstore.document import Document
 from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain_community.vectorstores import FAISS
 
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    stream=sys.stdout
+)
 logger = logging.getLogger(__name__)
 
 def load_jsonl(filepath: str) -> List[Dict]:
@@ -15,48 +19,98 @@ def load_jsonl(filepath: str) -> List[Dict]:
     with open(filepath, 'r', encoding='utf-8') as f:
         for i, line in enumerate(f, 1):
             if line.strip():
-                records.append(json.loads(line))
-            if i % 50000 == 0:
+                try:
+                    records.append(json.loads(line))
+                except json.JSONDecodeError as e:
+                    logger.warning(f"Linha {i} inválida: {e}")
+            if i % 10000 == 0:
                 logger.info(f"  {i:,} linhas...")
-    logger.info(f"✅ {len(records):,} registros")
+    logger.info(f"✅ {len(records):,} registros carregados")
     return records
 
 def create_documents(records: List[Dict]) -> List[Document]:
     documents = []
+    logger.info("📝 Criando documentos...")
+
     for i, record in enumerate(records, 1):
         ementa = record.get('ementa', '')
-        if ementa:
-            documents.append(Document(page_content=ementa, metadata={'id': str(record.get('id', f'u{i}')), 'source': 'tjpr'}))
-        if i % 50000 == 0:
+        doc_id = record.get('id', f'unknown_{i}')
+
+        if ementa and ementa.strip():
+            documents.append(
+                Document(
+                    page_content=ementa,
+                    metadata={
+                        'id': str(doc_id),
+                        'source': 'tjpr'
+                    }
+                )
+            )
+
+        if i % 10000 == 0:
             logger.info(f"  {i:,}/{len(records):,}...")
-    logger.info(f"✅ {len(documents):,} documentos")
+
+    logger.info(f"✅ {len(documents):,} documentos criados")
     return documents
 
-def build_vectorstore(input_file, output_dir='/app/faiss_index', model_name='sentence-transformers/all-MiniLM-L6-v2', batch_size=16):
+def build_vectorstore(
+    input_file,
+    output_dir='/app/faiss_index',
+    model_name='sentence-transformers/all-MiniLM-L6-v2',
+    batch_size=16
+):
     try:
         logger.info("="*80)
-        logger.info("🚀 RAG Builder")
+        logger.info("🚀 RAG Builder v2.0")
         logger.info("="*80)
-        logger.info("\nPASSO 1/5: Carregando JSONL")
+
+        logger.info("\nPASSO 1/4: Carregando JSONL")
         records = load_jsonl(input_file)
-        logger.info("\nPASSO 2/5: Criando Documents")
+
+        if not records:
+            raise Exception("Nenhum registro carregado!")
+
+        logger.info("\nPASSO 2/4: Criando Documents")
         documents = create_documents(records)
-        logger.info(f"\nPASSO 3/5: Inicializando Embeddings ({model_name})")
-        embeddings = HuggingFaceEmbeddings(model_name=model_name, model_kwargs={'device': 'cpu'}, encode_kwargs={'batch_size': batch_size, 'show_progress_bar': True, 'normalize_embeddings': True})
-        logger.info("✅ Embeddings OK")
-        logger.info(f"\nPASSO 4/5: Construindo FAISS ({len(documents):,} docs)")
+
+        if not documents:
+            raise Exception("Nenhum documento criado!")
+
+        logger.info(f"\nPASSO 3/4: Inicializando Embeddings ({model_name})")
+        embeddings = HuggingFaceEmbeddings(
+            model_name=model_name,
+            model_kwargs={'device': 'cpu'},
+            encode_kwargs={
+                'batch_size': batch_size,
+                'show_progress_bar': True,
+                'normalize_embeddings': True
+            }
+        )
+        logger.info("✅ Embeddings inicializados")
+
+        logger.info(f"\nPASSO 4/4: Construindo FAISS ({len(documents):,} docs)")
         start = time.time()
+
         vectorstore = FAISS.from_documents(documents, embeddings)
-        logger.info(f"✅ FAISS em {time.time()-start:.1f}s")
-        logger.info(f"\nPASSO 5/5: Salvando em {output_dir}")
+
+        elapsed = time.time() - start
+        logger.info(f"✅ FAISS construído em {elapsed:.1f}s")
+
+        logger.info(f"\nSalvando em {output_dir}")
         os.makedirs(output_dir, exist_ok=True)
         vectorstore.save_local(output_dir)
+
+        logger.info("="*80)
         logger.info("✅ BUILD COMPLETO!")
+        logger.info("="*80)
+
         return vectorstore
+
     except Exception as e:
         logger.error(f"\n❌ ERRO: {type(e).__name__}: {e}")
+        import traceback
         logger.error(traceback.format_exc())
-        raise
+        sys.exit(1)
 
 def main():
     parser = argparse.ArgumentParser()
@@ -65,6 +119,7 @@ def main():
     parser.add_argument('--model', default='sentence-transformers/all-MiniLM-L6-v2')
     parser.add_argument('--batch-size', type=int, default=16)
     args = parser.parse_args()
+
     build_vectorstore(args.input, args.output, args.model, args.batch_size)
 
 if __name__ == '__main__':

requirements.txt

@@ -1,28 +1,27 @@
 # FastAPI
-fastapi
-uvicorn[standard]
-pydantic
+fastapi==0.109.0
+uvicorn[standard]==0.27.0
+pydantic==2.5.0
 
 # LangChain + Embeddings
-langchain
-langchain-community
+langchain==0.1.11
+langchain-community==0.0.24
 
 # Sentence Transformers
-sentence-transformers
-transformers
-torch
-tokenizers
-safetensors
+sentence-transformers==2.5.1
+transformers==4.37.2
+torch==2.2.0
+tokenizers==0.15.2
+safetensors==0.4.2
 
 # FAISS
-faiss-cpu
+faiss-cpu==1.8.0
 
 # HuggingFace Hub
-huggingface-hub
+huggingface-hub==0.20.3
 
 # Utilities
-PyYAML
-GitPython
-pandas
-numpy
-tqdm
+PyYAML==6.0.1
+pandas==2.1.4
+numpy==1.26.3
+tqdm==4.66.1

setup.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python3
-import os, sys, yaml, json, subprocess, logging, traceback, time
+import os, sys, yaml, json, subprocess, logging, traceback, time, tarfile, io
 from pathlib import Path
 from datetime import datetime
 
-# Configurar logging VERBOSE
 logging.basicConfig(
-    level=logging.DEBUG,
+    level=logging.INFO,
     format='%(asctime)s [%(levelname)s] %(message)s',
     handlers=[
         logging.StreamHandler(sys.stdout),
@@ -26,364 +25,240 @@ def update_status(status, message, progress=0):
     }
     with open(STATUS_FILE, 'w') as f:
         json.dump(data, f)
-    logger.info(f"STATUS UPDATE [{progress}%]: {status} - {message}")
+    logger.info(f"STATUS [{progress}%]: {status} - {message}")
     sys.stdout.flush()
 
-def run_cmd(cmd, desc, capture=True, check=True):
+def run_cmd(cmd, desc, check=True, timeout=300):
     logger.info("="*80)
-    logger.info(f"🔧 EXECUTANDO: {desc}")
+    logger.info(f"🔧 {desc}")
     logger.info(f"📝 Comando: {cmd}")
-    logger.info(f"📂 PWD: {os.getcwd()}")
-    logger.info(f"👤 USER: {os.getenv('USER', 'unknown')}")
-    logger.info(f"🏠 HOME: {os.getenv('HOME', 'unknown')}")
     logger.info("-"*80)
 
     try:
         start = time.time()
-
-        if capture:
-            result = subprocess.run(
-                cmd,
-                shell=True,
-                capture_output=True,
-                text=True,
-                timeout=600
-            )
-        else:
-            result = subprocess.run(
-                cmd,
-                shell=True,
-                timeout=600
-            )
-            result.stdout = ""
-            result.stderr = ""
-
+        result = subprocess.run(
+            cmd,
+            shell=True,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            check=check
+        )
         elapsed = time.time() - start
 
-        logger.info(f"⏱️  Tempo: {elapsed:.2f}s")
-        logger.info(f"🔢 Exit code: {result.returncode}")
+        logger.info(f"⏱️  {elapsed:.2f}s | Exit: {result.returncode}")
 
         if result.stdout:
-            logger.info(f"📤 STDOUT ({len(result.stdout)} chars):")
-            for line in result.stdout.split('\n')[:50]:  # Primeiras 50 linhas
-                logger.info(f"  | {line}")
-            if len(result.stdout.split('\n')) > 50:
-                logger.info(f"  | ... ({len(result.stdout.split(chr(10))) - 50} linhas omitidas)")
-
+            logger.info(f"STDOUT: {result.stdout[:500]}")
         if result.stderr:
-            logger.warning(f"⚠️  STDERR ({len(result.stderr)} chars):")
-            for line in result.stderr.split('\n')[:50]:
-                logger.warning(f"  | {line}")
-            if len(result.stderr.split('\n')) > 50:
-                logger.warning(f"  | ... ({len(result.stderr.split(chr(10))) - 50} linhas omitidas)")
-
-        if check and result.returncode != 0:
-            logger.error("="*80)
-            logger.error(f"❌ COMANDO FALHOU: {desc}")
-            logger.error(f"Exit code: {result.returncode}")
-            logger.error("="*80)
-            raise Exception(f"{desc} falhou com exit code {result.returncode}")
+            logger.warning(f"STDERR: {result.stderr[:500]}")
 
         logger.info(f"✅ {desc} - OK")
-        logger.info("="*80)
-
-        return result.stdout if capture else ""
+        return result.stdout
 
     except subprocess.TimeoutExpired:
-        logger.error(f"⏰ TIMEOUT após 600s: {desc}")
-        raise Exception(f"{desc} - timeout")
+        logger.error(f"⏰ TIMEOUT após {timeout}s: {desc}")
+        raise
+    except subprocess.CalledProcessError as e:
+        logger.error(f"❌ FALHOU: {desc}")
+        logger.error(f"Exit code: {e.returncode}")
+        logger.error(f"STDERR: {e.stderr[:500]}")
+        raise
     except Exception as e:
         logger.error(f"💥 EXCEÇÃO: {type(e).__name__}: {e}")
-        logger.error(traceback.format_exc())
         raise
 
-def check_environment():
-    logger.info("\n" + "="*80)
-    logger.info("🔍 VERIFICANDO AMBIENTE")
-    logger.info("="*80)
+def filter_jsonl_record(record, fields_to_keep):
+    """Filtra campos de um registro JSONL"""
+    return {k: record.get(k) for k in fields_to_keep if k in record}
 
-    checks = [
-        ("pwd", "Diretório atual"),
-        ("whoami", "Usuário"),
-        ("id", "UID/GID"),
-        ("git --version", "Git version"),
-        ("python3 --version", "Python version"),
-        ("pip list | grep -E '(langchain|torch|transformers)'", "Pacotes principais"),
-        ("df -h /tmp", "Espaço em /tmp"),
-        ("free -h", "Memória disponível"),
-        ("ls -la /home/user/app", "Arquivos app"),
-        ("cat /etc/resolv.conf", "DNS config"),
-        ("ping -c 2 github.com || echo 'ping falhou'", "Conectividade GitHub"),
-    ]
+def process_tar_gz(tar_path, output_jsonl, fields_to_keep):
+    """Extrai TAR.GZ, filtra campos do JSONL, e concatena"""
+    logger.info(f"📦 Processando: {tar_path.name}")
 
-    for cmd, desc in checks:
-        try:
-            logger.info(f"\n🔎 {desc}:")
-            output = run_cmd(cmd, desc, capture=True, check=False)
-        except Exception as e:
-            logger.warning(f"  ⚠️  Falhou: {e}")
+    try:
+        with tarfile.open(tar_path, 'r:gz') as tar:
+            members = tar.getmembers()
+            logger.info(f"  Arquivos no TAR.GZ: {len(members)}")
+
+            for member in members:
+                if member.name.endswith('jurisprudencias.jsonl') and member.isfile():
+                    logger.info(f"  ✅ Encontrado: {member.name}")
+
+                    # Extrai JSONL para memória
+                    file_obj = tar.extractfile(member)
+                    content = file_obj.read().decode('utf-8')
+
+                    # Processa linha por linha
+                    lines = content.strip().split('\n')
+                    logger.info(f"  📋 Linhas: {len(lines)}")
+
+                    processed = 0
+                    with open(output_jsonl, 'a', encoding='utf-8') as out:
+                        for line in lines:
+                            if line.strip():
+                                try:
+                                    record = json.loads(line)
+                                    filtered = filter_jsonl_record(record, fields_to_keep)
+                                    out.write(json.dumps(filtered, ensure_ascii=False) + '\n')
+                                    processed += 1
+                                except json.JSONDecodeError:
+                                    continue
+
+                    logger.info(f"  ✅ Processados: {processed} registros")
+                    return processed
+
+        logger.warning(f"  ⚠️  Nenhum jurisprudencias.jsonl encontrado")
+        return 0
+
+    except Exception as e:
+        logger.error(f"  ❌ Erro ao processar {tar_path.name}: {e}")
+        raise
 
 def main():
     try:
         logger.info("\n" + "="*80)
-        logger.info("🚀 PARA.AI RAG SETUP - VERSÃO DEBUG")
+        logger.info("🚀 PARA.AI RAG SETUP - VERSÃO OTIMIZADA")
         logger.info("="*80)
         logger.info(f"⏰ Início: {datetime.now()}")
-        logger.info(f"🐍 Python: {sys.version}")
-        logger.info(f"📂 CWD: {os.getcwd()}")
-        logger.info("="*80)
 
-        # VERIFICAÇÃO DE AMBIENTE
-        check_environment()
+        # VERIFICAR SE JÁ PRONTO
+        if READY_FLAG.exists():
+            logger.info("✅ FAISS já existe")
+            update_status('ready', 'FAISS já pronto', 100)
+            return
 
         # CARREGAR CONFIG
-        logger.info("\n" + "="*80)
-        logger.info("📝 PASSO 0: Carregando configuração")
-        logger.info("="*80)
+        logger.info("\n📝 PASSO 0: Carregando configuração")
         update_status('loading', 'Carregando configuração', 0)
 
-        config_path = 'config.yaml'
-        logger.info(f"Config file: {config_path}")
-        logger.info(f"Existe? {os.path.exists(config_path)}")
-
-        if os.path.exists(config_path):
-            with open(config_path) as f:
-                config_content = f.read()
-                logger.info(f"Config content ({len(config_content)} chars):")
-                logger.info(config_content)
-
-        with open(config_path) as f:
+        with open('config.yaml') as f:
             config = yaml.safe_load(f)
 
-        logger.info(f"✅ Config carregado:")
-        for key, value in config.items():
-            logger.info(f"  {key}: {value}")
-
         cluster_id = config['cluster_id']
         chunk_start = config['chunk_start']
         chunk_end = config['chunk_end']
         github_repo = config['github_repo']
+        campos_filter = config['campos_filter']
 
-        logger.info(f"\n📊 Configuração:")
         logger.info(f"  Cluster: {cluster_id}")
         logger.info(f"  Chunks: {chunk_start} → {chunk_end} ({chunk_end - chunk_start + 1} chunks)")
-        logger.info(f"  Repo: {github_repo}")
-
-        # VERIFICAR SE JÁ PRONTO
-        if READY_FLAG.exists():
-            logger.info(f"\n✅ FAISS já existe em {READY_FLAG}")
-            update_status('ready', 'FAISS já existe', 100)
-            return
-
-        # PASSO 1: GIT CLONE
-        logger.info("\n" + "="*80)
-        logger.info("📥 PASSO 1: Git Clone (Sparse Checkout)")
-        logger.info("="*80)
-        update_status('cloning', 'Iniciando clone do repositório', 10)
+        logger.info(f"  Campos: {campos_filter}")
 
-        repo_dir = '/tmp/repo'
-        logger.info(f"Diretório destino: {repo_dir}")
+        # PREPARAR URLS
+        # URL correta: https://raw.githubusercontent.com/USER/REPO/BRANCH/PATH
+        base_url = github_repo.replace('https://github.com/', 'https://raw.githubusercontent.com/')
+        if base_url.endswith('.git'):
+            base_url = base_url[:-4]
+        base_url = f"{base_url}/main/chunks_dados"
 
-        # Criar diretório
-        logger.info(f"Criando {repo_dir}...")
-        os.makedirs(repo_dir, exist_ok=True)
-        logger.info(f"✅ Diretório criado")
+        logger.info(f"  Base URL: {base_url}")
 
-        # Listar /tmp antes
-        logger.info("\nConteúdo de /tmp ANTES:")
-        run_cmd("ls -la /tmp", "List /tmp", check=False)
+        # CRIAR DIRETÓRIOS
+        work_dir = Path('/tmp/work')
+        work_dir.mkdir(exist_ok=True)
 
-        # Mudar para diretório
-        logger.info(f"\nMudando para {repo_dir}...")
-        os.chdir(repo_dir)
-        logger.info(f"✅ PWD agora: {os.getcwd()}")
+        output_jsonl = work_dir / 'all_filtered.jsonl'
+        if output_jsonl.exists():
+            output_jsonl.unlink()
 
-        # Testar conectividade GitHub
-        logger.info("\n🌐 Testando conectividade com GitHub...")
-        run_cmd("curl -I https://github.com 2>&1 | head -5", "GitHub connectivity", check=False)
-
-        # Git clone
-        logger.info(f"\n📦 Clonando {github_repo}...")
-        logger.info("Comando: git clone --filter=blob:none --sparse ...")
+        # PASSO 1: DOWNLOAD E PROCESSAR CHUNKS (UM POR VEZ)
+        logger.info("\n" + "="*80)
+        logger.info("📥 PASSO 1: Download e Processamento de Chunks")
+        logger.info("="*80)
+        update_status('downloading', 'Baixando e processando chunks', 10)
 
-        try:
-            run_cmd(
-                f"git clone --filter=blob:none --no-checkout --sparse {github_repo} . 2>&1",
-                "Git clone sparse"
-            )
-        except Exception as e:
-            logger.error("\n❌ GIT CLONE FALHOU!")
-            logger.error("Tentando diagnóstico adicional...")
-
-            # Diagnóstico
-            run_cmd("ls -la", "List current dir", check=False)
-            run_cmd("git --version", "Git version", check=False)
-            run_cmd(f"git ls-remote {github_repo} 2>&1 | head -10", "Git ls-remote", check=False)
-            run_cmd("cat ~/.gitconfig 2>/dev/null || echo 'No gitconfig'", "Git config", check=False)
-
-            raise Exception(f"Git clone falhou: {e}")
-
-        # Verificar clone
-        logger.info("\n✅ Clone concluído, verificando...")
-        run_cmd("ls -la", "List repo dir")
-        run_cmd("git status 2>&1 | head -20", "Git status", check=False)
-
-        # Sparse checkout init
-        logger.info("\n🔧 Configurando sparse checkout...")
-        run_cmd("git sparse-checkout init --cone", "Sparse checkout init")
-
-        # Adicionar patterns em batches
-        logger.info(f"\n📋 Adicionando patterns para chunks {chunk_start}-{chunk_end}...")
-        patterns = [
-            f"chunks_dados/chunk_dados_{i:04d}.tar.gz"
-            for i in range(chunk_start, chunk_end + 1)
-        ]
-
-        logger.info(f"Total de patterns: {len(patterns)}")
-        logger.info(f"Primeiros 5: {patterns[:5]}")
-        logger.info(f"Últimos 5: {patterns[-5:]}")
-
-        batch_size = 50
-        for i in range(0, len(patterns), batch_size):
-            batch = patterns[i:i+batch_size]
-            batch_num = i // batch_size + 1
-            total_batches = (len(patterns) + batch_size - 1) // batch_size
-
-            logger.info(f"\nBatch {batch_num}/{total_batches} ({len(batch)} patterns)...")
-
-            patterns_str = ' '.join(batch)
-            run_cmd(
-                f"git sparse-checkout add {patterns_str}",
-                f"Add patterns batch {batch_num}"
-            )
+        total_records = 0
 
-        # Checkout
-        logger.info("\n📥 Fazendo checkout dos arquivos...")
-        run_cmd("git checkout main 2>&1", "Git checkout main")
+        for chunk_num in range(chunk_start, chunk_end + 1):
+            # Nome do arquivo: chunk_dados_000001.tar.gz
+            chunk_name = f"chunk_dados_{chunk_num:06d}.tar.gz"
+            chunk_url = f"{base_url}/{chunk_name}"
+            chunk_path = work_dir / chunk_name
 
-        # Verificar chunks baixados
-        logger.info("\n📊 Verificando chunks baixados...")
-        chunks_found = run_cmd(
-            "find chunks_dados -name '*.tar.gz' 2>/dev/null | wc -l",
-            "Count chunks"
-        ).strip()
+            logger.info(f"\n📦 Chunk {chunk_num}/{chunk_end}")
+            logger.info(f"  URL: {chunk_url}")
 
-        logger.info(f"✅ Chunks encontrados: {chunks_found}")
+            # Download com curl
+            progress = 10 + ((chunk_num - chunk_start) * 40 // (chunk_end - chunk_start + 1))
+            update_status('downloading', f'Baixando chunk {chunk_num}/{chunk_end}', progress)
 
-        if chunks_found == '0':
-            logger.error("❌ NENHUM CHUNK ENCONTRADO!")
-            run_cmd("find . -type f 2>/dev/null | head -20", "List all files", check=False)
-            raise Exception("Nenhum chunk foi baixado")
+            try:
+                run_cmd(
+                    f"curl -L -f -o {chunk_path} {chunk_url}",
+                    f"Download {chunk_name}",
+                    timeout=300
+                )
 
-        # PASSO 2: EXTRACT
-        logger.info("\n" + "="*80)
-        logger.info("📦 PASSO 2: Extraindo chunks")
-        logger.info("="*80)
-        update_status('extracting', f'Extraindo {chunks_found} chunks', 30)
+                if not chunk_path.exists() or chunk_path.stat().st_size == 0:
+                    logger.warning(f"  ⚠️  Arquivo vazio ou não baixado: {chunk_name}")
+                    continue
 
-        extract_dir = '/tmp/extracted'
-        logger.info(f"Diretório destino: {extract_dir}")
-        os.makedirs(extract_dir, exist_ok=True)
+                logger.info(f"  ✅ Baixado: {chunk_path.stat().st_size / 1024 / 1024:.2f} MB")
 
-        run_cmd(
-            "find chunks_dados -name '*.tar.gz' -exec tar -xzf {} -C /tmp/extracted \; 2>&1 | head -50",
-            "Extract all chunks",
-            check=False
-        )
+                # Processar TAR.GZ (extrai + filtra + concatena)
+                records = process_tar_gz(chunk_path, output_jsonl, campos_filter)
+                total_records += records
 
-        # Verificar extração
-        jsonl_count = run_cmd(
-            "find /tmp/extracted -name 'jurisprudencias.jsonl' 2>/dev/null | wc -l",
-            "Count JSONL files"
-        ).strip()
-        logger.info(f"✅ Arquivos JSONL extraídos: {jsonl_count}")
+                # Apagar TAR.GZ para economizar espaço
+                chunk_path.unlink()
+                logger.info(f"  🗑️  Arquivo TAR.GZ apagado (economizando espaço)")
 
-        # PASSO 3: CONCAT
-        logger.info("\n" + "="*80)
-        logger.info("📄 PASSO 3: Concatenando JSONL")
-        logger.info("="*80)
-        update_status('concatenating', 'Concatenando registros', 50)
+            except Exception as e:
+                logger.error(f"  ❌ Erro no chunk {chunk_num}: {e}")
+                # Continua com próximo chunk
+                if chunk_path.exists():
+                    chunk_path.unlink()
+                continue
 
-        run_cmd(
-            "find /tmp/extracted -name 'jurisprudencias.jsonl' -exec cat {} \; > /tmp/all_records.jsonl 2>&1",
-            "Concatenate JSONL"
-        )
+        logger.info(f"\n✅ Total de registros processados: {total_records}")
 
-        total_lines = run_cmd(
-            "wc -l < /tmp/all_records.jsonl 2>/dev/null || echo '0'",
-            "Count lines"
-        ).strip()
+        if total_records == 0:
+            raise Exception("Nenhum registro foi processado!")
 
-        logger.info(f"✅ Total de registros: {total_lines}")
+        # Verificar arquivo final
+        if not output_jsonl.exists():
+            raise Exception("Arquivo all_filtered.jsonl não foi criado!")
 
-        if total_lines == '0':
-            raise Exception("JSONL concatenado está vazio!")
+        final_lines = int(run_cmd(f"wc -l < {output_jsonl}", "Count lines").strip())
+        logger.info(f"✅ Linhas no JSONL final: {final_lines}")
 
-        # PASSO 4: FILTER
+        # PASSO 2: BUILD FAISS
         logger.info("\n" + "="*80)
-        logger.info("🔍 PASSO 4: Filtrando campos")
+        logger.info("🤖 PASSO 2: Construindo FAISS index")
         logger.info("="*80)
-        update_status('filtering', 'Filtrando campos (id + ementa)', 60)
+        update_status('building', f'Construindo FAISS com {final_lines} documentos', 70)
 
         os.chdir('/home/user/app')
-        logger.info(f"PWD: {os.getcwd()}")
-
-        run_cmd(
-            "python3 filter_fields.py --input /tmp/all_records.jsonl --output /tmp/filtered.jsonl 2>&1",
-            "Filter fields"
-        )
-
-        filtered_lines = run_cmd(
-            "wc -l < /tmp/filtered.jsonl 2>/dev/null || echo '0'",
-            "Count filtered"
-        ).strip()
-        logger.info(f"✅ Registros filtrados: {filtered_lines}")
 
-        # PASSO 5: BUILD FAISS
-        logger.info("\n" + "="*80)
-        logger.info("🤖 PASSO 5: Construindo FAISS index")
-        logger.info("="*80)
-        update_status('building', f'Construindo FAISS com {filtered_lines} documentos', 70)
-
-        logger.info("⚠️  Este passo pode demorar ~10-15 minutos...")
-
-        run_cmd(
-            "python3 rag_builder.py --input /tmp/filtered.jsonl 2>&1",
-            "Build FAISS",
-            capture=False
-        )
+        try:
+            run_cmd(
+                f"python3 rag_builder.py --input {output_jsonl} 2>&1",
+                "Build FAISS",
+                timeout=900  # 15 minutos
+            )
+        except Exception as e:
+            logger.error(f"❌ Build FAISS falhou: {e}")
+            # Tentar ler stderr do rag_builder
+            raise Exception(f"Build FAISS falhou: {e}")
 
         logger.info("✅ FAISS construído!")
 
-        # Verificar FAISS
-        faiss_files = run_cmd(
-            "ls -lh /app/faiss_index 2>&1",
-            "List FAISS files",
-            check=False
-        )
-
-        # PASSO 6: CLEANUP
-        logger.info("\n" + "="*80)
-        logger.info("🧹 PASSO 6: Limpando temporários")
-        logger.info("="*80)
+        # PASSO 3: CLEANUP
+        logger.info("\n🧹 PASSO 3: Limpando temporários")
         update_status('cleaning', 'Limpando arquivos temporários', 95)
 
-        run_cmd(
-            "rm -rf /tmp/repo /tmp/extracted /tmp/all_records.jsonl /tmp/filtered.jsonl",
-            "Cleanup temp files"
-        )
+        run_cmd(f"rm -rf {work_dir}", "Cleanup", check=False)
 
         # CONCLUÍDO
         logger.info("\n" + "="*80)
         logger.info("✅ SETUP COMPLETO!")
         logger.info("="*80)
 
-        update_status('ready', f'FAISS pronto com {total_lines} registros!', 100)
+        update_status('ready', f'FAISS pronto com {total_records} registros!', 100)
         READY_FLAG.touch()
 
         logger.info(f"⏰ Fim: {datetime.now()}")
-        logger.info(f"📁 Logs salvos em: /tmp/setup_debug.log")
-        logger.info("="*80)
 
     except Exception as e:
         logger.error("\n" + "="*80)
@@ -391,13 +266,10 @@ def main():
         logger.error("="*80)
         logger.error(f"Tipo: {type(e).__name__}")
         logger.error(f"Mensagem: {str(e)}")
-        logger.error("\nTraceback completo:")
+        logger.error("\nTraceback:")
         logger.error(traceback.format_exc())
-        logger.error("="*80)
-        logger.error(f"📁 Logs completos em: /tmp/setup_debug.log")
-        logger.error("="*80)
 
-        update_status('error', f'Build FAISS falhou: {str(e)}', 0)
+        update_status('error', f'Setup falhou: {str(e)}', 0)
         sys.exit(1)
 
 if __name__ == "__main__":