Update app.py

app.py

@@ -1,438 +1,3 @@
-
-import os
-import json
-import time
-import copy
-import logging
-import requests
-import urllib3
-from flask import Flask, request, jsonify
-from playwright.sync_api import sync_playwright
-
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-logger = logging.getLogger(__name__)
-app = Flask(__name__)
-
-# ============================================
-# PAYLOADS STF (igual app-7.py)
-# ============================================
-PAYLOAD_COMPLETO = {
-    "query": {
-        "function_score": {
-            "functions": [
-                {"exp": {"julgamento_data": {"origin": "now", "scale": "47450d", "offset": "1095d", "decay": 0.1}}}
-            ],
-            "query": {
-                "bool": {
-                    "filter": [
-                        {"query_string": {
-                            "default_operator": "AND",
-                            "fields": [
-                                "processo_codigo_completo.plural",
-                                "acordao_ata.plural^3",
-                                "ementa_texto.plural^3",
-                                "decisao_texto.plural^2",
-                                "documental_tese_texto.plural^3",
-                                "documental_tese_tema_texto.plural^3",
-                                "titulo.plural^6",
-                                "sumula_texto.plural^3",
-                                "ramo_direito.plural^1"
-                            ],
-                            "query": "*",
-                            "type": "cross_fields",
-                            "fuzziness": "AUTO:4,7"
-                        }}
-                    ]
-                }
-            }
-        }
-    },
-    "_source": [
-        "id", "titulo", "ementa_texto", "acordao_ata", "decisao_texto",
-        "documental_tese_texto", "orgao_julgador", "relator_processo_nome",
-        "julgamento_data", "processo_codigo_completo",
-        "inteiro_teor_url", "acompanhamento_processual_url", "dje_url",
-        "processo_classe_processual_unificada_extenso", "ministro_facet"
-    ],
-    "highlight": {
-        "fields": {
-            "ementa_texto": {"matched_fields": ["ementa_texto.plural"], "type": "fvh", "fragment_size": 24000},
-            "acordao_ata": {"matched_fields": ["acordao_ata.plural"], "type": "fvh", "fragment_size": 600},
-            "decisao_texto": {"matched_fields": ["decisao_texto.plural"], "type": "fvh", "fragment_size": 1200}
-        },
-        "pre_tags": ["<em>"],
-        "post_tags": ["</em>"],
-        "fragment_size": 300,
-        "number_of_fragments": 64,
-        "order": "score"
-    },
-    "size": 100,
-    "from": 0,
-    "sort": [{"julgamento_data": {"order": "desc"}}],
-    "track_total_hits": True
-}
-
-PAYLOAD_SEM_HIGHLIGHT = {
-    "query": PAYLOAD_COMPLETO["query"],
-    "_source": PAYLOAD_COMPLETO["_source"],
-    "size": 100,
-    "from": 0,
-    "sort": [{"julgamento_data": {"order": "desc"}}],
-    "track_total_hits": True
-}
-
-PAYLOAD_MINIMO = {
-    "query": {
-        "bool": {
-            "filter": [{"term": {"base": "acordaos"}}]
-        }
-    },
-    "_source": ["id", "titulo", "ementa_texto", "processo_numero", "julgamento_data", "relator_processo_nome", "inteiro_teor_url"],
-    "size": 100,
-    "sort": [{"julgamento_data": {"order": "desc"}}],
-    "track_total_hits": True
-}
-
-# Constantes
-URL_API = "https://jurisprudencia.stf.jus.br/api/search/search"
-HEADERS = {
-    "Accept": "application/json, text/plain, */*",
-    "Content-Type": "application/json",
-    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
-    "Referer": "https://jurisprudencia.stf.jus.br/pages/search",
-    "Origin": "https://jurisprudencia.stf.jus.br"
-}
-token_cache = {"token": None, "expires_at": 0}
-
-# ============================================
-# FUNÇÕES CORE (igual app-7.py)
-# ============================================
-def get_fresh_token():
-    global token_cache
-    if token_cache["token"] and time.time() < token_cache["expires_at"]:
-        logger.info("Usando token em cache")
-        return token_cache["token"]
-
-    logger.info("Obtendo novo token via Playwright")
-    try:
-        with sync_playwright() as p:
-            browser = p.chromium.launch(headless=True, args=["--no-sandbox"])
-            context = browser.new_context(
-                viewport={"width": 1920, "height": 1080},
-                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
-            )
-            page = context.new_page()
-            page.goto("https://jurisprudencia.stf.jus.br/pages/search", 
-                     wait_until="domcontentloaded", timeout=30000)
-            page.wait_for_timeout(3000)
-            cookies = context.cookies()
-            token = None
-            for cookie in cookies:
-                if cookie.get("name") == "aws-waf-token":
-                    token = cookie.get("value")
-                    break
-            browser.close()
-
-            if token:
-                token_cache["token"] = token
-                token_cache["expires_at"] = time.time() + 3300
-                logger.info(f"Token obtido: {token[:30]}...")
-                return token
-            else:
-                logger.warning("Token não encontrado nos cookies")
-                return None
-    except Exception as e:
-        logger.error(f"Erro ao obter token: {str(e)}")
-        return None
-
-
-def search_with_token(token, payload):
-    """
-    Busca com token. Retorna {"success": True/False, "data": {...}, "status": code}
-    HTTP 200 = sucesso
-    HTTP 403 = token expirado (limpa cache)
-    HTTP 202 = token inválido/não aceito (não é sucesso!)
-    Outros = erro
-    """
-    if not token:
-        return {"success": False, "error": "Token não fornecido"}
-
-    headers = HEADERS.copy()
-    headers["Cookie"] = f"aws-waf-token={token}"
-
-    try:
-        logger.debug(f"Enviando payload: {json.dumps(payload)[:200]}...")
-        response = requests.post(URL_API, headers=headers, json=payload, verify=False, timeout=30)
-        logger.info(f"Resposta: status {response.status_code}")
-
-        if response.status_code == 200:
-            return {"success": True, "data": response.json()}
-        elif response.status_code == 403:
-            token_cache["token"] = None  # Token expirado
-            return {"success": False, "error": "Token expirado", "status": 403}
-        elif response.status_code == 202:
-            # HTTP 202 = Accepted mas token não funcionou (WAF bloqueio)
-            return {"success": False, "error": "Token não aceito (HTTP 202)", "status": 202}
-        else:
-            return {"success": False, "error": f"HTTP {response.status_code}", 
-                   "status": response.status_code, "text": response.text[:500]}
-    except Exception as e:
-        logger.error(f"Erro na requisição: {str(e)}")
-        return {"success": False, "error": str(e)}
-
-
-def try_search_with_payloads(token, base_payload, sort_order="desc", base_filter=None):
-    """
-    Tenta vários payloads em ordem decrescente de complexidade até obter sucesso.
-    Retorna (success, data, payload_usado)
-    """
-    # Payload 1: completo com aggs e highlight
-    payload1 = base_payload.copy()
-    payload1["sort"] = [{"julgamento_data": {"order": sort_order}}]
-    if base_filter:
-        payload1["post_filter"] = {"bool": {"must": [{"term": {"base": base_filter}}]}}
-
-    logger.info("Tentando payload completo...")
-    res = search_with_token(token, payload1)
-    if res.get("success"):
-        return True, res["data"], "completo"
-
-    # Se falhou com 400, tenta sem highlight e sem aggs
-    if res.get("status") == 400:
-        logger.info("Payload completo falhou 400. Tentando payload sem highlight...")
-        payload2 = PAYLOAD_SEM_HIGHLIGHT.copy()
-        payload2["sort"] = [{"julgamento_data": {"order": sort_order}}]
-        if base_filter:
-            payload2["post_filter"] = {"bool": {"must": [{"term": {"base": base_filter}}]}}
-
-        res2 = search_with_token(token, payload2)
-        if res2.get("success"):
-            return True, res2["data"], "sem_highlight"
-
-        if res2.get("status") == 400:
-            logger.info("Payload sem highlight falhou. Tentando payload mínimo...")
-            payload3 = PAYLOAD_MINIMO.copy()
-            payload3["sort"] = [{"julgamento_data": {"order": sort_order}}]
-            if base_filter:
-                payload3["post_filter"] = {"bool": {"must": [{"term": {"base": base_filter}}]}}
-
-            res3 = search_with_token(token, payload3)
-            if res3.get("success"):
-                return True, res3["data"], "minimo"
-
-    return False, None, None
-
-
-def test_with_playwright_full(payload):
-    """Último recurso: Playwright com busca direta na API via browser"""
-    logger.info("Tentando acesso com Playwright...")
-    try:
-        with sync_playwright() as p:
-            browser = p.chromium.launch(headless=True, args=["--no-sandbox"])
-            context = browser.new_context(
-                viewport={"width": 1920, "height": 1080},
-                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
-            )
-            page = context.new_page()
-            page.goto("https://jurisprudencia.stf.jus.br/pages/search", 
-                     wait_until="domcontentloaded", timeout=30000)
-            page.wait_for_timeout(3000)
-
-            cookies = context.cookies()
-            token = None
-            for cookie in cookies:
-                if cookie.get("name") == "aws-waf-token":
-                    token = cookie.get("value")
-                    break
-
-            api_result = page.evaluate("""
-                async (payload) => {
-                    try {
-                        const response = await fetch('https://jurisprudencia.stf.jus.br/api/search/search', {
-                            method: 'POST',
-                            headers: {
-                                'Content-Type': 'application/json',
-                                'Accept': 'application/json'
-                            },
-                            body: JSON.stringify(payload)
-                        });
-                        if (response.ok) {
-                            return {success: true, data: await response.json()};
-                        } else {
-                            return {success: false, status: response.status};
-                        }
-                    } catch (error) {
-                        return {success: false, error: error.toString()};
-                    }
-                }
-            """, payload)
-
-            browser.close()
-
-            if api_result.get("success"):
-                if token:
-                    token_cache["token"] = token
-                    token_cache["expires_at"] = time.time() + 3300
-                return {"success": True, "data": api_result.get("data"), "token": token}
-            else:
-                return {"success": False, "error": api_result.get("error", "Falha desconhecida"), 
-                       "status": api_result.get("status"), "token": token}
-    except Exception as e:
-        logger.error(f"Erro no Playwright: {str(e)}")
-        return {"success": False, "error": str(e)}
-
-
-# ============================================
-# ENDPOINT /busca (simplificado para Para.AI)
-# ============================================
-@app.route("/busca")
-def busca():
-    """
-    GET /busca?q=termo&page=1&size=20
-    Endpoint simplificado para integração Para.AI
-    """
-    query = request.args.get("q", "").strip()
-    page = max(1, int(request.args.get("page", 1)))
-    size = min(int(request.args.get("size", 20)), 100)
-
-    if not query:
-        return jsonify({"error": "Parâmetro 'q' é obrigatório"}), 400
-
-    # Monta payload base
-    payload = copy.deepcopy(PAYLOAD_COMPLETO)
-    payload["query"]["function_score"]["query"]["bool"]["filter"][0]["query_string"]["query"] = query
-    payload["size"] = size
-    payload["from"] = (page - 1) * size
-
-    # Obtém token
-    token = get_fresh_token()
-    if not token:
-        return jsonify({"success": False, "error": "Falha ao obter token WAF"}), 500
-
-    # Tenta payloads em cascata
-    success, data, payload_type = try_search_with_payloads(token, payload, "desc", None)
-
-    if success:
-        logger.info(f"Busca bem-sucedida com payload {payload_type}")
-        # Extrai hits (API retorna result.hits.hits)
-        hits_root = data.get("result", data)
-        hits = hits_root.get("hits", {})
-        total = hits.get("total", {}).get("value", 0)
-        resultados = hits.get("hits", [])
-
-        # Formata resposta
-        results = []
-        for hit in resultados:
-            source = hit.get("_source", {})
-            hl = hit.get("highlight", {})
-
-            ementa = (
-                hl.get("ementa_texto", [None])[0] or
-                source.get("ementa_texto") or ""
-            )
-
-            results.append({
-                "id": source.get("id"),
-                "titulo": source.get("titulo"),
-                "processo": source.get("processo_codigo_completo"),
-                "orgao": source.get("orgao_julgador"),
-                "relator": source.get("relator_processo_nome"),
-                "data": source.get("julgamento_data"),
-                "url_documento": (
-                    source.get("inteiro_teor_url") or
-                    source.get("acompanhamento_processual_url") or
-                    source.get("dje_url")
-                ),
-                "ementa": ementa.strip(),
-                "score": hit.get("_score", 0)
-            })
-
-        return jsonify({
-            "success": True,
-            "query": query,
-            "total": total,
-            "page": page,
-            "size": size,
-            "results": results
-        })
-    else:
-        # Fallback Playwright
-        res = test_with_playwright_full(payload)
-        if res.get("success"):
-            return jsonify({"success": True, "method": "playwright", 
-                          "token": res.get("token"), "data": res["data"]})
-
-        return jsonify({"success": False, "error": "Todas as tentativas falharam"}), 500
-
-
-@app.route("/health")
-def health():
-    """Status do serviço"""
-    playwright_status = False
-    try:
-        with sync_playwright() as p:
-            p.chromium.launch(headless=True).close()
-            playwright_status = True
-    except:
-        pass
-
-    total_docs = None
-    token = get_fresh_token()
-    if token:
-        try:
-            res = search_with_token(token, {"size": 0, "track_total_hits": True})
-            if res.get("success") and res["data"].get("hits", {}).get("total"):
-                total_docs = res["data"]["hits"]["total"]["value"]
-        except:
-            pass
-
-    return jsonify({
-        "status": "healthy",
-        "timestamp": time.time(),
-        "playwright_ready": playwright_status,
-        "token_cached": bool(token_cache["token"]),
-        "token_expires_in": max(0, token_cache["expires_at"] - time.time()) if token_cache["expires_at"] else 0,
-        "total_docs": total_docs,
-        "python_version": sys.version
-    })
-
-
-if __name__ == "__main__":
-    try:
-        import certifi
-        os.environ["SSL_CERT_FILE"] = certifi.where()
-        os.environ["REQUESTS_CA_BUNDLE"] = certifi.where()
-    except:
-        pass
-
-    logger.info("=" * 50)
-    logger.info("🚀 Iniciando aplicação STF Jurisprudência")
-    logger.info(f"📋 Campos _source: {len(PAYLOAD_COMPLETO['_source'])}")
-    logger.info("=" * 50)
-
-    try:
-        with sync_playwright() as p:
-            p.chromium.launch(headless=True).close()
-            logger.info("✅ Playwright pronto para uso")
-    except Exception as e:
-        logger.warning(f"⚠️  Playwright pode não estar configurado: {e}")
-
-    port = int(os.environ.get("PORT", 7860))
-    app.run(host="0.0.0.0", port=port, debug=False)
-
-Identifiquei os seguintes problemas no código:
-
-1. **`import sys` ausente** — usado em `/health` mas nunca importado
-2. **Shallow copy em `PAYLOAD_SEM_HIGHLIGHT`** — referencia `PAYLOAD_COMPLETO["query"]` diretamente, mutações vazam entre payloads
-3. **`try_search_with_payloads` usa `.copy()` raso** — não isola o payload base
-4. **Payload desatualizado** — faltam as 3 funções de score, `post_filter`, e `aggs` do payload real capturado
-5. **`/busca` sem `post_filter`** — a API retorna todos os tipos (`acordaos`, `sumulas`, `decisoes`) sem o filtro explícito
-6. **`sort` padrão errado** — o endpoint usa `julgamento_data` por padrão, mas o portal usa `_score`
-
-Aqui está o código corrigido e atualizado:
-
-```python
 import os
 import sys
 import json
@@ -1158,5 +723,4 @@ if __name__ == "__main__":
         logger.warning(f"⚠️  Playwright: {e}")
 
     port = int(os.environ.get("PORT", 7860))
-    app.run(host="0.0.0.0", port=port, debug=False)
-
+    app.run(host="0.0.0.0", port=port, debug=False)