Spaces:
Runtime error
Runtime error
File size: 4,533 Bytes
e5e28c9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | import os
import sqlite3
import uvicorn
import requests
import threading
import time
import random
from bs4 import BeautifulSoup
from ddgs import DDGS
from contextlib import asynccontextmanager
from fastapi import FastAPI, Query
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse
# --- CONFIGURAÇÃO DE DIRETÓRIO ---
BASE_DIR = "./data"
if not os.path.exists(BASE_DIR):
try: os.makedirs(BASE_DIR)
except: BASE_DIR = "." # Fallback caso não tenha permissão no C:
DB_PATH = os.path.join(BASE_DIR, "gm_codesearch.db")
# --- LÓGICA DE BANCO DE DATOS ---
def init_db():
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
cursor.execute('''CREATE TABLE IF NOT EXISTS snippets
(id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT, code TEXT, language TEXT,
source_url TEXT UNIQUE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)''')
# Sementes iniciais para teste de busca
cursor.execute("INSERT OR IGNORE INTO snippets (title, code, language, source_url) VALUES (?, ?, ?, ?)",
("Bootstrap Navbar", "<nav class='navbar'>...</nav>", "html", "seed_1"))
cursor.execute("INSERT OR IGNORE INTO snippets (title, code, language, source_url) VALUES (?, ?, ?, ?)",
("Python List Comprehension", "[x for x in list if x > 0]", "python", "seed_2"))
conn.commit()
conn.close()
def add_snippet(title, code, url):
if not code or len(code) < 60: return False
try:
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
lang = "python" if "python" in url.lower() else "js"
if "html" in url.lower(): lang = "html"
cursor.execute("INSERT OR IGNORE INTO snippets (title, code, language, source_url) VALUES (?, ?, ?, ?)",
(title[:100], code.strip(), lang, url))
conn.commit()
success = cursor.rowcount > 0
conn.close()
return success
except: return False
# --- DUCK HUNTER (O CRAWLER) ---
CRAWLER_LOGS = []
def duck_hunter():
global CRAWLER_LOGS
print("🦆 Pato saiu para caçar...")
while True:
queries = ["python function snippets", "javascript async await", "css flexbox tricks", "fastapi examples"]
for q in queries:
try:
with DDGS() as ddgs:
results = [r for r in ddgs.text(q, max_results=5)]
for res in results:
time.sleep(random.uniform(5, 10))
try:
page = requests.get(res['href'], timeout=10, headers={'User-Agent': 'Mozilla/5.0'})
soup = BeautifulSoup(page.text, 'html.parser')
blocks = soup.find_all(['pre', 'code'])
for b in blocks:
if add_snippet(res['title'], b.get_text(), res['href']):
CRAWLER_LOGS.insert(0, f"✅ PESCOU: {res['title'][:20]}...")
break
except: continue
CRAWLER_LOGS = CRAWLER_LOGS[:12]
except: pass
time.sleep(30)
# --- CONFIGURAÇÃO DO FASTAPI ---
@asynccontextmanager
async def lifespan(app: FastAPI):
init_db()
threading.Thread(target=duck_hunter, daemon=True).start()
yield
app = FastAPI(lifespan=lifespan)
# Monta a pasta onde você vai colocar o CSS e o JS
# Seus arquivos devem estar dentro de uma pasta chamada 'frontend'
if not os.path.exists("frontend"): os.makedirs("frontend")
app.mount("/frontend", StaticFiles(directory="frontend"), name="frontend")
# Rota principal que entrega o seu HTML
@app.get("/")
async def get_index():
return FileResponse('frontend/index.html')
# Endpoint que o JavaScript vai consultar
@app.get("/api/data")
def api_data(q: str = ""):
conn = sqlite3.connect(DB_PATH)
conn.row_factory = sqlite3.Row
cursor = conn.cursor()
cursor.execute("SELECT COUNT(*) FROM snippets")
total = cursor.fetchone()[0]
search_q = f"%{q}%"
cursor.execute("SELECT * FROM snippets WHERE title LIKE ? OR code LIKE ? ORDER BY id DESC LIMIT 40",
(search_q, search_q))
results = [dict(r) for r in cursor.fetchall()]
conn.close()
return {
"total": total,
"results": results,
"logs": CRAWLER_LOGS
}
if __name__ == "__main__":
uvicorn.run(app, host="127.0.0.1", port=8000) |