Delete app.py

app.py

@@ -1,263 +0,0 @@
-import sqlite3
-import asyncio
-import aiohttp
-import json
-import os
-import time
-import threading
-from bs4 import BeautifulSoup
-from fastapi import FastAPI
-from huggingface_hub import HfApi, hf_hub_download
-import uvicorn
-
-# ─── НАСТРОЙКИ ────────────────────────────────────────────────
-DB_PATH         = '/app/linkm_posts.db'
-PROGRESS_FILE   = '/app/linkm_progress.json'
-BASE_URL        = 'https://linkm.me/posts/{}'
-START_ID        = 0
-END_ID          = 212915997
-WORKERS         = 150
-BATCH_SIZE      = 500
-REQUEST_TIMEOUT = 10
-HF_TOKEN        = os.environ.get('HF_TOKEN')
-HF_DATASET      = os.environ.get('HF_DATASET')
-PUSH_EVERY      = 10000  # пушить каждые 10k найденных
-
-HEADERS = {
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-    'Accept-Language': 'en-US,en;q=0.5',
-}
-
-# ─── СОСТОЯНИЕ ────────────────────────────────────────────────
-state = {
-    'status': 'starting',
-    'current_id': START_ID,
-    'found': 0,
-    'processed': 0,
-    'speed': 0,
-    'last_push': 0,
-    'started_at': time.time(),
-}
-
-# ─── PULL С HF ПРИ СТАРТЕ ─────────────────────────────────────
-def pull_from_hf():
-    if not HF_TOKEN or not HF_DATASET:
-        return
-    try:
-        hf_hub_download(
-            repo_id=HF_DATASET,
-            filename='linkm_posts.db',
-            repo_type='dataset',
-            local_dir='/app',
-            token=HF_TOKEN,
-        )
-        print('✅ База скачана с HF')
-    except Exception as e:
-        print(f'⚠️ Не удалось скачать базу: {e}')
-
-    try:
-        hf_hub_download(
-            repo_id=HF_DATASET,
-            filename='linkm_progress.json',
-            repo_type='dataset',
-            local_dir='/app',
-            token=HF_TOKEN,
-        )
-        print('✅ Прогресс скачан с HF')
-    except Exception as e:
-        print(f'⚠️ Прогресс не найден на HF: {e}')
-
-# ─── БАЗА ДАННЫХ ──────────────────────────────────────────────
-def init_db():
-    conn = sqlite3.connect(DB_PATH)
-    c = conn.cursor()
-    c.execute('''
-        CREATE TABLE IF NOT EXISTS posts (
-            post_id   INTEGER PRIMARY KEY,
-            text      TEXT NOT NULL,
-            parsed_at TEXT DEFAULT (datetime('now'))
-        )
-    ''')
-    conn.commit()
-    conn.close()
-
-# ─── ПРОГРЕСС ─────────────────────────────────────────────────
-def load_progress():
-    if os.path.exists(PROGRESS_FILE):
-        try:
-            with open(PROGRESS_FILE, 'r') as f:
-                data = json.load(f)
-            print(f'▶️  Продолжаем с ID: {data["next_id"]} (найдено: {data["found"]:,})')
-            return data
-        except Exception:
-            pass
-    return {'next_id': START_ID, 'processed': 0, 'found': 0}
-
-def save_progress(data):
-    with open(PROGRESS_FILE, 'w') as f:
-        json.dump(data, f)
-
-# ─── ПАРСИНГ ──────────────────────────────────────────────────
-def extract_text(html: str):
-    soup = BeautifulSoup(html, 'html.parser')
-
-    main = soup.find('main', class_='deeplink_main')
-    if main:
-        p = main.find('p')
-        if p:
-            parts = [node.strip() for node in p.descendants if isinstance(node, str) and node.strip()]
-            result = ' '.join(parts).strip()
-            if ' on Link. ' in result:
-                result = result.split(' on Link. ', 1)[1].strip()
-            elif result.endswith(' on Link.'):
-                return None
-            if result:
-                return result
-
-    og = soup.find('meta', property='og:description')
-    if og:
-        content = og.get('content', '').strip()
-        if ' on Link. ' in content:
-            content = content.split(' on Link. ', 1)[1].strip()
-        elif content.endswith(' on Link.'):
-            return None
-        if content:
-            return content
-
-    return None
-
-async def fetch_post(session, post_id):
-    try:
-        async with session.get(
-            BASE_URL.format(post_id),
-            timeout=aiohttp.ClientTimeout(total=REQUEST_TIMEOUT)
-        ) as resp:
-            if resp.status != 200:
-                return post_id, None
-            html = await resp.text(encoding='utf-8', errors='replace')
-            return post_id, extract_text(html)
-    except Exception:
-        return post_id, None
-
-def save_batch(rows):
-    if not rows:
-        return
-    conn = sqlite3.connect(DB_PATH)
-    conn.executemany('INSERT OR IGNORE INTO posts (post_id, text) VALUES (?, ?)', rows)
-    conn.commit()
-    conn.close()
-
-# ─── PUSH В HUGGINGFACE ───────────────────────────────────────
-def push_to_hf(progress):
-    if not HF_TOKEN or not HF_DATASET:
-        return
-    try:
-        api = HfApi(token=HF_TOKEN)
-        api.upload_file(
-            path_or_fileobj=DB_PATH,
-            path_in_repo='linkm_posts.db',
-            repo_id=HF_DATASET,
-            repo_type='dataset',
-        )
-        api.upload_file(
-            path_or_fileobj=PROGRESS_FILE,
-            path_in_repo='linkm_progress.json',
-            repo_id=HF_DATASET,
-            repo_type='dataset',
-        )
-        print(f'✅ Запушено в HF (найдено: {progress["found"]:,})')
-        state['last_push'] = progress['found']
-    except Exception as e:
-        print(f'❌ Ошибка пуша: {e}')
-
-# ─── ГЛАВНЫЙ ЦИКЛ ─────────────────────────────────────────────
-async def run_parser():
-    pull_from_hf()
-    init_db()
-
-    progress = load_progress()
-    current_id = progress['next_id']
-    start_time = time.time()
-
-    state['status'] = 'running'
-    state['current_id'] = current_id
-    state['found'] = progress['found']
-    state['processed'] = progress['processed']
-    state['last_push'] = progress['found']
-
-    connector = aiohttp.TCPConnector(limit=WORKERS, ssl=False)
-
-    async with aiohttp.ClientSession(connector=connector, headers=HEADERS) as session:
-        while current_id < END_ID:
-            batch_ids = list(range(current_id, min(current_id + BATCH_SIZE, END_ID)))
-            results = await asyncio.gather(*[fetch_post(session, pid) for pid in batch_ids])
-
-            found_rows = [(pid, txt) for pid, txt in results if txt]
-            save_batch(found_rows)
-
-            progress['processed'] += len(batch_ids)
-            progress['found'] += len(found_rows)
-            progress['next_id'] = current_id + BATCH_SIZE
-            save_progress(progress)
-
-            elapsed = time.time() - start_time
-            speed = progress['processed'] / max(elapsed, 1)
-
-            state['current_id'] = current_id
-            state['found'] = progress['found']
-            state['processed'] = progress['processed']
-            state['speed'] = round(speed)
-
-            print(f'ID: {current_id:,} | Найдено: {progress["found"]:,} | {speed:.0f} ID/сек')
-
-            if progress['found'] - state['last_push'] >= PUSH_EVERY:
-                push_to_hf(progress)
-
-            current_id += BATCH_SIZE
-
-    state['status'] = 'done'
-    push_to_hf(progress)
-
-def start_parser():
-    asyncio.run(run_parser())
-
-# ─── FASTAPI ──────────────────────────────────────────────────
-app = FastAPI()
-
-@app.get('/status')
-def get_status():
-    elapsed = time.time() - state['started_at']
-    remaining_ids = END_ID - state['current_id']
-    remaining_sec = remaining_ids / max(state['speed'], 1)
-    return {
-        'status': state['status'],
-        'current_id': state['current_id'],
-        'end_id': END_ID,
-        'percent': round(state['current_id'] / END_ID * 100, 2),
-        'found': state['found'],
-        'processed': state['processed'],
-        'speed_per_sec': state['speed'],
-        'elapsed_min': round(elapsed / 60, 1),
-        'remaining_min': round(remaining_sec / 60, 1),
-        'last_push_at': state['last_push'],
-    }
-
-@app.get('/search')
-def search(q: str, limit: int = 50):
-    conn = sqlite3.connect(DB_PATH)
-    c = conn.cursor()
-    c.execute('SELECT post_id, text FROM posts WHERE text LIKE ? LIMIT ?', (f'%{q}%', limit))
-    rows = c.fetchall()
-    conn.close()
-    return {
-        'query': q,
-        'count': len(rows),
-        'results': [{'post_id': r[0], 'text': r[1]} for r in rows]
-    }
-
-# ─── СТАРТ ────────────────────────────────────────────────────
-if __name__ == '__main__':
-    t = threading.Thread(target=start_parser, daemon=True)
-    t.start()
-    uvicorn.run(app, host='0.0.0.0', port=7860)