app.py

import asyncio
import aiohttp
import pandas as pd
import json
import os
import time
import math
import sys
from datetime import datetime, timedelta
from pathlib import Path
import gradio as gr

API_URL = "https://natiga.edudk.net/P20262026/public/api_result.php"
OUTPUT_DIR = Path("output")
OUTPUT_DIR.mkdir(exist_ok=True)
CONCURRENCY = 10
CHECKPOINT_EVERY = 2000
MAX_RETRIES = 3

state = {
    'running': False,
    'stage': 'idle',
    'found': [],
    'not_found_seats': [],
    'total_scraped': 0,
    'total_found': 0,
    'total_not_found': 0,
    'total_errors': 0,
    'current_seat': 0,
    'start_time': None,
    'elapsed': '00:00:00',
    'remaining': '--:--:--',
    'rate': 0,
    'progress_pct': 0,
    'logs': [],
}

def add_log(msg):
    ts = datetime.now().strftime("%H:%M:%S")
    state['logs'].append(f"[{ts}] {msg}")
    if len(state['logs']) > 500:
        state['logs'] = state['logs'][-500:]

def save_checkpoint():
    if state['found']:
        df = pd.DataFrame(state['found'])
        path = OUTPUT_DIR / "students_full.csv"
        df.to_csv(path, index=False, encoding='utf-8-sig')
        add_log(f"حفظ {len(state['found'])} طالب → students_full.csv")
    if state['not_found_seats']:
        pd.DataFrame({'seat_no': state['not_found_seats']}).to_csv(
            OUTPUT_DIR / "not_found.csv", index=False, encoding='utf-8-sig')

async def fetch_one(session, sem, seat):
    for attempt in range(MAX_RETRIES + 2):
        try:
            async with sem:
                async with session.get(f"{API_URL}?seat={seat}", timeout=aiohttp.ClientTimeout(total=12)) as resp:
                    if resp.status == 429:
                        wait = 5 * (attempt + 1)
                        add_log(f"⏳ Rate-limit على المقعد {seat}، انتظار {wait}ث")
                        await asyncio.sleep(wait)
                        continue
                    body = await resp.text()
                    data = json.loads(body)
                    if data.get('ok') and data.get('data'):
                        return 'found', data['data']
                    return 'not_found', seat
        except (asyncio.TimeoutError, aiohttp.ClientError) as e:
            if attempt < MAX_RETRIES + 1:
                await asyncio.sleep(1.5 ** attempt)
                continue
            return 'error', f"{seat}: {str(e)[:60]}"
        except json.JSONDecodeError:
            return 'error', f"{seat}: bad json"
    return 'not_found', seat

async def scrape_range(start, end, retry_mode=False):
    sem = asyncio.Semaphore(CONCURRENCY)
    state['start_time'] = time.time()
    total = end - start + 1
    done = 0
    connector = aiohttp.TCPConnector(limit=CONCURRENCY + 5, force_close=True)

    async with aiohttp.ClientSession(connector=connector) as session:
        batch = []
        for seat in range(start, end + 1):
            if not state['running']:
                add_log("❌ تم إيقاف السكريب")
                break
            batch.append(fetch_one(session, sem, seat))
            if len(batch) >= CONCURRENCY * 3:
                yield await process_batch(batch, total, retry_mode)
                done += len(batch)
                batch = []
        if batch:
            yield await process_batch(batch, total, retry_mode)

        if not retry_mode and state['running']:
            add_log("🏁 انتهى الشوط الأول — بدء إعادة فحص الغائبين...")
            save_checkpoint()
            state['stage'] = 'retry'
            nf = state['not_found_seats'][:]
            if nf:
                async for upd in scrape_range(min(nf), max(nf), retry_mode=True):
                    yield upd

async def process_batch(batch, grand_total, retry_mode):
    results = await asyncio.gather(*batch)
    newly_found = []
    for status, data in results:
        state['total_scraped'] += 1
        if status == 'found':
            state['total_found'] += 1
            state['found'].append(data)
        elif status == 'not_found':
            state['total_not_found'] += 1
            if not retry_mode:
                state['not_found_seats'].append(data)
        else:
            state['total_errors'] += 1
            add_log(f"⚠️ {data}")

    if not retry_mode and len(state['found']) // CHECKPOINT_EVERY > (len(state['found']) - len(newly_found)) // CHECKPOINT_EVERY:
        save_checkpoint()

    elapsed = time.time() - state['start_time']
    state['elapsed'] = str(timedelta(seconds=int(elapsed)))
    state['progress_pct'] = min(100, state['total_scraped'] / grand_total * 100)
    state['rate'] = state['total_scraped'] / elapsed if elapsed > 0 else 0
    if state['rate'] > 0:
        rem = (grand_total - state['total_scraped']) / state['rate']
        state['remaining'] = str(timedelta(seconds=int(rem)))
    else:
        state['remaining'] = '--:--:--'

    return {
        'progress': state['progress_pct'],
        'scraped': state['total_scraped'],
        'found': state['total_found'],
        'not_found': state['total_not_found'],
        'errors': state['total_errors'],
        'elapsed': state['elapsed'],
        'remaining': state['remaining'],
        'rate': f"{state['rate']:.1f}/ثانية",
        'stage': '🔄 تمرير أول' if not retry_mode else '🔁 إعادة غائبين',
    }

def run_scrape(start, end, progress=gr.Progress()):
    state['running'] = True
    state['stage'] = 'running'
    state['total_scraped'] = 0
    state['total_found'] = 0
    state['total_not_found'] = 0
    state['total_errors'] = 0
    state['current_seat'] = start

    async def runner():
        async for update in scrape_range(start, end):
            yield update

    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    gen = runner()
    try:
        while True:
            try:
                upd = loop.run_until_complete(gen.__anext__())
                progress(upd['progress'] / 100, desc=f"{upd['stage']} — {upd['scraped']:,}")
                yield upd
            except StopAsyncIteration:
                break
    finally:
        loop.close()
        state['running'] = False
        save_checkpoint()
        add_log("✅ السكريب اكتمل بالكامل")
        yield {
            'progress': 100, 'scraped': state['total_scraped'],
            'found': state['total_found'], 'not_found': state['total_not_found'],
            'errors': state['total_errors'], 'elapsed': state['elapsed'],
            'remaining': '00:00:00', 'rate': '0', 'stage': '✅ مكتمل'
        }

def stop_scrape():
    state['running'] = False
    add_log("⏹️ جاري إيقاف السكريب...")
    return "تم الإيقاف"

def reset_state():
    state['found'] = []
    state['not_found_seats'] = []
    state['total_scraped'] = 0
    state['total_found'] = 0
    state['total_not_found'] = 0
    state['total_errors'] = 0
    state['progress_pct'] = 0
    state['elapsed'] = '00:00:00'
    state['remaining'] = '--:--:--'
    state['rate'] = 0
    state['logs'] = []
    state['stage'] = 'idle'
    add_log("🔄 تم تصفير الحالة")
    return "تم التصفير"

ESTIMATED = """
⏱️ **تقدير الوقت المستغرق:**

| عدد الطلاب | زمن تقريبي |
|---|---|
| 10,000 | ~ 25 دقيقة |
| 50,000 | ~ 2 ساعة |
| 100,000 | ~ 4 ساعات |
| 500,000 | ~ 20 ساعة |
| 1,000,000 | ~ 40 ساعة (~1.7 يوم) |

بافتراض 10 اتصالات متزامنة، متوسط ~3 ثواني لكل طالب في أسوأ الأحوال.
"""

with gr.Blocks(title="سكريب نتائج الصف التاسع - الدقهلية", theme=gr.themes.Soft(primary_hue="indigo")) as demo:
    gr.Markdown("# 📊 سكريب نتائج الصف التاسع الأساسي - الدقهلية")
    gr.Markdown("استخراج بيانات الطلاب من 1 إلى 1,000,000 من موقع النتيجة")

    with gr.Row():
        with gr.Column(scale=2):
            start_box = gr.Number(label="بداية من رقم الجلوس", value=1, minimum=1, maximum=1_000_000, step=1)
            end_box = gr.Number(label="نهاية عند رقم الجلوس", value=1_000_000, minimum=1, maximum=1_000_000, step=1)

        with gr.Column(scale=1):
            gr.Markdown(ESTIMATED)

    with gr.Row():
        start_btn = gr.Button("▶️ بدء السكريب", variant="primary", size="lg")
        stop_btn = gr.Button("⏹️ إيقاف", variant="stop", size="lg")
        reset_btn = gr.Button("🔄 تصفير", size="lg")

    with gr.Row():
        with gr.Column(scale=2):
            progress_bar = gr.HTML(label="التقدم")
            stage_display = gr.Textbox(label="المرحلة", interactive=False)

        with gr.Column(scale=1):
            stats = gr.JSON(label="الإحصائيات", value={
                'تم السكريب': 0, 'موجود': 0, 'غير موجود': 0, 'أخطاء': 0,
                'الوقت المنقضي': '00:00:00', 'المتبقي': '--:--:--', 'المعدل': 0
            })

    log_display = gr.Textbox(label="سجل الأحداث", lines=15, max_lines=30, interactive=False)
    output_files = gr.File(label="ملفات الإخراج", visible=True)

    def format_progress(pct):
        filled = int(pct / 5)
        bar = "█" * filled + "░" * (20 - filled)
        return f"""
        <div style="background:#1f2937;border-radius:12px;padding:16px;text-align:center">
            <div style="font-size:2rem;font-weight:800;color:#818cf8">{pct:.1f}%</div>
            <div style="font-family:monospace;font-size:1.1rem;color:#e2e8f0;letter-spacing:2px">{bar}</div>
        </div>
        """

    def on_start(start, end):
        if start > end:
            return [
                format_progress(0), "⚠️ البداية أكبر من النهاية",
                {'error': 'تأكد من الأرقام'}, '', None
            ]
        if state['running']:
            return [
                format_progress(state['progress_pct']),
                "⚠️ السكريب يعمل بالفعل", stats.value, '', None
            ]
        state['running'] = True
        for upd in run_scrape(int(start), int(end)):
            stats_val = {
                'تم السكريب': upd['scraped'],
                'موجود': upd['found'],
                'غير موجود': upd['not_found'],
                'أخطاء': upd['errors'],
                'الوقت المنقضي': upd['elapsed'],
                'المتبقي': upd['remaining'],
                'المعدل': upd['rate'],
            }
            log_text = "\n".join(state['logs'][-30:])
            yield [
                format_progress(upd['progress']),
                f"{upd['stage']} — {upd['scraped']:,} / 1,000,000",
                stats_val,
                log_text,
                OUTPUT_DIR / "students_full.csv" if Path(OUTPUT_DIR / "students_full.csv").exists() else None
            ]

    start_btn.click(
        fn=on_start,
        inputs=[start_box, end_box],
        outputs=[progress_bar, stage_display, stats, log_display, output_files]
    )

    stop_btn.click(fn=stop_scrape, outputs=[stage_display])

    def on_reset():
        reset_state()
        return [
            format_progress(0), "🔄 تم التصفير - جاهز",
            {'تم السكريب': 0, 'موجود': 0, 'غير موجود': 0, 'أخطاء': 0,
             'الوقت المنقضي': '00:00:00', 'المتبقي': '--:--:--', 'المعدل': 0},
            "", None
        ]

    reset_btn.click(fn=on_reset, outputs=[progress_bar, stage_display, stats, log_display, output_files])

    iface = demo

if __name__ == "__main__":
    iface.launch(server_name="0.0.0.0", server_port=7860)