Upload 3 files

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.14-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY app.py .
+
+RUN mkdir -p /app/output
+
+EXPOSE 7860
+
+CMD ["python", "app.py"]

app.py

@@ -0,0 +1,314 @@
+import asyncio
+import aiohttp
+import pandas as pd
+import json
+import os
+import time
+import math
+import sys
+from datetime import datetime, timedelta
+from pathlib import Path
+import gradio as gr
+
+API_URL = "https://natiga.edudk.net/P20262026/public/api_result.php"
+OUTPUT_DIR = Path("output")
+OUTPUT_DIR.mkdir(exist_ok=True)
+CONCURRENCY = 10
+CHECKPOINT_EVERY = 2000
+MAX_RETRIES = 3
+
+state = {
+    'running': False,
+    'stage': 'idle',
+    'found': [],
+    'not_found_seats': [],
+    'total_scraped': 0,
+    'total_found': 0,
+    'total_not_found': 0,
+    'total_errors': 0,
+    'current_seat': 0,
+    'start_time': None,
+    'elapsed': '00:00:00',
+    'remaining': '--:--:--',
+    'rate': 0,
+    'progress_pct': 0,
+    'logs': [],
+}
+
+def add_log(msg):
+    ts = datetime.now().strftime("%H:%M:%S")
+    state['logs'].append(f"[{ts}] {msg}")
+    if len(state['logs']) > 500:
+        state['logs'] = state['logs'][-500:]
+
+def save_checkpoint():
+    if state['found']:
+        df = pd.DataFrame(state['found'])
+        path = OUTPUT_DIR / "students_full.csv"
+        df.to_csv(path, index=False, encoding='utf-8-sig')
+        add_log(f"حفظ {len(state['found'])} طالب → students_full.csv")
+    if state['not_found_seats']:
+        pd.DataFrame({'seat_no': state['not_found_seats']}).to_csv(
+            OUTPUT_DIR / "not_found.csv", index=False, encoding='utf-8-sig')
+
+async def fetch_one(session, sem, seat):
+    for attempt in range(MAX_RETRIES + 2):
+        try:
+            async with sem:
+                async with session.get(f"{API_URL}?seat={seat}", timeout=aiohttp.ClientTimeout(total=12)) as resp:
+                    if resp.status == 429:
+                        wait = 5 * (attempt + 1)
+                        add_log(f"⏳ Rate-limit على المقعد {seat}، انتظار {wait}ث")
+                        await asyncio.sleep(wait)
+                        continue
+                    body = await resp.text()
+                    data = json.loads(body)
+                    if data.get('ok') and data.get('data'):
+                        return 'found', data['data']
+                    return 'not_found', seat
+        except (asyncio.TimeoutError, aiohttp.ClientError) as e:
+            if attempt < MAX_RETRIES + 1:
+                await asyncio.sleep(1.5 ** attempt)
+                continue
+            return 'error', f"{seat}: {str(e)[:60]}"
+        except json.JSONDecodeError:
+            return 'error', f"{seat}: bad json"
+    return 'not_found', seat
+
+async def scrape_range(start, end, retry_mode=False):
+    sem = asyncio.Semaphore(CONCURRENCY)
+    state['start_time'] = time.time()
+    total = end - start + 1
+    done = 0
+    connector = aiohttp.TCPConnector(limit=CONCURRENCY + 5, force_close=True)
+
+    async with aiohttp.ClientSession(connector=connector) as session:
+        batch = []
+        for seat in range(start, end + 1):
+            if not state['running']:
+                add_log("❌ تم إيقاف السكريب")
+                break
+            batch.append(fetch_one(session, sem, seat))
+            if len(batch) >= CONCURRENCY * 3:
+                yield await process_batch(batch, total, retry_mode)
+                done += len(batch)
+                batch = []
+        if batch:
+            yield await process_batch(batch, total, retry_mode)
+
+        if not retry_mode and state['running']:
+            add_log("🏁 انتهى الشوط الأول — بدء إعادة فحص الغائبين...")
+            save_checkpoint()
+            state['stage'] = 'retry'
+            nf = state['not_found_seats'][:]
+            if nf:
+                async for upd in scrape_range(min(nf), max(nf), retry_mode=True):
+                    yield upd
+
+async def process_batch(batch, grand_total, retry_mode):
+    results = await asyncio.gather(*batch)
+    newly_found = []
+    for status, data in results:
+        state['total_scraped'] += 1
+        if status == 'found':
+            state['total_found'] += 1
+            state['found'].append(data)
+        elif status == 'not_found':
+            state['total_not_found'] += 1
+            if not retry_mode:
+                state['not_found_seats'].append(data)
+        else:
+            state['total_errors'] += 1
+            add_log(f"⚠️ {data}")
+
+    if not retry_mode and len(state['found']) // CHECKPOINT_EVERY > (len(state['found']) - len(newly_found)) // CHECKPOINT_EVERY:
+        save_checkpoint()
+
+    elapsed = time.time() - state['start_time']
+    state['elapsed'] = str(timedelta(seconds=int(elapsed)))
+    state['progress_pct'] = min(100, state['total_scraped'] / grand_total * 100)
+    state['rate'] = state['total_scraped'] / elapsed if elapsed > 0 else 0
+    if state['rate'] > 0:
+        rem = (grand_total - state['total_scraped']) / state['rate']
+        state['remaining'] = str(timedelta(seconds=int(rem)))
+    else:
+        state['remaining'] = '--:--:--'
+
+    return {
+        'progress': state['progress_pct'],
+        'scraped': state['total_scraped'],
+        'found': state['total_found'],
+        'not_found': state['total_not_found'],
+        'errors': state['total_errors'],
+        'elapsed': state['elapsed'],
+        'remaining': state['remaining'],
+        'rate': f"{state['rate']:.1f}/ثانية",
+        'stage': '🔄 تمرير أول' if not retry_mode else '🔁 إعادة غائبين',
+    }
+
+def run_scrape(start, end, progress=gr.Progress()):
+    state['running'] = True
+    state['stage'] = 'running'
+    state['total_scraped'] = 0
+    state['total_found'] = 0
+    state['total_not_found'] = 0
+    state['total_errors'] = 0
+    state['current_seat'] = start
+
+    async def runner():
+        async for update in scrape_range(start, end):
+            yield update
+
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    gen = runner()
+    try:
+        while True:
+            try:
+                upd = loop.run_until_complete(gen.__anext__())
+                progress(upd['progress'] / 100, desc=f"{upd['stage']} — {upd['scraped']:,}")
+                yield upd
+            except StopAsyncIteration:
+                break
+    finally:
+        loop.close()
+        state['running'] = False
+        save_checkpoint()
+        add_log("✅ السكريب اكتمل بالكامل")
+        yield {
+            'progress': 100, 'scraped': state['total_scraped'],
+            'found': state['total_found'], 'not_found': state['total_not_found'],
+            'errors': state['total_errors'], 'elapsed': state['elapsed'],
+            'remaining': '00:00:00', 'rate': '0', 'stage': '✅ مكتمل'
+        }
+
+def stop_scrape():
+    state['running'] = False
+    add_log("⏹️ جاري إيقاف السكريب...")
+    return "تم الإيقاف"
+
+def reset_state():
+    state['found'] = []
+    state['not_found_seats'] = []
+    state['total_scraped'] = 0
+    state['total_found'] = 0
+    state['total_not_found'] = 0
+    state['total_errors'] = 0
+    state['progress_pct'] = 0
+    state['elapsed'] = '00:00:00'
+    state['remaining'] = '--:--:--'
+    state['rate'] = 0
+    state['logs'] = []
+    state['stage'] = 'idle'
+    add_log("🔄 تم تصفير الحالة")
+    return "تم التصفير"
+
+ESTIMATED = """
+⏱️ **تقدير الوقت المستغرق:**
+
+| عدد الطلاب | زمن تقريبي |
+|---|---|
+| 10,000 | ~ 25 دقيقة |
+| 50,000 | ~ 2 ساعة |
+| 100,000 | ~ 4 ساعات |
+| 500,000 | ~ 20 ساعة |
+| 1,000,000 | ~ 40 ساعة (~1.7 يوم) |
+
+بافتراض 10 اتصالات متزامنة، متوسط ~3 ثواني لكل طالب في أسوأ الأحوال.
+"""
+
+with gr.Blocks(title="سكريب نتائج الصف التاسع - الدقهلية", theme=gr.themes.Soft(primary_hue="indigo")) as demo:
+    gr.Markdown("# 📊 سكريب نتائج الصف التاسع الأساسي - الدقهلية")
+    gr.Markdown("استخراج بيانات الطلاب من 1 إلى 1,000,000 من موقع النتيجة")
+
+    with gr.Row():
+        with gr.Column(scale=2):
+            start_box = gr.Number(label="بداية من رقم الجلوس", value=1, minimum=1, maximum=1_000_000, step=1)
+            end_box = gr.Number(label="نهاية عند رقم الجلوس", value=1_000_000, minimum=1, maximum=1_000_000, step=1)
+
+        with gr.Column(scale=1):
+            gr.Markdown(ESTIMATED)
+
+    with gr.Row():
+        start_btn = gr.Button("▶️ بدء السكريب", variant="primary", size="lg")
+        stop_btn = gr.Button("⏹️ إيقاف", variant="stop", size="lg")
+        reset_btn = gr.Button("🔄 تصفير", size="lg")
+
+    with gr.Row():
+        with gr.Column(scale=2):
+            progress_bar = gr.HTML(label="التقدم")
+            stage_display = gr.Textbox(label="المرحلة", interactive=False)
+
+        with gr.Column(scale=1):
+            stats = gr.JSON(label="الإحصائيات", value={
+                'تم السكريب': 0, 'موجود': 0, 'غير موجود': 0, 'أخطاء': 0,
+                'الوقت المنقضي': '00:00:00', 'المتبقي': '--:--:--', 'المعدل': 0
+            })
+
+    log_display = gr.Textbox(label="سجل الأحداث", lines=15, max_lines=30, interactive=False)
+    output_files = gr.File(label="ملفات الإخراج", visible=True)
+
+    def format_progress(pct):
+        filled = int(pct / 5)
+        bar = "█" * filled + "░" * (20 - filled)
+        return f"""
+        <div style="background:#1f2937;border-radius:12px;padding:16px;text-align:center">
+            <div style="font-size:2rem;font-weight:800;color:#818cf8">{pct:.1f}%</div>
+            <div style="font-family:monospace;font-size:1.1rem;color:#e2e8f0;letter-spacing:2px">{bar}</div>
+        </div>
+        """
+
+    def on_start(start, end):
+        if start > end:
+            return [
+                format_progress(0), "⚠️ البداية أكبر من النهاية",
+                {'error': 'تأكد من الأرقام'}, '', None
+            ]
+        if state['running']:
+            return [
+                format_progress(state['progress_pct']),
+                "⚠️ السكريب يعمل بالفعل", stats.value, '', None
+            ]
+        state['running'] = True
+        for upd in run_scrape(int(start), int(end)):
+            stats_val = {
+                'تم السكريب': upd['scraped'],
+                'موجود': upd['found'],
+                'غير موجود': upd['not_found'],
+                'أخطاء': upd['errors'],
+                'الوقت المنقضي': upd['elapsed'],
+                'المتبقي': upd['remaining'],
+                'المعدل': upd['rate'],
+            }
+            log_text = "\n".join(state['logs'][-30:])
+            yield [
+                format_progress(upd['progress']),
+                f"{upd['stage']} — {upd['scraped']:,} / 1,000,000",
+                stats_val,
+                log_text,
+                OUTPUT_DIR / "students_full.csv" if Path(OUTPUT_DIR / "students_full.csv").exists() else None
+            ]
+
+    start_btn.click(
+        fn=on_start,
+        inputs=[start_box, end_box],
+        outputs=[progress_bar, stage_display, stats, log_display, output_files]
+    )
+
+    stop_btn.click(fn=stop_scrape, outputs=[stage_display])
+
+    def on_reset():
+        reset_state()
+        return [
+            format_progress(0), "🔄 تم التصفير - جاهز",
+            {'تم السكريب': 0, 'موجود': 0, 'غير موجود': 0, 'أخطاء': 0,
+             'الوقت المنقضي': '00:00:00', 'المتبقي': '--:--:--', 'المعدل': 0},
+            "", None
+        ]
+
+    reset_btn.click(fn=on_reset, outputs=[progress_bar, stage_display, stats, log_display, output_files])
+
+    iface = demo
+
+if __name__ == "__main__":
+    iface.launch(server_name="0.0.0.0", server_port=7860)

requirements.txt

@@ -0,0 +1,4 @@
+gradio==5.23.3
+aiohttp==3.11.16
+pandas==3.0.3
+openpyxl==3.1.5