Update bot.py

bot.py

@@ -607,9 +607,19 @@ class ImageToolbox:
             return False
 
 # ================================
-# 📥 9. Universal Downloader & Advanced Web Scraper
+# 📥 Universal Downloader (Drive, Gofile, Web & gdown Fallback)
 # ================================
 class DownloadManager:
+    @staticmethod
+    def get_drive_real_name(url):
+        try:
+            r = requests.get(url, timeout=10)
+            soup = BeautifulSoup(r.text, 'html.parser')
+            title = soup.title.string
+            if title: return title.replace(" - Google Drive", "").strip()
+            return "Drive_Content"
+        except: return "Drive_Content"
+
     @staticmethod
     def extract_archives(folder):
         for root, _, files in os.walk(folder):
@@ -630,8 +640,7 @@ class DownloadManager:
                     except: pass
 
     @staticmethod
-    def scrape_web_selenium(url, output_folder):
-        # 🚀 تحديث جذري لسحب مواقع الويب وحل مشكلة صورتين فقط (Lazy Loading Fix)
+    def scrape_drive_selenium(url, output_folder):
         driver = None
         try:
             chrome_options = Options()
@@ -645,70 +654,163 @@ class DownloadManager:
             driver.get(url)
             time.sleep(5)
             
-            # النزول البطيء والمستمر لتحفيز تحميل الصور
-            last_h = driver.execute_script("return document.body.scrollHeight")
-            for _ in range(60): # محاولة النزول 60 مرة
-                driver.execute_script("window.scrollBy(0, 1000);")
-                time.sleep(1.2) # انتظار كافٍ لرد السيرفر
-                new_h = driver.execute_script("return document.body.scrollHeight")
-                if new_h == last_h:
-                    time.sleep(3) # محاولة أخيرة لو توقف النزول
-                    new_h = driver.execute_script("return document.body.scrollHeight")
-                    if new_h == last_h: break
-                last_h = new_h
+            page_title = driver.title.replace(" - Google Drive", "").strip()
+            if not page_title or page_title == "Google Drive": page_title = "Drive_Folder"
+
+            last_height = driver.execute_script("return document.body.scrollHeight")
+            for _ in range(15):
+                driver.execute_script("window.scrollBy(0, 1500);")
+                time.sleep(1)
+                new_height = driver.execute_script("return document.body.scrollHeight")
+                if new_height == last_height: break
+                last_height = new_height
             
-            time.sleep(3)
+            time.sleep(2)
             
-            # استخراج الصور بذكاء (تجاهل الأيقونات والبنرات)
-            elems = driver.find_elements(By.TAG_NAME, 'img')
-            urls = []
-            for e in elems:
-                src = e.get_attribute('src') or e.get_attribute('data-src') or e.get_attribute('data-original')
-                if src and src.startswith('http') and not any(x in src.lower() for x in ['logo','ad','banner','icon','avatar', 'profile']):
-                    urls.append(src)
+            elements = driver.find_elements(By.XPATH, "//*[@data-id]")
+            file_ids = []
+            for el in elements:
+                fid = el.get_attribute("data-id")
+                if fid and len(fid) > 20 and fid not in file_ids:
+                    file_ids.append(fid)
             
-            # إزالة التكرار مع الحفاظ على الترتيب الأصلي
-            urls = list(dict.fromkeys(urls))
+            if not file_ids:
+                page_source = driver.page_source
+                found = re.findall(r'\["([^"]{25,})",', page_source)
+                file_ids = list(set([fid for fid in found if len(fid) > 28]))
+
+            if not file_ids:
+                driver.quit()
+                return False, "Error"
+
+            saved = False
+            session = requests.Session()
+            session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"})
             
-            saved_paths = []
-            req_headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}
+            for idx, fid in enumerate(file_ids):
+                try:
+                    dl_url = f"https://drive.google.com/uc?export=download&id={fid}"
+                    r = session.get(dl_url, stream=True, timeout=15)
+                    if r.status_code == 200 and len(r.content) > 10000:
+                        with open(os.path.join(output_folder, f"drive_img_{idx:03d}.jpg"), 'wb') as f:
+                            f.write(r.content)
+                        saved = True
+                except: pass
             
+            driver.quit()
+            return saved, page_title
+        except Exception as e:
+            if driver: driver.quit()
+            return False, "Error"
+
+    @staticmethod
+    async def scrape_drive(url, output_folder):
+        real_name = DownloadManager.get_drive_real_name(url)
+        # 1. المحاولة الأساسية باستخدام المتصفح الخفي (Selenium)
+        success, name = await asyncio.to_thread(DownloadManager.scrape_drive_selenium, url, output_folder)
+        
+        # 2. خط الدفاع الأخير (Fallback) باستخدام gdown لو فشل المتصفح
+        if not success:
+            try:
+                print("⚠️ فشل السحب باستخدام Selenium، جاري التحويل إلى خطة الطوارئ (gdown)...")
+                if "folder" in url or "drive.google.com/drive/folders" in url:
+                    res = await asyncio.to_thread(gdown.download_folder, url, output=output_folder, quiet=False, use_cookies=False)
+                    if res: success = True
+                else:
+                    res = await asyncio.to_thread(gdown.download, url, output=os.path.join(output_folder, "gdown_fallback.zip"), quiet=False, fuzzy=True)
+                    if res: success = True
+                
+                if success: name = real_name
+            except Exception as e:
+                print(f"❌ فشلت خطة الطوارئ gdown أيضاً: {e}")
+                
+        return success, name
+
+    @staticmethod
+    def scrape_gofile_selenium(url, output_folder):
+        driver = None
+        try:
+            chrome_options = Options()
+            chrome_options.add_argument('--headless=new')
+            chrome_options.add_argument('--no-sandbox')
+            chrome_options.add_argument('--disable-dev-shm-usage')
+            service = Service(ChromeDriverManager().install())
+            driver = webdriver.Chrome(service=service, options=chrome_options)
+            driver.get(url)
+            WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.CSS_SELECTOR, ".contentName, .downloadButton")))
+            time.sleep(3)
+            try: page_title = driver.find_element(By.CSS_SELECTOR, ".contentName").text
+            except: page_title = "Gofile_Content"
+            links = driver.find_elements(By.TAG_NAME, 'a')
+            download_urls = [link.get_attribute('href') for link in links if link.get_attribute('href') and 'download' in link.get_attribute('href').lower()]
+            cookies = driver.get_cookies()
+            session = requests.Session()
+            for cookie in cookies: session.cookies.set(cookie['name'], cookie['value'])
+            saved = False
+            for idx, d_url in enumerate(list(set(download_urls))):
+                r = session.get(d_url, stream=True)
+                if r.status_code == 200:
+                    cd = r.headers.get('content-disposition', '')
+                    fname = re.findall("filename=(.+)", cd)
+                    name = fname[0].strip('"') if fname else f"file_{idx}.zip"
+                    with open(os.path.join(output_folder, name), 'wb') as f: f.write(r.content)
+                    saved = True
+            driver.quit()
+            return saved, page_title
+        except:
+            if driver: driver.quit()
+            return False, "Error"
+
+    @staticmethod
+    def scrape_web_selenium(url, output_folder):
+        driver = None
+        try:
+            chrome_options = Options()
+            chrome_options.add_argument('--headless=new')
+            chrome_options.add_argument('--no-sandbox')
+            chrome_options.add_argument('--disable-dev-shm-usage')
+            service = Service(ChromeDriverManager().install())
+            driver = webdriver.Chrome(service=service, options=chrome_options)
+            driver.get(url)
+            time.sleep(5)
+            last_h = driver.execute_script("return document.body.scrollHeight")
+            for _ in range(30):
+                driver.execute_script("window.scrollBy(0, 800);")
+                time.sleep(0.5)
+                new_h = driver.execute_script("return window.pageYOffset + window.innerHeight")
+                if new_h >= last_h: break
+                last_h = driver.execute_script("return document.body.scrollHeight")
+            time.sleep(2)
+            elems = driver.find_elements(By.TAG_NAME, 'img')
+            urls = [e.get_attribute('src') or e.get_attribute('data-src') for e in elems]
+            urls = list(set([u for u in urls if u and u.startswith('http') and not any(x in u.lower() for x in ['logo','ad','banner','icon'])]))
+            saved_paths = []
+            req_headers = {'User-Agent': 'Mozilla/5.0'}
             for idx, u in enumerate(urls):
                 try:
-                    c = requests.get(u, headers=req_headers, timeout=15).content
+                    c = requests.get(u, headers=req_headers, timeout=10).content
                     im = Image.open(io.BytesIO(c)).convert('RGB')
-                    # التأكد من أن الصورة هي صفحة مانهوا فعلية (أبعاد كبيرة) وليست أيقونة صغيرة
-                    if im.width > 300 and im.height > 300:
-                        path = os.path.join(output_folder, f"page_{idx:03d}.jpg")
+                    if im.width > 150 and im.height > 150:
+                        path = os.path.join(output_folder, f"{idx:03d}.jpg")
                         im.save(path)
                         saved_paths.append(path)
                 except: pass
-                
             title = driver.title.split('|')[0].strip() if driver.title else "Web_Chapter"
             title = re.sub(r'[\\/*?:"<>|]', "", title)
             driver.quit()
             return saved_paths, title
         except Exception as e:
             if driver: driver.quit()
-            print(f"Scraper Error: {e}")
             return [], "Error"
 
     @staticmethod
     async def download(url, output_folder):
         success = False
         name = "Downloaded_Content"
-        
-        # التوجه مباشرة لخطة الطوارئ gdown لتحميل الملفات
         if "drive.google.com" in url:
-            try:
-                if "folder" in url or "drive.google.com/drive/folders" in url:
-                    res = await asyncio.to_thread(gdown.download_folder, url, output=output_folder, quiet=False, use_cookies=False)
-                    if res: success = True
-                else:
-                    res = await asyncio.to_thread(gdown.download, url, output=os.path.join(output_folder, "gdown_fallback.zip"), quiet=False, fuzzy=True)
-                    if res: success = True
-                if success: name = "Drive_Content"
-            except Exception as e: print(f"Drive Error: {e}")
+            success, name = await DownloadManager.scrape_drive(url, output_folder)
+        elif "gofile.io" in url:
+            success, name = await asyncio.to_thread(DownloadManager.scrape_gofile_selenium, url, output_folder)
         else:
             try:
                 async with aiohttp.ClientSession() as s:
@@ -716,11 +818,10 @@ class DownloadManager:
                         if r.status == 200:
                             cd = r.headers.get('Content-Disposition', '')
                             if 'filename=' in cd: name = re.findall("filename=(.+)", cd)[0].strip('"')
-                            else: name = "file.zip"
+                            else: name = "file.bin"
                             with open(os.path.join(output_folder, name), 'wb') as f: f.write(await r.read())
                             success = True
             except: pass
-            
         DownloadManager.extract_archives(output_folder)
         has_images = any(f.lower().endswith(('.png','.jpg','.jpeg','.webp')) for r, d, files in os.walk(output_folder) for f in files)
         if success and has_images: return True, name
@@ -732,13 +833,12 @@ class DownloadManager:
         subdirs = [d for d in os.listdir(folder) if os.path.isdir(os.path.join(folder, d))]
         if not subdirs:
             imgs = [os.path.join(folder, f) for f in os.listdir(folder) if f.lower().endswith(('.png','.jpg','.jpeg','.webp'))]
-            # هنا نستخدم دالة natural_sort_key التي وضعناها في الجزء الأول لترتيب الصور بشكل بشري دقيق
-            if imgs: chapters["Main"] = sorted(imgs, key=natural_sort_key)
+            if imgs: chapters["Main"] = sorted(imgs)
         else:
-            for d in sorted(subdirs, key=natural_sort_key):
+            for d in sorted(subdirs):
                 d_path = os.path.join(folder, d)
                 imgs = [os.path.join(d_path, f) for f in os.listdir(d_path) if f.lower().endswith(('.png','.jpg','.jpeg','.webp'))]
-                if imgs: chapters[d] = sorted(imgs, key=natural_sort_key)
+                if imgs: chapters[d] = sorted(imgs)
         return chapters
 
 # ================================