Update scrape_3gpp.py

scrape_3gpp.py

@@ -62,12 +62,18 @@ def extract_statuses(url):
                 return []
 
 
+import os
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import gradio as gr
+
 def scrape(url, excel_file, folder_name, status_list, progress=gr.Progress()):
     filenames = []
     status_filenames = []
     df = pd.DataFrame()  # Initialize df to ensure it's always defined
 
-    # Only proceed if excel_file is not None and it exists
+    # Try to process the Excel file if provided and valid
     if excel_file and os.path.exists(excel_file):
         try:
             df = pd.read_excel(excel_file)
@@ -76,25 +82,27 @@ def scrape(url, excel_file, folder_name, status_list, progress=gr.Progress()):
             if 'TDoc Status' in df.columns and status_list:
                 df = df[df['TDoc Status'].isin(status_list)]
                 print(f"Filtered DataFrame size: {len(df)}")
-            else:
-                print("No filtering applied based on TDoc Status")
 
             if not df.empty:
                 if 'TDoc' in df.columns and not df['TDoc'].isnull().all():
                     status_filenames = [f"{url}{row['TDoc']}.zip" for index, row in df.iterrows()]
                 elif 'URL' in df.columns and not df['URL'].isnull().all():
                     status_filenames = df['URL'].tolist()
-                else:
-                    print("No valid 'TDoc' or 'URL' entries found.")
-
-                print(f"Filenames: {status_filenames}")
-            else:
-                print("DataFrame is empty after filtering.")
 
+                print(f"Filenames from Excel: {status_filenames}")
         except Exception as e:
             print(f"Error reading Excel file: {e}")
-    else:
-        print("No valid excel_file path provided.")
+
+    # If no valid Excel file is given or no status_filenames are found, download zip files directly from the URL
+    if not excel_file or not status_filenames:
+        print("Downloading zip files directly from the URL...")
+        response = requests.get(url)
+        soup = BeautifulSoup(response.content, 'html.parser')
+        zip_links = [a['href'] for a in soup.find_all('a', href=True) if a['href'].endswith('.zip')]
+
+        # Construct absolute URLs for zip files
+        status_filenames = [url + link if not link.startswith('http') else link for link in zip_links]
+        print(f"Filenames from URL: {status_filenames}")
 
     download_directory = folder_name
     if not os.path.exists(download_directory):
@@ -102,16 +110,12 @@ def scrape(url, excel_file, folder_name, status_list, progress=gr.Progress()):
 
     pourcentss = 0.05
 
-    if not status_filenames:
-        print("No Excel file provided, or no valid URLs found in the file.")
-        return False, 0
-
-    # Proceed with downloading files using the filenames list
+    # Proceed with downloading files
     for file_url in status_filenames:
         filename = os.path.basename(file_url)
         save_path = os.path.join(download_directory, filename)
         progress(pourcentss, desc='Downloading')
-        pourcentss += 0.4 / len(status_filenames) if status_filenames else 1  # Adjust to prevent division by zero
+        pourcentss += 0.4 / max(len(status_filenames), 1)  # Ensure non-zero division
         try:
             with requests.get(file_url, stream=True) as r:
                 r.raise_for_status()
@@ -119,7 +123,7 @@ def scrape(url, excel_file, folder_name, status_list, progress=gr.Progress()):
                     for chunk in r.iter_content(chunk_size=8192):
                         f.write(chunk)
         except requests.exceptions.HTTPError as e:
-            print(f"HTTP error occurred: {file_url}: {e}")
+            print(f"HTTP error occurred while downloading {file_url}: {e}")
 
     return True, len(status_filenames)
 
@@ -127,6 +131,7 @@ def scrape(url, excel_file, folder_name, status_list, progress=gr.Progress()):
 
 
 
+
 def extractZip(url):
     # Répertoire où les fichiers zip sont déjà téléchargés
     nom_extract = url.split("/")[-3] + "_extraction"