Update scrape_3gpp.py

scrape_3gpp.py

@@ -9,6 +9,31 @@ import textract
 import gradio as gr
 
 
+def count_links(url):
+    # Define common file extensions for downloadable content
+    file_extensions = ('.zip')
+    
+    try:
+        # Send a HTTP request to the URL
+        response = requests.get(url)
+        response.raise_for_status()  # Raise an exception for HTTP errors
+        
+        # Parse the HTML content of the page
+        soup = BeautifulSoup(response.text, 'html.parser')
+        
+        # Find all <a> tags in the HTML
+        links = soup.find_all('a')
+        
+        # Count the number of links that point to downloadable files
+        count = sum(1 for link in links if any(link.get('href', '').endswith(ext) for ext in file_extensions))
+        
+        return count
+    except requests.RequestException as e:
+        print(f"Error fetching the page: {e}")
+        return None
+
+
+
 def browse_folder(url):
     if url.lower().endswith(('docs', 'docs/')):
       return gr.update(choices=[])
@@ -62,7 +87,7 @@ def extract_statuses(url):
                 return []
 
 
-def scrape(url, excel_file, folder_name, status_list, progress=gr.Progress()):
+def scrape(url, excel_file, folder_name, status_list,count, progress=gr.Progress()):
     filenames = []
     status_filenames = []
     # Check if the excel_file argument is provided and if the file exists.
@@ -116,8 +141,8 @@ def scrape(url, excel_file, folder_name, status_list, progress=gr.Progress()):
     download_directory = folder_name
     if not os.path.exists(download_directory):
         os.makedirs(download_directory)
-    download_num = 0
-    pourcentss = 0.1
+    
+    pourcentss = 0.05
     print(f'filenames: {status_filenames}')
     if not filenames and not status_filenames:
         print("No Excel file provided, or no valid URLs found in the file.")
@@ -135,11 +160,8 @@ def scrape(url, excel_file, folder_name, status_list, progress=gr.Progress()):
         
         # Télécharger chaque fichier zip
         for zip_link in zip_links:
-            if download_num%10 == 0:
-              pourcentss = pourcentss + download_num/500
-              progress(pourcentss,desc='Telechargement')
-              download_num = 0
-            download_num+=1
+            progress(pourcentss,desc='Downloading')
+            pourcentss+=0.4/count
             # Construire l'URL absolue du fichier zip
             absolute_url = urljoin(url, zip_link)
 
@@ -161,11 +183,8 @@ def scrape(url, excel_file, folder_name, status_list, progress=gr.Progress()):
       for file_url in status_filenames:
           filename = os.path.basename(file_url)
           save_path = os.path.join(download_directory, filename)
-          if download_num%10 == 0:
-              pourcentss = pourcentss + download_num/500
-              progress(pourcentss,desc='Telechargement')
-              download_num = 0
-          download_num+=1
+          progress(pourcentss,desc='Downloading')
+          pourcentss+=0.4/count
           try:
               with requests.get(file_url, stream=True) as r:
                   r.raise_for_status()
@@ -283,7 +302,8 @@ def update_excel(data, excel_file, url):
 def extractionPrincipale(url, excel_file=None, status_list=None, progress=gr.Progress()):
     folder_name = 'nom provisoire'
     temp_excel = url.split("/")[-2] + "_status.xlsx"
-    progress(0.0,desc='Telechargement')
+    progress(0.0,desc='Downloading')
+    count = count_links(url)
     result, message = scrape(url, excel_file, folder_name, status_list)
     if result:
         print("Success:", message)
@@ -294,7 +314,7 @@ def extractionPrincipale(url, excel_file=None, status_list=None, progress=gr.Pro
     extractZip(folder_name)
     progress(0.5,desc='Extraction 2')
     excel3gpp(url)
-    progress(0.6,desc='Mise en forme Excel')
+    progress(0.6,desc='Creating Excel File')
 
 
     extract_directory = folder_name +" extraction"
@@ -311,7 +331,7 @@ def extractionPrincipale(url, excel_file=None, status_list=None, progress=gr.Pro
         "pptx": ["URL", "File", "Type", "Title", "Source", "Content"]
     }
 
-    num=0.6
+    pourcents2=0.6
     data = []
     errors_count = 0
     processed_count = 0   # Counter for processed files
@@ -328,8 +348,8 @@ def extractionPrincipale(url, excel_file=None, status_list=None, progress=gr.Pro
         folder_path = os.path.join(extract_directory, folder)
         if os.path.isdir(folder_path):
             for file in os.listdir(folder_path):
-                num = min(num + 0.001, 0.9)
-                progress(num,desc='Mise en forme Excel')
+                progress(pourcents2,desc='Creating Excel File')
+                pourcents2+=0.4/count
 
 
                 if file == "__MACOSX":