Duplicate from dwancin/web-scraping

Co-authored-by: DW <dwancin@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Web Scraping
+emoji: 🕵️
+colorFrom: red
+colorTo: red
+sdk: gradio
+sdk_version: 3.35.2
+app_file: app.py
+pinned: false
+duplicated_from: dwancin/web-scraping
+---
+
+https://huggingface.co/spaces/dwancin/web-scraping

app.py

@@ -0,0 +1,160 @@
+import os
+import re
+import requests
+import uuid
+import zipfile
+import hashlib
+import shutil
+import gradio as gr
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+
+# Function to validate URLs
+def is_valid(url):
+    parsed = urlparse(url)
+    return bool(parsed.netloc) and bool(parsed.scheme)
+
+# Function to find files on webpage
+def find_files(url, soup, file_type):
+    files = []
+    if file_type == "image":
+        tags = ['jpg', 'jpeg', 'png', 'svg', 'gif']
+        for tag in soup.find_all('img'):
+            file = tag.get('src')
+            if any(tag in file for tag in tags):
+                file_url = file
+                if not is_valid(file_url):
+                    file_url = urljoin(url, file_url)
+                files.append(file_url)
+    elif file_type == "text":
+        text_tags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'span', 'strong']
+        for tag in text_tags:
+            for element in soup.find_all(tag):
+                files.append(element.get_text())
+    else:
+        for link in soup.find_all('a'):
+            file = link.get('href')
+            if file_type in file:
+                file_url = file
+                if not is_valid(file_url):
+                    file_url = urljoin(url, file_url)
+                files.append(file_url)
+    return files
+
+
+
+
+# Function to download files
+def download_files(urls, folder_name):
+    os.makedirs(folder_name, exist_ok=True)
+    for i, url in enumerate(urls):
+        response = requests.get(url, stream=True)
+        file_extension = url.split(".")[-1].split("&")[0]
+        url_hash = hashlib.md5(url.encode()).hexdigest()
+        unique_id = str(uuid.uuid4())[:8]
+        file_name = f'{url_hash}-{unique_id}.{file_extension}'
+        file_name = file_name[:255]  # Truncate the file name to avoid exceeding the limit
+        file_name = re.sub(r'[\\/:"*?<>|]+', '_', file_name)  # Replace special characters with underscores
+        with open(f'{folder_name}/{file_name}', 'wb') as out_file:
+            out_file.write(response.content)
+        print(f"Downloaded file: {file_name}")
+
+# Function to create zip file
+def create_zip_file(folder_name):
+    # Only create zip file if there are files in the directory
+    if os.listdir(folder_name):
+        with zipfile.ZipFile(f'{folder_name}.zip', 'w') as zipf:
+            for file in os.listdir(folder_name):
+                zipf.write(f'{folder_name}/{file}')
+        return f'{folder_name}.zip'
+    else:
+        return ""
+
+
+    
+# Function to scrape website
+def scrape_website(url, images=False, text=False):
+    try:
+        response = requests.get(url, timeout=10)
+        response.raise_for_status()  # Raise an exception if the GET request was unsuccessful
+    except (requests.exceptions.RequestException, ValueError):
+        raise gr.Error(f"Unable to access URL: {url}")
+        return None, None
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+
+    # Clear the contents of the folders
+    if images:
+        shutil.rmtree('images', ignore_errors=True)
+    if text:
+        shutil.rmtree('text', ignore_errors=True)
+
+    # Download files
+    if images:
+        image_urls = find_files(url, soup, 'image')
+        download_files(image_urls, 'images')
+    if text:
+        text_content = find_files(url, soup, 'text')
+        os.makedirs('text', exist_ok=True)  # Make sure the directory exists before writing
+        if text_content:  # Only create the file if there is text to write
+            with open('text/content.txt', 'w') as text_file:
+                for line in text_content:
+                    text_file.write(line + '\n')
+
+    # Create zip files and return paths
+    images_zip_file, text_zip_file = None, None
+    if images and os.path.exists('images') and os.listdir('images'):
+        images_zip_file = create_zip_file('images')
+    if text and os.path.exists('text') and os.listdir('text'):
+        text_zip_file = create_zip_file('text')
+
+    return images_zip_file, text_zip_file
+
+
+
+# Function for web scraping
+def web_scraping(url, file_types):
+    # Check if the URL is empty
+    if not url:
+        raise gr.Error("URL cannot be empty.")
+
+    # Check if the URL begins with https://
+    if not url.startswith("https://"):
+        raise gr.Error("The URL must begin with https://")
+
+    # Check if at least one checkbox is selected
+    if not file_types:
+        raise gr.Error("At least one media type must be selected.")
+
+    images = "Images" in file_types
+    text = "Text" in file_types
+    return scrape_website(url, images, text)
+
+with gr.Blocks(theme="dwancin/yellow", css=".lg.svelte-1ipelgc {max-height: 60px !important;}") as app:
+    with gr.Row():
+        with gr.Column(scale=2):
+            url_name = gr.Textbox(
+                placeholder="Enter URL here",
+                show_label=True,
+                label="Website",
+                info="Example: https://en.wikipedia.org/wiki/Main_Page",
+            )
+            media_types = gr.CheckboxGroup(
+                [
+                    "Images", 
+                    "Text",
+                ], 
+                value="Images",
+                label="Media types",
+            )
+            submit_button = gr.Button(
+                "Scrape",
+                variant="primary",
+                interactive=True,
+            )
+        with gr.Column(scale=1): 
+            output_images_zip_file = gr.File(label="Images ZIP-file")
+            output_text_zip_file = gr.File(label="Text ZIP-file")
+
+    submit_button.click(web_scraping, inputs=[url_name, media_types], outputs=[output_images_zip_file, output_text_zip_file])
+app.launch()

requirements.txt

@@ -0,0 +1,11 @@
+gradio>=3.35.2
+beautifulsoup4>=4.12.2
+Pillow>=9.5.0
+requests>=2.31.0
+Flask>=2.3.2
+zipfile2>= 0.0.12
+urllib3>=2.0.3
+pytest-shutil>=1.7.0
+mime>=0.1.0
+mimetypes-extensions>=0.1.0
+uuid>=1.30