filename with url paths

app.py

@@ -10,6 +10,31 @@ import os
 import gradio as gr
 from collections import deque
 
+
+# ===========================================================
+# FORMATING FOR FILENAMES TO PRESERVE URL PATH
+# ===========================================================
+
+
+def url_to_zip_path(url, extension=".md"):
+    parsed = urlparse(url)
+
+    # example.com
+    domain = parsed.netloc.replace("www.", "")
+
+    # about/team/
+    path = parsed.path.strip("/")
+
+    if path == "":
+        # Homepage
+        return os.path.join(domain, "index" + extension)
+
+    # Remove extension if present
+    path = re.sub(r"\.[^.]+$", "", path)
+
+    return os.path.join(domain, path + extension)
+
+
 # ===========================================================
 # 🌐 WEBSITE CRAWLER
 # ===========================================================
@@ -126,7 +151,7 @@ def extract_all_content_as_zip(url: str, max_links: int, max_depth: int) -> Tupl
                     if title:
                         markdown_text = f"# {title.text.strip()}\n\n{markdown_text}"
 
-                    filename = f"page_{i}.md"
+                    filename = url_to_zip_path(link_url, ".md")
                     zip_file.writestr(filename, markdown_text)
                     html_ok += 1
 
@@ -137,7 +162,8 @@ def extract_all_content_as_zip(url: str, max_links: int, max_depth: int) -> Tupl
             for j, pdf_url in enumerate(pdf_links, 1):
                 try:
                     resp = session.get(pdf_url, timeout=20)
-                    zip_file.writestr(f"pdfs/document_{j}.pdf", resp.content)
+                    pdf_path = url_to_zip_path(pdf_url, ".pdf")
+                    zip_file.writestr(pdf_path, resp.content)
                     pdf_ok += 1
                 except Exception:
                     pass