Duplicate from awacke1/GetAllContent

Co-authored-by: Aaron C Wacker <awacke1@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: GetAllContent
+emoji: 📚
+colorFrom: red
+colorTo: indigo
+sdk: streamlit
+sdk_version: 1.21.0
+app_file: app.py
+pinned: false
+license: mit
+duplicated_from: awacke1/GetAllContent
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,64 @@
+import streamlit as st
+import requests
+from bs4 import BeautifulSoup
+import os
+import urllib
+import base64
+
+EXCLUDED_FILES = ['app.py', 'requirements.txt', 'pre-requirements.txt', 'packages.txt', 'README.md','.gitattributes', "backup.py","Dockerfile"]
+
+def download_file(url, local_filename):
+    if url.startswith('http://') or url.startswith('https://'):
+        try:
+            with requests.get(url, stream=True) as r:
+                r.raise_for_status()
+                with open(local_filename, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=8192): 
+                        f.write(chunk)
+            return local_filename
+        except requests.exceptions.HTTPError as err:
+            print(f"HTTP error occurred: {err}")
+
+def download_html_and_files(url):
+    html_content = requests.get(url).text
+    soup = BeautifulSoup(html_content, 'html.parser')
+
+    base_url = urllib.parse.urlunparse(urllib.parse.urlparse(url)._replace(path='', params='', query='', fragment=''))
+
+    for link in soup.find_all('a'):
+        file_url = urllib.parse.urljoin(base_url, link.get('href'))
+        local_filename = urllib.parse.urlparse(file_url).path.split('/')[-1]
+        if local_filename:
+            link['href'] = local_filename
+            download_file(file_url, local_filename)
+    
+    with open("index.html", "w") as file:
+        file.write(str(soup))
+
+def list_files(directory_path='.'):
+    files = [f for f in os.listdir(directory_path) if os.path.isfile(os.path.join(directory_path, f))]
+    return [f for f in files if f not in EXCLUDED_FILES]
+
+def get_download_link(file):
+    with open(file, "rb") as f:
+        bytes = f.read()
+        b64 = base64.b64encode(bytes).decode()
+        href = f'<a href="data:file/octet-stream;base64,{b64}" download=\'{file}\'>Click to download {file}</a>'
+    return href
+
+def show_download_links():
+    st.sidebar.write('Here are the files you can download:')
+    for file in list_files():
+        st.sidebar.markdown(get_download_link(file), unsafe_allow_html=True)
+
+def main():
+    st.sidebar.title('Bulk Download Tool')
+    url = st.sidebar.text_input('Please enter a URL to bulk download text and files')
+    if st.sidebar.button('📥 Get All the Content'):
+        download_html_and_files(url)
+        show_download_links()
+    if st.sidebar.button('📂 Show Download Links'):
+        show_download_links()
+
+if __name__ == "__main__":
+    main()

backup.py

@@ -0,0 +1,58 @@
+import streamlit as st
+import requests
+from bs4 import BeautifulSoup
+import os
+import urllib
+import base64
+
+def download_file(url, local_filename):
+    if url.startswith('http://') or url.startswith('https://'):  # add this line
+        try:
+            with requests.get(url, stream=True) as r:
+                r.raise_for_status()
+                with open(local_filename, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=8192): 
+                        f.write(chunk)
+            return local_filename
+        except requests.exceptions.HTTPError as err:
+            print(f"HTTP error occurred: {err}")  # or use logging
+
+
+def download_html_and_files(url):
+    html_content = requests.get(url).text
+    soup = BeautifulSoup(html_content, 'html.parser')
+
+    base_url = urllib.parse.urlunparse(urllib.parse.urlparse(url)._replace(path='', params='', query='', fragment=''))
+
+    for link in soup.find_all('a'):
+        file_url = urllib.parse.urljoin(base_url, link.get('href'))
+        local_filename = urllib.parse.urlparse(file_url).path.split('/')[-1]
+        if local_filename:  # add this line
+            link['href'] = local_filename
+            download_file(file_url, local_filename)
+    
+    with open("index.html", "w") as file:
+        file.write(str(soup))
+
+        
+def list_files(directory_path='.'):
+    return [f for f in os.listdir(directory_path) if os.path.isfile(os.path.join(directory_path, f))]
+
+def main():
+    st.sidebar.title('Bulk Download Tool')
+    url = st.sidebar.text_input('Please enter a URL to bulk download text and files')
+    if st.sidebar.button('📥 Get All the Content'):
+        download_html_and_files(url)
+        st.sidebar.write('Download complete. Here are the files you can download:')
+        for file in list_files():
+            st.sidebar.markdown(get_download_link(file), unsafe_allow_html=True)
+
+def get_download_link(file):
+    with open(file, "rb") as f:
+        bytes = f.read()
+        b64 = base64.b64encode(bytes).decode()
+        href = f'<a href="data:file/octet-stream;base64,{b64}" download=\'{file}\'>Click to download {file}</a>'
+    return href
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,4 @@
+streamlit
+requests
+beautifulsoup4
+urllib3