| | import streamlit as st |
| | import requests |
| | import base64 |
| | import os |
| | import asyncio |
| | from huggingface_hub import HfApi, snapshot_download |
| | import plotly.express as px |
| | import zipfile |
| | import tempfile |
| | import shutil |
| |
|
| | |
| | api = HfApi() |
| |
|
| | |
| | HTML_DIR = "generated_html_pages" |
| | ZIP_DIR = "generated_zips" |
| | SNAPSHOT_DIR = "snapshot_downloads" |
| |
|
| | for directory in [HTML_DIR, ZIP_DIR, SNAPSHOT_DIR]: |
| | if not os.path.exists(directory): |
| | os.makedirs(directory) |
| |
|
| | |
| | default_users = { |
| | "users": [ |
| | "awacke1", "rogerxavier", "jonatasgrosman", "kenshinn", "Csplk", "DavidVivancos", |
| | "cdminix", "Jaward", "TuringsSolutions", "Severian", "Wauplin", |
| | "phosseini", "Malikeh1375", "gokaygokay", "MoritzLaurer", "mrm8488", |
| | "TheBloke", "lhoestq", "xw-eric", "Paul", "Muennighoff", |
| | "ccdv", "haonan-li", "chansung", "lukaemon", "hails", |
| | "pharmapsychotic", "KingNish", "merve", "ameerazam08", "ashleykleynhans" |
| | ] |
| | } |
| |
|
| | async def fetch_user_content(username): |
| | try: |
| | models = list(await asyncio.to_thread(api.list_models, author=username)) |
| | datasets = list(await asyncio.to_thread(api.list_datasets, author=username)) |
| | return { |
| | "username": username, |
| | "models": models, |
| | "datasets": datasets |
| | } |
| | except Exception as e: |
| | return {"username": username, "error": str(e)} |
| |
|
| | def download_user_page(username): |
| | url = f"https://huggingface.co/{username}" |
| | try: |
| | response = requests.get(url) |
| | response.raise_for_status() |
| | html_content = response.text |
| | html_file_path = os.path.join(HTML_DIR, f"{username}.html") |
| | with open(html_file_path, "w", encoding='utf-8') as html_file: |
| | html_file.write(html_content) |
| | return html_file_path, None |
| | except Exception as e: |
| | return None, str(e) |
| |
|
| | @st.cache_resource |
| | def create_zip_of_files(files, zip_name): |
| | zip_file_path = os.path.join(ZIP_DIR, zip_name) |
| | with zipfile.ZipFile(zip_file_path, 'w') as zipf: |
| | for file in files: |
| | zipf.write(file, arcname=os.path.basename(file)) |
| | return zip_file_path |
| |
|
| | @st.cache_resource |
| | def get_download_link(file_path, link_text): |
| | with open(file_path, 'rb') as f: |
| | data = f.read() |
| | b64 = base64.b64encode(data).decode() |
| | return f'<a href="data:application/zip;base64,{b64}" download="{os.path.basename(file_path)}">{link_text}</a>' |
| |
|
| | async def fetch_all_users(usernames): |
| | tasks = [fetch_user_content(username) for username in usernames] |
| | return await asyncio.gather(*tasks) |
| |
|
| | def get_all_html_files(usernames): |
| | html_files = [] |
| | errors = {} |
| | for username in usernames: |
| | html_file, error = download_user_page(username) |
| | if html_file: |
| | html_files.append(html_file) |
| | else: |
| | errors[username] = error |
| | return html_files, errors |
| |
|
| | def perform_snapshot_download(repo_id, repo_type): |
| | try: |
| | temp_dir = tempfile.mkdtemp() |
| | snapshot_download(repo_id=repo_id, repo_type=repo_type, local_dir=temp_dir) |
| | zip_name = f"{repo_id.replace('/', '_')}_{repo_type}.zip" |
| | zip_path = os.path.join(SNAPSHOT_DIR, zip_name) |
| | shutil.make_archive(zip_path[:-4], 'zip', temp_dir) |
| | shutil.rmtree(temp_dir) |
| | return zip_path |
| | except Exception as e: |
| | return str(e) |
| |
|
| | st.title("Hugging Face User Page Downloader & Zipper 📄➕📦") |
| |
|
| | user_input = st.text_area( |
| | "Enter Hugging Face usernames (one per line):", |
| | value="\n".join(default_users["users"]), |
| | height=300 |
| | ) |
| |
|
| | if st.button("Show User Content and Download Snapshots"): |
| | if user_input: |
| | username_list = [username.strip() for username in user_input.split('\n') if username.strip()] |
| | |
| | user_data_list = asyncio.run(fetch_all_users(username_list)) |
| | |
| | stats = {"username": [], "models_count": [], "datasets_count": []} |
| | successful_html_files = [] |
| | snapshot_downloads = [] |
| | |
| | st.markdown("### User Content Overview") |
| | for user_data in user_data_list: |
| | username = user_data["username"] |
| | with st.container(): |
| | st.markdown(f"**{username}** [🔗 Profile](https://huggingface.co/{username})") |
| | |
| | if "error" in user_data: |
| | st.warning(f"{username}: {user_data['error']} - Something went wrong! ⚠️") |
| | else: |
| | models = user_data["models"] |
| | datasets = user_data["datasets"] |
| | |
| | html_file_path, download_error = download_user_page(username) |
| | if html_file_path: |
| | successful_html_files.append(html_file_path) |
| | st.success(f"✅ Successfully downloaded {username}'s page.") |
| | else: |
| | st.error(f"❌ Failed to download {username}'s page: {download_error}") |
| | |
| | stats["username"].append(username) |
| | stats["models_count"].append(len(models)) |
| | stats["datasets_count"].append(len(datasets)) |
| | |
| | with st.expander(f"🧠 Models ({len(models)})", expanded=False): |
| | if models: |
| | for model in models: |
| | model_name = model.modelId.split("/")[-1] |
| | st.markdown(f"- [{model_name}](https://huggingface.co/{model.modelId})") |
| | if st.button(f"Download Snapshot: {model_name}", key=f"model_{model.modelId}"): |
| | with st.spinner(f"Downloading snapshot for {model_name}..."): |
| | result = perform_snapshot_download(model.modelId, "model") |
| | if isinstance(result, str): |
| | st.error(f"Failed to download {model_name}: {result}") |
| | else: |
| | snapshot_downloads.append(result) |
| | st.success(f"Successfully downloaded snapshot for {model_name}") |
| | else: |
| | st.markdown("No models found. 🤷♂️") |
| | |
| | with st.expander(f"📚 Datasets ({len(datasets)})", expanded=False): |
| | if datasets: |
| | for dataset in datasets: |
| | dataset_name = dataset.id.split("/")[-1] |
| | st.markdown(f"- [{dataset_name}](https://huggingface.co/datasets/{dataset.id})") |
| | if st.button(f"Download Snapshot: {dataset_name}", key=f"dataset_{dataset.id}"): |
| | with st.spinner(f"Downloading snapshot for {dataset_name}..."): |
| | result = perform_snapshot_download(dataset.id, "dataset") |
| | if isinstance(result, str): |
| | st.error(f"Failed to download {dataset_name}: {result}") |
| | else: |
| | snapshot_downloads.append(result) |
| | st.success(f"Successfully downloaded snapshot for {dataset_name}") |
| | else: |
| | st.markdown("No datasets found. 🤷♀️") |
| | |
| | st.markdown("---") |
| | |
| | if successful_html_files: |
| | html_zip_path = create_zip_of_files(successful_html_files, "HuggingFace_User_Pages.zip") |
| | html_download_link = get_download_link(html_zip_path, "📥 Download All HTML Pages as ZIP") |
| | st.markdown(html_download_link, unsafe_allow_html=True) |
| | else: |
| | st.warning("No HTML files were successfully downloaded to create a ZIP archive.") |
| | |
| | if snapshot_downloads: |
| | snapshot_zip_path = create_zip_of_files(snapshot_downloads, "HuggingFace_Snapshots.zip") |
| | snapshot_download_link = get_download_link(snapshot_zip_path, "📥 Download All Snapshots as ZIP") |
| | st.markdown(snapshot_download_link, unsafe_allow_html=True) |
| | |
| | if stats["username"]: |
| | st.markdown("### User Content Statistics") |
| | |
| | fig_models = px.bar( |
| | x=stats["username"], |
| | y=stats["models_count"], |
| | labels={'x': 'Username', 'y': 'Number of Models'}, |
| | title="Number of Models per User" |
| | ) |
| | st.plotly_chart(fig_models) |
| | |
| | fig_datasets = px.bar( |
| | x=stats["username"], |
| | y=stats["datasets_count"], |
| | labels={'x': 'Username', 'y': 'Number of Datasets'}, |
| | title="Number of Datasets per User" |
| | ) |
| | st.plotly_chart(fig_datasets) |
| | |
| | else: |
| | st.warning("Please enter at least one username. Don't be shy! 😅") |
| |
|
| | st.sidebar.markdown(""" |
| | ## How to use: |
| | 1. The text area is pre-filled with a list of Hugging Face usernames. You can edit this list or add more usernames. |
| | 2. Click **'Show User Content and Download Snapshots'**. |
| | 3. View each user's models and datasets along with a link to their Hugging Face profile. |
| | 4. For each model or dataset, you can click the "Download Snapshot" button to download a snapshot. |
| | 5. **Download ZIP archives** containing all the HTML pages and snapshots by clicking the download links. |
| | 6. Check out the statistics visualizations below! |
| | """) |
