feat: add Pinterest crawling support, update dependencies and meta_info

crawl_datasets.py

@@ -5,6 +5,11 @@ import pandas as pd
 from PIL import Image
 import requests
 import time
+from bs4 import BeautifulSoup
+# Selenium for dynamic page rendering (Pinterest loads images via JS)
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from webdriver_manager.chrome import ChromeDriverManager
 
 # ─────────────────────────────────────────────
 # HuggingFace dataset sources
@@ -31,6 +36,15 @@ SOURCES = {
         "prefix": "anime",
         "description": "Anime Face Dataset 10k illustrations",
     },
+    "blue_archive": {
+        # This source is a web‑scrape target rather than a parquet dataset.
+        "type": "web",
+        "search_url": "https://www.pinterest.com/search/pins/?q=blue%20archive",
+        "num_images": 300,
+        "subdir": "blue_archive",
+        "prefix": "blue",
+        "description": "Blue Archive images scraped from Pinterest",
+    },
 }
 
 # Fallback: direct image download lists for anime / landscape
@@ -75,6 +89,89 @@ def extract_image_bytes(cell) -> bytes | None:
     return None
 
 
+# ---------------------------------------------------------------------
+# Helper functions for web‑scraping sources
+# ---------------------------------------------------------------------
+def scrape_image_urls(search_url: str, max_images: int) -> list:
+    """Scrape image URLs from a Pinterest search page.
+
+    The function fetches the HTML of the search results, extracts ``<img>``
+    tags and returns up to ``max_images`` URLs that end with common image file
+    extensions. Pagination is handled by appending ``&page=N`` to the query URL
+    when needed.
+    """
+    print(f"  Scraping Pinterest for up to {max_images} images …")
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
+    }
+    collected: list[str] = []
+    page = 1
+    # Initialize a headless Chrome driver for dynamic content.
+    chrome_options = Options()
+    chrome_options.add_argument("--headless")
+    chrome_options.add_argument("--disable-gpu")
+    chrome_options.add_argument("--no-sandbox")
+    driver = webdriver.Chrome(ChromeDriverManager().install(), options=chrome_options)
+
+    while len(collected) < max_images:
+        # Build the URL for the current page.
+        url = (
+            f"{search_url}&page={page}" if "page=" not in search_url else f"{search_url.split('page=')[0]}page={page}"
+        )
+        try:
+            driver.get(url)
+            # Give the page a moment to load dynamic content.
+            driver.implicitly_wait(3)
+            page_html = driver.page_source
+        except Exception as e:
+            print(f"  [WARN] Selenium failed to load page {page}: {e}")
+            break
+
+        # Parse the rendered HTML.
+        soup = BeautifulSoup(page_html, "html.parser")
+        imgs = soup.find_all("img")
+        new_urls = [img.get("src") for img in imgs if img.get("src")]
+        # Filter duplicates and keep only image files.
+        new_urls = [u for u in new_urls if u not in collected and u.lower().endswith((".png", ".jpg", ".jpeg"))]
+        if not new_urls:
+            # No more new images on this page; stop pagination.
+            break
+        collected.extend(new_urls)
+        print(f"  Found {len(collected)} image URLs so far …")
+        page += 1
+
+    # Clean up the driver.
+    driver.quit()
+    return collected[:max_images]
+
+
+def download_images_from_urls(urls: list, save_dir: str, prefix: str, start_idx: int, target: int) -> int:
+    """Download, resize and save images from a list of URLs.
+
+    Mirrors the parquet pipeline: images are resized to 512×512 and saved as PNG
+    files named ``{prefix}_{index:05d}.png``.
+    """
+    saved = 0
+    for url in urls:
+        if saved >= target:
+            break
+        try:
+            resp = requests.get(url, stream=True, timeout=10)
+            resp.raise_for_status()
+            img = Image.open(resp.raw).convert("RGB")
+            img = img.resize((512, 512), Image.Resampling.LANCZOS)
+            filename = f"{prefix}_{start_idx + saved:05d}.png"
+            filepath = os.path.join(save_dir, filename)
+            img.save(filepath, "PNG")
+            saved += 1
+            if saved % 20 == 0 or saved == target:
+                print(f"  [{prefix}] {saved}/{target} images saved …")
+        except Exception as e:
+            print(f"  [warn] Failed to download {url}: {e}")
+            continue
+    return saved
+
 def crawl_source(name: str, cfg: dict, base_dataset_dir: str):
     """Download images for a single source category."""
     save_dir = os.path.join(base_dataset_dir, cfg["subdir"])
@@ -86,6 +183,28 @@ def crawl_source(name: str, cfg: dict, base_dataset_dir: str):
     print(f"  Target   : {cfg['num_images']} images")
     print(f"{'='*55}")
 
+    # ---------------------------------------------------------------------
+    # Determine source type. Parquet datasets are the default; "web" indicates
+    # we need to scrape image URLs.
+    # ---------------------------------------------------------------------
+    # Compute existing highest index first – needed for both flows.
+    existing = [f for f in os.listdir(save_dir) if f.endswith('.png') and f.startswith(cfg['prefix'])]
+    highest_idx = -1
+    for f in existing:
+        try:
+            idx = int(f.replace(cfg['prefix'] + '_', '').replace('.png', ''))
+            highest_idx = max(highest_idx, idx)
+        except ValueError:
+            continue
+    start_idx = highest_idx + 1
+
+    if cfg.get('type') == 'web':
+        urls = scrape_image_urls(cfg['search_url'], cfg['num_images'])
+        saved = download_images_from_urls(urls, save_dir, cfg['prefix'], start_idx, cfg['num_images'])
+        print(f"\n  [OK] [{name.upper()}] Done: {saved} images saved to {save_dir}")
+        return saved
+
+    # Existing parquet flow
     df = load_parquet_safe(cfg["parquet_url"])
 
     # Try fallbacks if primary failed

datasets/realesrgan_gt/meta_info.txt

@@ -3940,6 +3940,306 @@ face_00596.png 1
 face_00597.png 1
 face_00598.png 1
 face_00599.png 1
+face_00600.png 1
+face_00601.png 1
+face_00602.png 1
+face_00603.png 1
+face_00604.png 1
+face_00605.png 1
+face_00606.png 1
+face_00607.png 1
+face_00608.png 1
+face_00609.png 1
+face_00610.png 1
+face_00611.png 1
+face_00612.png 1
+face_00613.png 1
+face_00614.png 1
+face_00615.png 1
+face_00616.png 1
+face_00617.png 1
+face_00618.png 1
+face_00619.png 1
+face_00620.png 1
+face_00621.png 1
+face_00622.png 1
+face_00623.png 1
+face_00624.png 1
+face_00625.png 1
+face_00626.png 1
+face_00627.png 1
+face_00628.png 1
+face_00629.png 1
+face_00630.png 1
+face_00631.png 1
+face_00632.png 1
+face_00633.png 1
+face_00634.png 1
+face_00635.png 1
+face_00636.png 1
+face_00637.png 1
+face_00638.png 1
+face_00639.png 1
+face_00640.png 1
+face_00641.png 1
+face_00642.png 1
+face_00643.png 1
+face_00644.png 1
+face_00645.png 1
+face_00646.png 1
+face_00647.png 1
+face_00648.png 1
+face_00649.png 1
+face_00650.png 1
+face_00651.png 1
+face_00652.png 1
+face_00653.png 1
+face_00654.png 1
+face_00655.png 1
+face_00656.png 1
+face_00657.png 1
+face_00658.png 1
+face_00659.png 1
+face_00660.png 1
+face_00661.png 1
+face_00662.png 1
+face_00663.png 1
+face_00664.png 1
+face_00665.png 1
+face_00666.png 1
+face_00667.png 1
+face_00668.png 1
+face_00669.png 1
+face_00670.png 1
+face_00671.png 1
+face_00672.png 1
+face_00673.png 1
+face_00674.png 1
+face_00675.png 1
+face_00676.png 1
+face_00677.png 1
+face_00678.png 1
+face_00679.png 1
+face_00680.png 1
+face_00681.png 1
+face_00682.png 1
+face_00683.png 1
+face_00684.png 1
+face_00685.png 1
+face_00686.png 1
+face_00687.png 1
+face_00688.png 1
+face_00689.png 1
+face_00690.png 1
+face_00691.png 1
+face_00692.png 1
+face_00693.png 1
+face_00694.png 1
+face_00695.png 1
+face_00696.png 1
+face_00697.png 1
+face_00698.png 1
+face_00699.png 1
+face_00700.png 1
+face_00701.png 1
+face_00702.png 1
+face_00703.png 1
+face_00704.png 1
+face_00705.png 1
+face_00706.png 1
+face_00707.png 1
+face_00708.png 1
+face_00709.png 1
+face_00710.png 1
+face_00711.png 1
+face_00712.png 1
+face_00713.png 1
+face_00714.png 1
+face_00715.png 1
+face_00716.png 1
+face_00717.png 1
+face_00718.png 1
+face_00719.png 1
+face_00720.png 1
+face_00721.png 1
+face_00722.png 1
+face_00723.png 1
+face_00724.png 1
+face_00725.png 1
+face_00726.png 1
+face_00727.png 1
+face_00728.png 1
+face_00729.png 1
+face_00730.png 1
+face_00731.png 1
+face_00732.png 1
+face_00733.png 1
+face_00734.png 1
+face_00735.png 1
+face_00736.png 1
+face_00737.png 1
+face_00738.png 1
+face_00739.png 1
+face_00740.png 1
+face_00741.png 1
+face_00742.png 1
+face_00743.png 1
+face_00744.png 1
+face_00745.png 1
+face_00746.png 1
+face_00747.png 1
+face_00748.png 1
+face_00749.png 1
+face_00750.png 1
+face_00751.png 1
+face_00752.png 1
+face_00753.png 1
+face_00754.png 1
+face_00755.png 1
+face_00756.png 1
+face_00757.png 1
+face_00758.png 1
+face_00759.png 1
+face_00760.png 1
+face_00761.png 1
+face_00762.png 1
+face_00763.png 1
+face_00764.png 1
+face_00765.png 1
+face_00766.png 1
+face_00767.png 1
+face_00768.png 1
+face_00769.png 1
+face_00770.png 1
+face_00771.png 1
+face_00772.png 1
+face_00773.png 1
+face_00774.png 1
+face_00775.png 1
+face_00776.png 1
+face_00777.png 1
+face_00778.png 1
+face_00779.png 1
+face_00780.png 1
+face_00781.png 1
+face_00782.png 1
+face_00783.png 1
+face_00784.png 1
+face_00785.png 1
+face_00786.png 1
+face_00787.png 1
+face_00788.png 1
+face_00789.png 1
+face_00790.png 1
+face_00791.png 1
+face_00792.png 1
+face_00793.png 1
+face_00794.png 1
+face_00795.png 1
+face_00796.png 1
+face_00797.png 1
+face_00798.png 1
+face_00799.png 1
+face_00800.png 1
+face_00801.png 1
+face_00802.png 1
+face_00803.png 1
+face_00804.png 1
+face_00805.png 1
+face_00806.png 1
+face_00807.png 1
+face_00808.png 1
+face_00809.png 1
+face_00810.png 1
+face_00811.png 1
+face_00812.png 1
+face_00813.png 1
+face_00814.png 1
+face_00815.png 1
+face_00816.png 1
+face_00817.png 1
+face_00818.png 1
+face_00819.png 1
+face_00820.png 1
+face_00821.png 1
+face_00822.png 1
+face_00823.png 1
+face_00824.png 1
+face_00825.png 1
+face_00826.png 1
+face_00827.png 1
+face_00828.png 1
+face_00829.png 1
+face_00830.png 1
+face_00831.png 1
+face_00832.png 1
+face_00833.png 1
+face_00834.png 1
+face_00835.png 1
+face_00836.png 1
+face_00837.png 1
+face_00838.png 1
+face_00839.png 1
+face_00840.png 1
+face_00841.png 1
+face_00842.png 1
+face_00843.png 1
+face_00844.png 1
+face_00845.png 1
+face_00846.png 1
+face_00847.png 1
+face_00848.png 1
+face_00849.png 1
+face_00850.png 1
+face_00851.png 1
+face_00852.png 1
+face_00853.png 1
+face_00854.png 1
+face_00855.png 1
+face_00856.png 1
+face_00857.png 1
+face_00858.png 1
+face_00859.png 1
+face_00860.png 1
+face_00861.png 1
+face_00862.png 1
+face_00863.png 1
+face_00864.png 1
+face_00865.png 1
+face_00866.png 1
+face_00867.png 1
+face_00868.png 1
+face_00869.png 1
+face_00870.png 1
+face_00871.png 1
+face_00872.png 1
+face_00873.png 1
+face_00874.png 1
+face_00875.png 1
+face_00876.png 1
+face_00877.png 1
+face_00878.png 1
+face_00879.png 1
+face_00880.png 1
+face_00881.png 1
+face_00882.png 1
+face_00883.png 1
+face_00884.png 1
+face_00885.png 1
+face_00886.png 1
+face_00887.png 1
+face_00888.png 1
+face_00889.png 1
+face_00890.png 1
+face_00891.png 1
+face_00892.png 1
+face_00893.png 1
+face_00894.png 1
+face_00895.png 1
+face_00896.png 1
+face_00897.png 1
+face_00898.png 1
+face_00899.png 1
 hsr_acheron_sb_6723087.png 1
 hsr_acheron_sb_6723868.png 1
 hsr_acheron_sb_6723975.png 1

requirements.txt

@@ -8,6 +8,8 @@ attrs==26.1.0
 audioop-lts==0.2.2
 # basicsr (installed via patch_and_install_basicsr.py)
 beautifulsoup4==4.15.0
+selenium==4.21.0
+webdriver-manager==4.0.2
 blinker==1.9.0
 brotli==1.2.0
 cachetools==7.1.4