Create sync.py

sync.py

@@ -0,0 +1,132 @@
+import os
+import json
+import time
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, unquote
+from huggingface_hub import HfApi
+
+# Configuration from Environment Variables (Secrets)
+ONION_URL = os.getenv("ONION_URL")  # e.g., http://xyz.onion/dirname/
+HF_TOKEN = os.getenv("HF_TOKEN")
+DATASET_REPO_ID = os.getenv("DATASET_REPO_ID")  # e.g., username/dataset-name
+
+HISTORY_FILE = "history.json"
+MAX_LOCAL_STORAGE_BYTES = 35 * 1024 * 1024 * 1024  # Keep under 40GB limit (35GB threshold)
+
+# Configure requests session to route through Tor SOCKS5 proxy
+session = requests.Session()
+session.proxies = {
+    'http': 'socks5h://127.0.0.1:9050',
+    'https': 'socks5h://127.0.0.1:9050'
+}
+
+def load_history():
+    if os.path.exists(HISTORY_FILE):
+        with open(HISTORY_FILE, "r") as f:
+            return set(json.load(f))
+    return set()
+
+def save_history(completed_files):
+    with open(HISTORY_FILE, "w") as f:
+        json.dump(list(completed_files), f)
+
+def get_dir_size(start_path='.'):
+    total_size = 0
+    for dirpath, dirnames, filenames in os.walk(start_path):
+        for f in filenames:
+            if f == HISTORY_FILE:
+                continue
+            fp = os.path.join(dirpath, f)
+            if os.path.exists(fp):
+                total_size += os.path.getsize(fp)
+    return total_size
+
+def crawl_and_collect(url, relative_path=""):
+    """Recursively discover all files from the web directory listing."""
+    print(f"Scanning: {url}")
+    files_to_download = []
+    try:
+        response = session.get(url, timeout=30)
+        if response.status_code != 200:
+            print(f"Failed to fetch {url}: Status {response.status_code}")
+            return files_to_download
+        
+        soup = BeautifulSoup(response.text, 'html.parser')
+        for link in soup.find_all('a'):
+            href = link.get('href')
+            if not href or href.startswith('?') or href == '../' or href == './':
+                continue
+            
+            clean_href = unquote(href)
+            full_url = urljoin(url, href)
+            target_rel_path = os.path.join(relative_path, clean_href)
+
+            if href.endswith('/'):
+                # Recursive call for subdirectories
+                files_to_download.extend(crawl_and_collect(full_url, target_rel_path))
+            else:
+                files_to_download.append((full_url, target_rel_path))
+                
+    except Exception as e:
+        print(f"Error crawling {url}: {e}")
+        
+    return files_to_download
+
+def main():
+    if not all([ONION_URL, HF_TOKEN, DATASET_REPO_ID]):
+        raise ValueError("Please set ONION_URL, HF_TOKEN, and DATASET_REPO_ID secrets.")
+
+    api = HfApi(token=HF_TOKEN)
+    completed_files = load_history()
+    
+    print("Discovering file structure from .onion source...")
+    all_files = crawl_and_collect(ONION_URL)
+    print(f"Total files discovered: {len(all_files)}")
+
+    for file_url, rel_path in all_files:
+        if rel_path in completed_files:
+            continue
+
+        # Check local space utilization before downloading
+        while get_dir_size() > MAX_LOCAL_STORAGE_BYTES:
+            print("Storage threshold reached. Waiting/Cleaning...")
+            time.sleep(10)
+
+        print(f"Downloading: {rel_path}")
+        local_file_path = os.path.join("downloads", rel_path)
+        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+        try:
+            # Stream download to handle large files efficiently
+            with session.get(file_url, stream=True, timeout=60) as r:
+                r.raise_for_status()
+                with open(local_file_path, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=8192):
+                        if chunk:
+                            f.write(chunk)
+
+            # Upload immediately to Hugging Face Dataset Repo
+            print(f"Uploading {rel_path} to HF dataset...")
+            api.upload_file(
+                path_or_fileobj=local_file_path,
+                path_in_repo=rel_path,
+                repo_id=DATASET_REPO_ID,
+                repo_type="dataset"
+            )
+
+            # Mark as completed and remove locally to free up space
+            completed_files.add(rel_path)
+            save_history(completed_files)
+            
+            if os.path.exists(local_file_path):
+                os.remove(local_file_path)
+
+        except Exception as e:
+            print(f"Error processing {rel_path}: {e}")
+            time.sleep(5)  # Backoff on error
+
+    print("Synchronization complete!")
+
+if __name__ == "__main__":
+    main()