File size: 4,867 Bytes
46c0813 eb81349 46c0813 eb81349 46c0813 b1f3f99 46c0813 eb81349 46c0813 eb81349 46c0813 eb81349 46c0813 eb81349 46c0813 b1f3f99 46c0813 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | import os
import json
import time
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, unquote
from huggingface_hub import HfApi
# Configuration from Environment Variables (Secrets)
ONION_URL = os.getenv("ONION_URL") # Must be: http://6qqz6m3b6htudohg2mlf5gdcalonxy3sh5g4dix4mpyirjcgelqqufad.onion/bankofbaroda.bank.in/
HF_TOKEN = os.getenv("HF_TOKEN")
DATASET_REPO_ID = os.getenv("DATASET_REPO_ID") # username/dataset-name
HISTORY_FILE = "history.json"
MAX_LOCAL_STORAGE_BYTES = 35 * 1024 * 1024 * 1024 # Keep below 40GB limit (35GB threshold)
# Configure requests session to route through Tor SOCKS5 proxy
session = requests.Session()
session.proxies = {
'http': 'socks5h://127.0.0.1:9050',
'https': 'socks5h://127.0.0.1:9050'
}
def load_history():
if os.path.exists(HISTORY_FILE):
with open(HISTORY_FILE, "r") as f:
return set(json.load(f))
return set()
def save_history(completed_files):
with open(HISTORY_FILE, "w") as f:
json.dump(list(completed_files), f)
def get_dir_size(start_path='.'):
total_size = 0
for dirpath, dirnames, filenames in os.walk(start_path):
for f in filenames:
if f == HISTORY_FILE:
continue
fp = os.path.join(dirpath, f)
if os.path.exists(fp):
total_size += os.path.getsize(fp)
return total_size
def crawl_and_collect(url, relative_path=""):
"""Recursively discover all files from the web directory listing."""
print(f"Scanning: {url}")
files_to_download = []
try:
response = session.get(url, timeout=30)
if response.status_code != 200:
print(f"Failed to fetch {url}: Status {response.status_code}")
return files_to_download
soup = BeautifulSoup(response.text, 'html.parser')
for link in soup.find_all('a'):
href = link.get('href')
if not href or href.startswith('?') or href in ['../', './', '..', '.']:
continue
clean_href = unquote(href).strip('/')
if not clean_href:
continue
full_url = urljoin(url, href)
target_rel_path = os.path.join(relative_path, clean_href)
# Check if it's a directory (ends with / in href or has trailing slash)
if href.endswith('/') or link.text.endswith('/'):
files_to_download.extend(crawl_and_collect(full_url, target_rel_path))
else:
files_to_download.append((full_url, target_rel_path))
except Exception as e:
print(f"Error crawling {url}: {e}")
return files_to_download
def main():
if not all([ONION_URL, HF_TOKEN, DATASET_REPO_ID]):
raise ValueError("Please set ONION_URL, HF_TOKEN, and DATASET_REPO_ID secrets.")
api = HfApi(token=HF_TOKEN)
completed_files = load_history()
print("Waiting for Tor circuits to finalize...")
time.sleep(10)
print("Discovering file structure from .onion root source...")
all_files = crawl_and_collect(ONION_URL)
print(f"Total files discovered: {len(all_files)}")
for file_url, rel_path in all_files:
if rel_path in completed_files:
continue
# Check local space utilization before downloading
while get_dir_size() > MAX_LOCAL_STORAGE_BYTES:
print("Storage threshold reached (35GB). Waiting/Cleaning...")
time.sleep(10)
print(f"Downloading: {rel_path}")
local_file_path = os.path.join("downloads", rel_path)
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
try:
# Stream download to handle large files efficiently
with session.get(file_url, stream=True, timeout=60) as r:
r.raise_for_status()
with open(local_file_path, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
# Upload immediately to Hugging Face Dataset Repo
print(f"Uploading {rel_path} to HF dataset...")
api.upload_file(
path_or_fileobj=local_file_path,
path_in_repo=rel_path,
repo_id=DATASET_REPO_ID,
repo_type="dataset"
)
# Mark as completed and remove locally to free up space
completed_files.add(rel_path)
save_history(completed_files)
if os.path.exists(local_file_path):
os.remove(local_file_path)
except Exception as e:
print(f"Error processing {rel_path}: {e}")
time.sleep(5) # Backoff on error
print("Synchronization complete!")
if __name__ == "__main__":
main() |