Update sync.py

sync.py

@@ -7,9 +7,9 @@ from urllib.parse import urljoin, unquote
 from huggingface_hub import HfApi
 
 # Configuration from Environment Variables (Secrets)
-ONION_URL = os.getenv("ONION_URL")  # e.g., http://xyz.onion/dirname/
+ONION_URL = os.getenv("ONION_URL")  # Must be: http://6qqz6m3b6htudohg2mlf5gdcalonxy3sh5g4dix4mpyirjcgelqqufad.onion/bankofbaroda.bank.in/
 HF_TOKEN = os.getenv("HF_TOKEN")
-DATASET_REPO_ID = os.getenv("DATASET_REPO_ID")  # e.g., username/dataset-name
+DATASET_REPO_ID = os.getenv("DATASET_REPO_ID")  # username/dataset-name
 
 HISTORY_FILE = "history.json"
 MAX_LOCAL_STORAGE_BYTES = 35 * 1024 * 1024 * 1024  # Keep below 40GB limit (35GB threshold)
@@ -55,15 +55,18 @@ def crawl_and_collect(url, relative_path=""):
         soup = BeautifulSoup(response.text, 'html.parser')
         for link in soup.find_all('a'):
             href = link.get('href')
-            if not href or href.startswith('?') or href == '../' or href == './':
+            if not href or href.startswith('?') or href in ['../', './', '..', '.']:
                 continue
             
-            clean_href = unquote(href)
+            clean_href = unquote(href).strip('/')
+            if not clean_href:
+                continue
+
             full_url = urljoin(url, href)
             target_rel_path = os.path.join(relative_path, clean_href)
 
-            if href.endswith('/'):
-                # Recursive call for subdirectories
+            # Check if it's a directory (ends with / in href or has trailing slash)
+            if href.endswith('/') or link.text.endswith('/'):
                 files_to_download.extend(crawl_and_collect(full_url, target_rel_path))
             else:
                 files_to_download.append((full_url, target_rel_path))
@@ -80,7 +83,10 @@ def main():
     api = HfApi(token=HF_TOKEN)
     completed_files = load_history()
     
-    print("Discovering file structure from .onion source...")
+    print("Waiting for Tor circuits to finalize...")
+    time.sleep(10)
+
+    print("Discovering file structure from .onion root source...")
     all_files = crawl_and_collect(ONION_URL)
     print(f"Total files discovered: {len(all_files)}")