Upload hf_sync.py with huggingface_hub

hf_sync.py

@@ -1,12 +1,36 @@
 import os
 import sys
 import shutil
+import sqlite3
 from huggingface_hub import snapshot_download, HfApi
 
 # Configuration
 REPO_ID = os.environ.get("DATASET_REPO_ID")
 HF_TOKEN = os.environ.get("HF_TOKEN")
 
+def verify_data():
+    db_path = "data_repo/database.db"
+    if not os.path.exists(db_path):
+        print(f"VERIFICATION FAILED: {db_path} does not exist.")
+        return False
+    
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        cursor.execute("SELECT username FROM users WHERE username = ?", ("akshit",))
+        row = cursor.fetchone()
+        conn.close()
+        
+        if row:
+            print("VERIFICATION SUCCESS: User 'akshit' found in database.")
+            return True
+        else:
+            print("VERIFICATION FAILED: User 'akshit' NOT found in database.")
+            return False
+    except Exception as e:
+        print(f"VERIFICATION ERROR: {e}")
+        return False
+
 def download():
     if not REPO_ID:
         print("DATASET_REPO_ID not set, skipping download.")
@@ -23,6 +47,7 @@ def download():
             max_workers=8
         )
         print("Download successful.")
+        verify_data()
     except Exception as e:
         print(f"Download failed: {e}")
 
@@ -41,7 +66,7 @@ def upload():
             folder_path="data_repo",
             repo_id=REPO_ID,
             repo_type="dataset",
-            # This handles large folders by committing in chunks if necessary
+            delete_patterns="*", # Optional: sync deletion if needed
         )
         print("Upload successful.")
     except Exception as e:
@@ -55,7 +80,7 @@ def init_local():
 
 if __name__ == "__main__":
     if len(sys.argv) < 2:
-        print("Usage: python hf_sync.py [download|upload|init]")
+        print("Usage: python hf_sync.py [download|upload|init|verify]")
         sys.exit(1)
     
     action = sys.argv[1]
@@ -65,5 +90,7 @@ if __name__ == "__main__":
         upload()
     elif action == "init":
         init_local()
+    elif action == "verify":
+        verify_data()
     else:
-        print(f"Unknown action: {action}")
+        print(f"Unknown action: {action}")