Update app.py

app.py

@@ -8,61 +8,60 @@ TOKEN = os.environ.get("HF_TOKEN")
 PASS = "aAsHiSh_Cyb3r_H4CK$=666=DUmp*&~"
 SOURCE_REPO = "Watchhrr/HITECH_DB"
 TARGET_REPO = "Watchhrr/HITECH_MASTER_DB"
-DB_NAME = "hitech_master.db"
+MASTER_DB = "HITECH_FULL_MASTER.db"
 
 api = HfApi(token=TOKEN)
-con = duckdb.connect(DB_NAME)
 
-# Table setup (Mobile, ID, Name, Address, State)
-con.execute("CREATE TABLE IF NOT EXISTS master_data (mobile VARCHAR, id VARCHAR, name VARCHAR, address VARCHAR, state VARCHAR)")
+def process_and_merge():
+    # 1. Main connection jo sabko jodega
+    con = duckdb.connect(MASTER_DB)
+    con.execute("CREATE TABLE IF NOT EXISTS master_data (mobile VARCHAR, id VARCHAR, name VARCHAR, address VARCHAR, state VARCHAR)")
 
-print("🚀 STREAMING PROCESSOR STARTED...")
-
-# Ek-ek part process karenge taaki storage na bhare
-for i in range(1, 37):
-    part = f"Hi-Tek-DB.zip.{str(i).zfill(3)}"
-    print(f"\n📥 [{i}/36] Processing {part}...")
-    
-    try:
-        # 1. Download only ONE part
-        file_path = hf_hub_download(repo_id=SOURCE_REPO, filename=part, repo_type="dataset", token=TOKEN, local_dir=".")
-        
-        # 2. Extract specifically this part (7z can handle single parts of a split if they are handled correctly)
-        # Hum -p password use karenge
-        print(f"⚡ Extracting {part}...")
-        os.system(f"7z e {part} -p'{PASS}' -y -o./temp_data")
+    # 36 files ko 6-6 ke groups mein process karenge
+    for start in range(1, 37, 6):
+        end = min(start + 5, 36)
+        print(f"\n🚀 Processing Range: {start} to {end}")
         
-        # 3. Inject CSV/TXT files from this part
-        if os.path.exists("./temp_data"):
-            files = os.listdir("./temp_data")
-            for f in files:
-                print(f"💉 Injecting {f}...")
-                con.execute(f"INSERT INTO master_data SELECT * FROM read_csv_auto('./temp_data/{f}', ignore_errors=true)")
-            
-            # 4. CLEANUP IMMEDIATELY (Very important for storage)
-            os.system("rm -rf ./temp_data")
+        # A. Download
+        current_parts = []
+        for i in range(start, end + 1):
+            part_name = f"Hi-Tek-DB.zip.{str(i).zfill(3)}"
+            print(f"📥 Downloading {part_name}...")
+            hf_hub_download(repo_id=SOURCE_REPO, filename=part_name, repo_type="dataset", token=TOKEN, local_dir=".")
+            current_parts.append(part_name)
+
+        # B. Merge Splits & Extract
+        print("🔗 Merging splits...")
+        os.system(f"cat Hi-Tek-DB.zip.* > temp_chunk.zip")
+        os.system("rm Hi-Tek-DB.zip.*") # Turant delete
         
-        os.remove(part) # Original part delete karo
-        print(f"✅ Part {i} Cleaned up.")
+        print("⚡ Extracting with Password...")
+        os.system(f"7z e temp_chunk.zip -p'{PASS}' -y -o./temp_data")
+        os.remove("temp_chunk.zip")
 
-        # Backup upload every 5 parts
-        if i % 5 == 0:
-            print("☁️ Syncing partial DB to Hugging Face...")
-            api.upload_file(path_or_fileobj=DB_NAME, path_in_repo=DB_NAME, repo_id=TARGET_REPO, repo_type="dataset")
+        # C. Inject into Master Table
+        print("💉 Injecting into Master Table...")
+        con.execute("INSERT INTO master_data SELECT * FROM read_csv_auto('./temp_data/*.csv', ignore_errors=true)")
+        
+        # D. Cleanup
+        os.system("rm -rf ./temp_data")
+        print(f"✅ Range {start}-{end} merged into Master.")
 
-    except Exception as e:
-        print(f"🚨 Error in part {i}: {e}")
+    # 2. Final Indexing (Search fast karne ke liye)
+    print("\n📌 Creating Turbo Search Index...")
+    con.execute("CREATE INDEX idx_mobile ON master_data (mobile)")
+    con.commit()
+    con.close()
 
-# Final Indexing (Search speed badhane ke liye)
-print("\n📌 Creating Turbo Indexes...")
-con.execute("CREATE INDEX IF NOT EXISTS idx_mobile ON master_data (mobile)")
-con.commit()
+    # 3. Final Upload
+    print("☁️ Uploading FULL MASTER DB to Hugging Face...")
+    api.upload_file(path_or_fileobj=MASTER_DB, path_in_repo=MASTER_DB, repo_id=TARGET_REPO, repo_type="dataset")
 
-# Final Upload
-print("☁️ Final Uploading...")
-api.upload_file(path_or_fileobj=DB_NAME, path_in_repo=DB_NAME, repo_id=TARGET_REPO, repo_type="dataset")
+# --- EXECUTION ---
+try:
+    process_and_merge()
+    print("🏁 MISSION COMPLETE! Ek hi master file ban gayi.")
+except Exception as e:
+    print(f"🚨 Final Error: {e}")
 
-print("🏁 MISSION COMPLETE!")
-while True:
-    time.sleep(3600)
-    
+while True: time.sleep(3600)