import os import sqlite3 import sys from huggingface_hub import HfApi # --- CONFIGURATION --- DB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ktu_index.db") DATASET_REPO_ID = "KeralaTimetable/ktu-pyq-archive" # Reads from GitHub input, defaults to GAMAT101 if run locally without an environment variable SUBJECT_CODE = os.getenv("SUBJECT_CODE", "GAMAT101").strip() HF_TOKEN = os.getenv("HF_TOKEN") # Optional for public datasets, but good to have api = HfApi() def verify_subject_papers(): print(f"šŸ” Starting verification for subject: {SUBJECT_CODE}") if not os.path.exists(DB_PATH): print(f"āŒ Error: Database file not found at {DB_PATH}") sys.exit(1) # 1. Download the list of files currently on Hugging Face print("Scanning Hugging Face for uploaded files...") try: hf_files = set(api.list_repo_files(repo_id=DATASET_REPO_ID, repo_type="dataset", token=HF_TOKEN)) print(f"Found {len(hf_files)} files safely in the cloud.") except Exception as e: print(f"āŒ Error fetching Hugging Face repository files: {e}") sys.exit(1) # 2. Connect to your local database con = sqlite3.connect(DB_PATH) cursor = con.cursor() # We will dynamically check columns to ensure we don't crash if your column is named differently try: cursor.execute("PRAGMA table_info(papers)") columns = [col[1] for col in cursor.fetchall()] except Exception as e: print(f"āŒ Error reading database schema: {e}") con.close() sys.exit(1) # Determine which column to look up (handles 'subject', 'course_code', or falls back to 'title') search_column = "subject" if "subject" not in columns: if "course_code" in columns: search_column = "course_code" elif "title" in columns: search_column = "title" else: # Fallback to the first column that isn't 'handle' non_handle_cols = [c for c in columns if c != "handle"] if non_handle_cols: search_column = non_handle_cols[0] print(f"Using database column '{search_column}' for subject matching.") # 3. Query the database for the chosen subject try: query = f"SELECT handle FROM papers WHERE {search_column} LIKE ?" cursor.execute(query, (f"%{SUBJECT_CODE}%",)) rows = cursor.fetchall() except Exception as e: print(f"āŒ Database query failed: {e}") con.close() sys.exit(1) if not rows: print(f"\nāŒ No papers found in your local database for '{SUBJECT_CODE}'.") print("This means the scraper never extracted this code from the JEC website.") con.close() return print(f"\nFound {len(rows)} papers for {SUBJECT_CODE} in local database. Verifying cloud sync...") print("-" * 60) # 4. Cross-reference database handles with Hugging Face files missing_count = 0 for row in rows: handle = row[0] safe_filename = f"{handle.replace('/', '_')}.pdf" if safe_filename in hf_files: print(f"āœ… MATCHED: {safe_filename} is safe on Hugging Face") else: print(f"āŒ MISSING: {safe_filename} (Handle: {handle})") missing_count += 1 print("-" * 60) if missing_count == 0: print(f"šŸŽ‰ SUCCESS: All {len(rows)} papers for {SUBJECT_CODE} are fully backed up!") else: print(f"āš ļø Warning: {missing_count} out of {len(rows)} papers are missing from Hugging Face.") con.close() if __name__ == "__main__": verify_subject_papers()