NAAC updated

app.py

@@ -309,8 +309,14 @@ def _run_inc_download_and_build(selected_states: List[str]):
 def _run_naac_download():
     """Background thread logic for NAAC download and merge"""
     output_lines = []
+    
+    def get_ui_update(done=False):
+        if done:
+            return "\n".join(output_lines), gr.update(interactive=True), gr.update(visible=False)
+        return "\n".join(output_lines), gr.update(interactive=False), gr.update(visible=True, interactive=True)
+
     output_lines.append("🚀 Starting NAAC Download...")
-    yield "\n".join(output_lines)
+    yield get_ui_update()
 
     base_dir = Path.cwd() / "output"
     downloads_dir = base_dir / "downloads"
@@ -322,19 +328,20 @@ def _run_naac_download():
             output_lines.append(msg)
         
         output_lines.append("\n⏳ Downloading NAAC Accredited Institutions...")
-        yield "\n".join(output_lines)
+        yield get_ui_update()
         
-        raw_naac_file = download_naac(output_dir=downloads_dir, headless=True, log_fn=log_naac)
+        selected_states = _get_selected_states()
+        raw_naac_file = download_naac(output_dir=downloads_dir, selected_states=selected_states, headless=True, log_fn=log_naac)
         
     except Exception as e:
         output_lines.append(f"❌ NAAC Download error: {e}")
         output_lines.append("\n❌ Process stopped or download failed.")
-        yield "\n".join(output_lines)
+        yield get_ui_update(done=True)
         return
 
     # 2. Merge and Filter
     output_lines.append("\n⏳ Filtering and formatting NAAC data...")
-    yield "\n".join(output_lines)
+    yield get_ui_update()
     
     try:
         selected_states = _get_selected_states()
@@ -342,20 +349,17 @@ def _run_naac_download():
         final_filename = f"naac_colleges_{timestamp}.xlsx"
         final_output_path = base_dir / final_filename
 
-        filter_naac(
-            raw_file_path=raw_naac_file,
-            output_file_path=final_output_path,
-            selected_states=selected_states
-        )
+        import shutil
+        shutil.copy2(raw_naac_file, final_output_path)
         
         output_lines.append(f"✅ Successfully filtered NAAC data based on selected states!")
         output_lines.append(f"💾 Saved locally as: {final_filename}")
-        yield "\n".join(output_lines)
+        yield get_ui_update()
         
         # 3. Upload to HuggingFace
         if is_configured():
             output_lines.append("\n⏳ Uploading NAAC file to Hugging Face dataset...")
-            yield "\n".join(output_lines)
+            yield get_ui_update()
             
             try:
                 upload_combined_excel(final_output_path)
@@ -367,11 +371,11 @@ def _run_naac_download():
             output_lines.append("\n⚠️ Hugging Face upload skipped (HF_TOKEN not configured).")
             output_lines.append("🎉 All NAAC steps completed successfully!")
             
-        yield "\n".join(output_lines)
+        yield get_ui_update(done=True)
 
     except Exception as e:
         output_lines.append(f"❌ Error filtering NAAC data: {e}")
-        yield "\n".join(output_lines)
+        yield get_ui_update(done=True)
         return
 
 def _run_aicte_download():
@@ -643,7 +647,9 @@ Click the **Download** button below. The app will automatically open the NAAC we
 **Currently keeping data from {state_count} states** *(You can change this in the State Filter tab)*.
                         """.replace("{state_count}", str(len(_get_selected_states()))))
                         
-                        btn_download_naac = gr.Button("▶ Download NAAC Data Now", variant="primary")
+                        with gr.Row():
+                            btn_download_naac = gr.Button("▶ Download NAAC Data Now", variant="primary")
+                            stop_naac_btn = gr.Button("🛑 Stop Scrape", variant="stop", visible=False)
                         
                     with gr.Column(scale=1):
                         naac_output = gr.Textbox(
@@ -655,7 +661,12 @@ Click the **Download** button below. The app will automatically open the NAAC we
                 
                 btn_download_naac.click(
                     fn=_run_naac_download,
-                    outputs=naac_output
+                    outputs=[naac_output, btn_download_naac, stop_naac_btn]
+                )
+                stop_naac_btn.click(
+                    fn=stop_scrape_process,
+                    inputs=[],
+                    outputs=[naac_output, stop_naac_btn, btn_download_naac]
                 )
 
             # ── Tab 4: AICTE ──────────────────────────────────────────────
@@ -755,7 +766,8 @@ All your previous scrapes are saved automatically. You can view and download the
                     refresh_hf_btn = gr.Button("🔄 Refresh List")
                     
                 with gr.Row():
-                    fetch_hf_btn = gr.DownloadButton("📥 Fetch & Download Selected", variant="primary")
+                    fetch_hf_btn = gr.Button("☁️ Fetch Selected from HF", variant="primary")
+                    download_hf_btn = gr.File(label="Ready to Download", visible=False)
                 
                 hf_status = gr.Textbox(label="Status", interactive=False, lines=1)
 
@@ -815,10 +827,16 @@ All your previous scrapes are saved automatically. You can view and download the
             outputs=[hf_files_dropdown],
         )
 
+        def handle_hf_fetch(filename):
+            path = download_file_from_hf(filename)
+            if path:
+                return gr.update(value="Fetched successfully!"), gr.update(value=path, visible=True)
+            return gr.update(value="Failed to fetch."), gr.update(visible=False)
+
         fetch_hf_btn.click(
-            fn=download_file_from_hf,
+            fn=handle_hf_fetch,
             inputs=[hf_files_dropdown],
-            outputs=[fetch_hf_btn],
+            outputs=[hf_status, download_hf_btn],
         )
         
         # Load dataset files on startup

hf_store.py

@@ -134,6 +134,17 @@ def download_combined_file(remote_filename: str, local_dir: Path) -> Optional[Pa
     from huggingface_hub import hf_hub_download
     try:
         local_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Fast path: Check if we already have it locally to bypass network lag
+        expected_local = local_dir / remote_filename
+        if expected_local.exists():
+            return expected_local
+            
+        # Check without 'combined/' prefix just in case
+        fallback_local = local_dir / remote_filename.split('/')[-1]
+        if fallback_local.exists():
+            return fallback_local
+
         local_path = hf_hub_download(
             repo_id=repo_id,
             filename=remote_filename,

requirements.txt

@@ -8,3 +8,5 @@ huggingface_hub>=0.20.0
 pytest>=7.0.0
 pytest-playwright>=0.4.0
 spaces
+thefuzz>=0.22.0
+python-Levenshtein>=0.27.0

scraper/master_merger.py

@@ -0,0 +1,315 @@
+import pandas as pd
+from pathlib import Path
+import re
+from thefuzz import process, fuzz
+import time
+from typing import Optional, Tuple, Dict, Any, List
+
+def get_latest_file(directory: Path, prefix: str) -> Optional[Path]:
+    files = list(directory.glob(f"{prefix}*.xlsx"))
+    if not files:
+        return None
+    return max(files, key=lambda p: p.stat().st_mtime)
+
+def clean_text(text: Any) -> str:
+    if pd.isna(text):
+        return ""
+    text = str(text).lower()
+    text = re.sub(r'[^a-z0-9\s]', '', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+def fuzzy_match_college(target_name: str, candidates: list, threshold: int = 85) -> Optional[str]:
+    if not candidates or not target_name:
+        return None
+    best_match = process.extractOne(target_name, candidates, scorer=fuzz.token_sort_ratio)
+    if best_match and best_match[1] >= threshold:
+        return best_match[0]
+    return None
+
+def merge_master_database(output_dir: Path):
+    yield "Starting Relational Master Database Merger..."
+    
+    files = {
+        "aishe": get_latest_file(output_dir, "aishe_colleges"),
+        "naac": get_latest_file(output_dir, "naac_colleges"),
+        "ugc": get_latest_file(output_dir, "ugc_colleges"),
+        "aicte": get_latest_file(output_dir, "aicte_colleges"),
+        "inc": get_latest_file(output_dir, "inc_colleges")
+    }
+    
+    for k, v in files.items():
+        if v:
+            yield f"Found {k.upper()} file: {v.name}"
+        else:
+            yield f"WARNING: No file found for {k.upper()}"
+            
+    if not files["aishe"]:
+        yield "ERROR: AISHE master file is required. Aborting."
+        return
+        
+    yield "\nLoading AISHE..."
+    aishe_sheets = pd.read_excel(files["aishe"], sheet_name=None, header=2)
+    aishe_dfs = []
+    for sheet_name, df in aishe_sheets.items():
+        df['Institution_Category'] = sheet_name
+        
+        # Try to find Management column
+        m_col = None
+        for col in ['Manegement', 'Management', 'Management Type']:
+            if col in df.columns:
+                m_col = col
+                break
+        
+        if m_col:
+            df = df.rename(columns={m_col: "AISHE_Management"})
+        else:
+            df["AISHE_Management"] = None
+            
+        aishe_dfs.append(df)
+        
+    colleges_df = pd.concat(aishe_dfs, ignore_index=True)
+    colleges_df = colleges_df.rename(columns={
+        "Aishe Code": "AISHE_ID",
+        "Name": "Institution_Name",
+        "State": "State",
+        "District": "District",
+        "Website": "Website",
+        "Year Of Establishment": "Year_Of_Establishment"
+    })
+    
+    colleges_df['Clean_Name'] = colleges_df['Institution_Name'].apply(clean_text)
+    colleges_df['Clean_State'] = colleges_df['State'].apply(clean_text)
+    colleges_df['Clean_District'] = colleges_df['District'].apply(clean_text)
+    
+    yield "Removing basic duplicates from AISHE..."
+    colleges_df = colleges_df.drop_duplicates(subset=['Clean_Name', 'Clean_State', 'Clean_District'], keep='first')
+    
+    # Initialize APF_ID for base colleges
+    colleges_df.insert(0, "APF_ID", [f"APF{i:06d}" for i in range(1, len(colleges_df) + 1)])
+    next_apf_id = len(colleges_df) + 1
+    
+    courses_list = []
+    
+    # --- NAAC Overlay ---
+    if files["naac"]:
+        yield "Overlaying NAAC Data..."
+        naac_df = pd.read_excel(files["naac"])
+        if "Aishe-Id" in naac_df.columns:
+            naac_subset = naac_df[["Aishe-Id", "Current Grade", "Current CGPA"]].copy()
+            naac_subset = naac_subset.rename(columns={
+                "Aishe-Id": "AISHE_ID",
+                "Current Grade": "NAAC_Grade",
+                "Current CGPA": "NAAC_CGPA"
+            })
+            colleges_df = colleges_df.merge(naac_subset, on="AISHE_ID", how="left")
+            
+    def process_overlay(overlay_df, name_col, state_col, district_col, source_name, is_course_source=False, course_cols_map=None, college_cols_map=None):
+        nonlocal colleges_df, next_apf_id, courses_list
+        print(f"Overlaying {source_name} Data via Fuzzy Matching...")
+        
+        overlay_df['Clean_Name'] = overlay_df[name_col].apply(clean_text)
+        overlay_df['Clean_State'] = overlay_df[state_col].apply(clean_text)
+        if district_col and district_col in overlay_df.columns:
+            overlay_df['Clean_District'] = overlay_df[district_col].apply(clean_text)
+            
+        if college_cols_map:
+            for v in college_cols_map.values():
+                if v not in colleges_df.columns:
+                    colleges_df[v] = None
+                    
+        # Group by college to prevent duplicating the college for every course
+        grouped = overlay_df.groupby(['Clean_Name', 'Clean_State', 'Clean_District'] if district_col else ['Clean_Name', 'Clean_State'])
+        
+        match_count = 0
+        unmatched_count = 0
+        new_colleges = []
+        
+        for group_keys, group_df in grouped:
+            # group_keys is a tuple of (Name, State, [District])
+            nm = group_keys[0]
+            st = group_keys[1]
+            dist = group_keys[2] if len(group_keys) > 2 else None
+            
+            candidates = colleges_df[colleges_df['Clean_State'] == st]
+            if dist:
+                dist_candidates = candidates[candidates['Clean_District'] == dist]
+                if not dist_candidates.empty:
+                    candidates = dist_candidates
+                    
+            candidate_names = candidates['Clean_Name'].tolist()
+            best_match = fuzzy_match_college(nm, candidate_names)
+            
+            target_apf_id = None
+            if best_match:
+                idx = candidates[candidates['Clean_Name'] == best_match].index[0]
+                target_apf_id = colleges_df.at[idx, 'APF_ID']
+                # Update college-level metadata
+                if college_cols_map:
+                    first_row = group_df.iloc[0]
+                    for old_col, new_col in college_cols_map.items():
+                        if old_col in first_row and pd.notna(first_row[old_col]):
+                            colleges_df.at[idx, new_col] = first_row[old_col]
+                match_count += 1
+            else:
+                # Create a new college entry
+                target_apf_id = f"APF{next_apf_id:06d}"
+                next_apf_id += 1
+                first_row = group_df.iloc[0]
+                
+                new_coll = {
+                    'APF_ID': target_apf_id,
+                    'Institution_Name': first_row[name_col],
+                    'State': first_row[state_col],
+                    'Clean_Name': nm,
+                    'Clean_State': st,
+                    'Missing_In_AISHE': True
+                }
+                if district_col and district_col in first_row:
+                    new_coll['District'] = first_row[district_col]
+                    new_coll['Clean_District'] = dist
+                    
+                if college_cols_map:
+                    for old_col, new_col in college_cols_map.items():
+                        if old_col in first_row:
+                            new_coll[new_col] = first_row[old_col]
+                            
+                new_colleges.append(new_coll)
+                unmatched_count += 1
+                
+            # Process courses if this source provides them
+            if is_course_source and course_cols_map:
+                for _, row in group_df.iterrows():
+                    course_entry = {
+                        'APF_ID': target_apf_id,
+                        'Source': source_name
+                    }
+                    for old_col, new_col in course_cols_map.items():
+                        if old_col in row:
+                            course_entry[new_col] = row[old_col]
+                    courses_list.append(course_entry)
+                    
+        print(f"  [SUCCESS] {source_name}: Mapped {match_count} colleges. {unmatched_count} new colleges appended.")
+        if new_colleges:
+            colleges_df = pd.concat([colleges_df, pd.DataFrame(new_colleges)], ignore_index=True)
+
+    # --- UGC Overlay ---
+    if files["ugc"]:
+        ugc_df = pd.read_excel(files["ugc"])
+        process_overlay(
+            ugc_df, 
+            name_col="Name of the college", 
+            state_col="State", 
+            district_col="District",
+            source_name="UGC",
+            is_course_source=False,
+            college_cols_map={
+                "Status": "UGC_Status",
+                "Affiliated To University": "UGC_Affiliated_University",
+                "Govt or Non Govt": "Funding_Type"
+            }
+        )
+
+    # --- AICTE Overlay ---
+    if files["aicte"]:
+        aicte_df = pd.read_excel(files["aicte"]).dropna(subset=['Institution Name'])
+        process_overlay(
+            aicte_df,
+            name_col="Institution Name",
+            state_col="State",
+            district_col="District",
+            source_name="AICTE",
+            is_course_source=True,
+            college_cols_map={"AICTE ID": "AICTE_ID"},
+            course_cols_map={
+                "AICTE ID": "Source_ID",
+                "Program": "Program_Name",
+                "Total Students": "Annual_Intake"
+            }
+        )
+
+    # --- INC Overlay ---
+    if files["inc"]:
+        inc_df = pd.read_excel(files["inc"])
+        process_overlay(
+            inc_df,
+            name_col="Institution Name & Address",
+            state_col="State",
+            district_col="District",
+            source_name="INC",
+            is_course_source=True,
+            college_cols_map={},
+            course_cols_map={
+                "Programme": "Program_Name",
+                "Annual Intake": "Annual_Intake"
+            }
+        )
+        
+    yield "\nCleaning up columns..."
+    
+    # Unify Management Type (AISHE vs UGC)
+    def resolve_mgmt(row):
+        f_type = str(row.get('Funding_Type', '')).lower()
+        a_mgmt = str(row.get('AISHE_Management', '')).lower()
+        
+        if 'govt' in f_type or 'government' in f_type or 'govt' in a_mgmt or 'government' in a_mgmt:
+            return 'Government'
+        elif 'private' in f_type or 'private' in a_mgmt or 'unaided' in f_type:
+            return 'Private / Non-Government'
+        elif 'aided' in f_type or 'aided' in a_mgmt:
+            return 'Aided'
+        return 'Unknown'
+        
+    colleges_df['Management_Type'] = colleges_df.apply(resolve_mgmt, axis=1)
+
+    # Final Columns Selection
+    compulsory_columns = [
+        "APF_ID", 
+        "AISHE_ID", 
+        "AICTE_ID", 
+        "Institution_Name", 
+        "State", 
+        "District", 
+        "Management_Type",
+        "Institution_Category", 
+        "Year_Of_Establishment", 
+        "Website",
+        "UGC_Status",
+        "UGC_Affiliated_University",
+        "NAAC_Grade",
+        "NAAC_CGPA"
+    ]
+    
+    for col in compulsory_columns:
+        if col not in colleges_df.columns:
+            colleges_df[col] = None
+            
+    colleges_df = colleges_df[compulsory_columns]
+    
+    courses_df = pd.DataFrame(courses_list)
+    course_columns = ["APF_ID", "Source", "Source_ID", "Program_Name", "Annual_Intake"]
+    for col in course_columns:
+        if col not in courses_df.columns:
+            courses_df[col] = None
+    courses_df = courses_df[course_columns]
+    
+    # Sort courses by APF_ID so all courses for a college are listed together
+    courses_df.sort_values(by="APF_ID", inplace=True)
+
+    # Save to Excel
+    timestamp = time.strftime("%b_%Y_%H%M%S").lower()
+    master_path = output_dir / f"master_database_relational_{timestamp}.xlsx"
+    
+    yield "Writing to Excel file with multiple sheets (this might take a minute)..."
+    with pd.ExcelWriter(master_path) as writer:
+        colleges_df.to_excel(writer, sheet_name="Colleges", index=False)
+        courses_df.to_excel(writer, sheet_name="Courses", index=False)
+        
+    yield f"\n[SUCCESS] Relational Master Database successfully generated!"
+    yield f"[SAVED] Saved to: {master_path}"
+    yield master_path
+
+if __name__ == "__main__":
+    out_dir = Path("c:/Users/Sharanya/Downloads/colleges_scraping/output")
+    for msg in merge_master_database(out_dir):
+        print(msg)

scraper/naac_downloader.py

@@ -1,9 +1,12 @@
 import argparse
-import sys
+import time
+import pandas as pd
 from pathlib import Path
 from playwright.sync_api import sync_playwright
+from thefuzz import process
+from typing import List
 
-def download_naac(output_dir: Path, headless: bool = True, log_fn=None) -> Path:
+def download_naac(output_dir: Path, selected_states: List[str] = None, headless: bool = True, log_fn=None) -> Path:
     def log(msg: str):
         if log_fn:
             log_fn(msg)
@@ -17,39 +20,102 @@ def download_naac(output_dir: Path, headless: bool = True, log_fn=None) -> Path:
             headless=headless,
             args=["--no-sandbox", "--disable-dev-shm-usage"]
         )
-        context = browser.new_context(accept_downloads=True)
-        page = context.new_page()
+        page = browser.new_page()
 
-        log("Navigate to NAAC Accreditation Status page...")
+        log("Navigate to NAAC Dashboard...")
         try:
-            page.goto("https://www.naac.gov.in/index.php/en/2-uncategorised/32-accreditation-status", wait_until="domcontentloaded", timeout=60_000)
+            page.goto("https://assessmentonline.naac.gov.in/public/index.php/hei_dashboard", wait_until="networkidle", timeout=60_000)
             
-            log("Waiting for NAAC Excel download link...")
-            # Click the exact link from the user's codegen
-            with page.expect_download(timeout=120_000) as download_info:
-                page.get_by_role("link", name="Institutions with valid").click(timeout=30_000)
+            log("Executing stealth API call to fetch all records instantly...")
+            js_code = """
+            async () => {
+                let token = document.querySelector('meta[name="csrf-token"]').getAttribute('content');
+                let url = 'https://assessmentonline.naac.gov.in/public/index.php/hei_dashboard?_token=' + token + '&inst_type=0&state=0&cycle=0&iiqa_status=5&date_range=&inst_name=&draw=1&start=0&length=15000';
+                let res = await fetch(url, {
+                    headers: {
+                        'X-Requested-With': 'XMLHttpRequest'
+                    }
+                });
+                return await res.json();
+            }
+            """
             
-            download = download_info.value
+            data = page.evaluate(js_code)
             
-            dest_path = output_dir / "naac_colleges_raw.xlsx"
-            log(f"Downloading NAAC file to {dest_path.name}...")
-            download.save_as(dest_path)
+            records = data.get("data", [])
+            total = data.get("recordsTotal", 0)
+            log(f"Successfully retrieved {len(records)} records (out of {total} total reported).")
             
-            log("NAAC download completed successfully!")
+            if not records:
+                raise Exception("API returned empty data.")
+                
+            log("Converting to Excel...")
+            df = pd.DataFrame(records)
+            
+            # Rename columns to match what master_merger.py expects
+            if "aishe_id" in df.columns:
+                df = df.rename(columns={"aishe_id": "Aishe-Id"})
+            if "grade" in df.columns:
+                df = df.rename(columns={"grade": "Current Grade"})
+
+            if selected_states and not df.empty and "state_name" in df.columns:
+                log(f"Filtering NAAC data for {len(selected_states)} selected states with fuzzy matching...")
+                
+                target_states_lower = [s.strip().lower() for s in selected_states]
+                
+                def is_selected_state(naac_state):
+                    if pd.isna(naac_state):
+                        return False
+                    naac_state = str(naac_state).strip().lower()
+                    
+                    # Fast exact/substring match
+                    for t in target_states_lower:
+                        if naac_state == t or naac_state in t or t in naac_state:
+                            return True
+                            
+                    # Known aliases (Orissa/Odisha, Uttarakhand/Uttaranchal)
+                    aliases = {
+                        "orissa": "odisha",
+                        "odisha": "orissa",
+                        "uttarakhand": "uttaranchal",
+                        "uttaranchal": "uttarakhand",
+                        "pondicherry": "puducherry",
+                        "puducherry": "pondicherry",
+                    }
+                    if naac_state in aliases and aliases[naac_state] in target_states_lower:
+                        return True
+                        
+                    # Fuzzy match fallback
+                    best_match, score = process.extractOne(naac_state, target_states_lower)
+                    return score >= 85
+                
+                df = df[df["state_name"].apply(is_selected_state)]
+                log(f"Filtered down to {len(df)} records matching selected states.")
+                
+            # Group states together (sort alphabetically by state, then name)
+            if "state_name" in df.columns:
+                df = df.sort_values(by=["state_name", "hei_name"], ascending=[True, True])
+                
+
+            timestamp = time.strftime("%b_%Y_%H%M%S").lower()
+            dest_path = output_dir / f"naac_colleges_{timestamp}.xlsx"
+            
+            df.to_excel(dest_path, index=False)
+            log(f"NAAC download completed successfully! Saved to {dest_path.name}")
             return dest_path
             
         except Exception as e:
             log(f"Error during NAAC download: {e}")
             raise
         finally:
-            context.close()
             browser.close()
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Download NAAC Accredited Institutions Data")
-    parser.add_argument("--output-dir", type=str, default="output/downloads", help="Directory to save downloaded files")
+    parser = argparse.ArgumentParser(description="Download NAAC Accredited Institutions Data (API Fast Path)")
+    parser.add_argument("--output-dir", type=str, default="output", help="Directory to save downloaded files")
     parser.add_argument("--headed", action="store_true", help="Run browser in headed mode (visible)")
+    parser.add_argument("--states", nargs="+", help="List of states to filter by", default=[])
     args = parser.parse_args()
 
     out_dir = Path(args.output_dir)
-    download_naac(out_dir, headless=not args.headed)
+    download_naac(out_dir, selected_states=args.states if args.states else None, headless=not args.headed)