feat: restore and update project database sync script

src/similarity_model/__init__.py

@@ -36,4 +36,8 @@ from .embedding_engine import (
 
 from .similarity_engine import (
     find_similar_projects
+)
+
+from .sync_projects import (
+    sync_projects
 )

src/similarity_model/sync_projects.py

@@ -0,0 +1,144 @@
+import json
+import logging
+import sys
+import pandas as pd
+
+from Data.database.sql_connector import engine
+from src.similarity_model.preprocessing import preprocess_dataset
+from src.similarity_model.embedding_engine import train_embedding_engine
+
+# =====================================================
+# Logging
+# =====================================================
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s | %(levelname)s | %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+def sync_projects():
+    logger.info("Initializing project synchronization service...")
+
+    # Verify database connection first
+    try:
+        # Since engine is a LazyEngine, calling connect() tests connection
+        with engine.connect() as conn:
+            logger.info("Database connection verified successfully.")
+    except Exception as exc:
+        logger.error(
+            "Unable to connect to the SQL database. Please ensure you are connected "
+            "to the university network / VPN and that your IP is whitelisted. Error: %s",
+            exc
+        )
+        sys.exit(1)
+
+    # 1. Fetch raw active projects
+    projects_query = """
+    SELECT *
+    FROM Projects
+    WHERE Status IN (
+        'Completed',
+        'UnderReview',
+        'In_Progress'
+    )
+    """
+    logger.info("Fetching raw active projects from 'Projects' table...")
+    projects_df = pd.read_sql(projects_query, engine)
+    logger.info(f"Loaded {len(projects_df)} active projects from database.")
+
+    # 2. Fetch existing preprocessed projects
+    logger.info("Fetching existing records from 'PreProcessed_Projects'...")
+    existing_df = pd.read_sql("SELECT id FROM PreProcessed_Projects", engine)
+    
+    allowed_ids = set(projects_df["Id"].tolist())
+    processed_ids = set(existing_df["id"].tolist())
+
+    changed = False
+
+    # 3. Remove projects no longer active or allowed
+    ids_to_remove = processed_ids - allowed_ids
+    if ids_to_remove:
+        logger.info(f"Found {len(ids_to_remove)} projects to remove (status changed or deleted).")
+        ids_str = ",".join(map(str, ids_to_remove))
+
+        with engine.begin() as conn:
+            conn.exec_driver_sql(
+                f"DELETE FROM PreProcessed_Projects WHERE id IN ({ids_str})"
+            )
+        logger.info(f"Successfully removed {len(ids_to_remove)} projects from 'PreProcessed_Projects'.")
+        changed = True
+
+    # 4. Filter for new projects
+    new_projects = projects_df[~projects_df["Id"].isin(processed_ids)].copy()
+
+    if len(new_projects) > 0:
+        logger.info(f"Found {len(new_projects)} new projects to preprocess and insert.")
+        
+        # Run preprocessing (cleaning, tokenization, feature extraction)
+        processed_df = preprocess_dataset(new_projects)
+
+        if len(processed_df) > 0:
+            # Map columns to target schema
+            cols_to_keep = [
+                "id",
+                "submittedat",
+                "project_title",
+                "studentnames",
+                "year",
+                "abstract",
+                "description",
+                "problemstatement",
+                "proposedsolution",
+                "objectives",
+                "full_content",
+                "clean_text",
+                "word_count",
+                "features"
+            ]
+            
+            # Ensure columns exist before filtering
+            for col in cols_to_keep:
+                if col not in processed_df.columns:
+                    processed_df[col] = ""
+
+            processed_df = processed_df[cols_to_keep]
+
+            processed_df = processed_df.rename(
+                columns={
+                    "submittedat": "submitted_at",
+                    "studentnames": "student_names",
+                    "problemstatement": "problem_statement",
+                    "proposedsolution": "proposed_solution"
+                }
+            )
+
+            # Stringify feature lists for SQL insertion
+            processed_df["features"] = processed_df["features"].apply(json.dumps)
+
+            # Insert into database
+            logger.info("Uploading preprocessed records to database...")
+            processed_df.to_sql(
+                "PreProcessed_Projects",
+                engine,
+                if_exists="append",
+                index=False
+            )
+            logger.info(f"Successfully processed and inserted {len(processed_df)} projects.")
+            changed = True
+        else:
+            logger.warning("No new projects remained after preprocessing filters.")
+    else:
+        logger.info("No new projects found.")
+
+    # 5. Rebuild local FAISS index & embeddings if anything changed
+    if changed:
+        logger.info("Changes detected. Rebuilding local embeddings and FAISS index...")
+        train_embedding_engine()
+        logger.info("Local embeddings and index updated successfully.")
+    else:
+        logger.info("No database changes detected. Embeddings remain in sync.")
+
+
+if __name__ == "__main__":
+    sync_projects()