Added Research paper feature, removed python slim in docker, ingested with playwrite

Dockerfile

@@ -1,9 +1,10 @@
-FROM python:3.11-slim
+FROM mcr.microsoft.com/playwright/python:v1.41.0-jammy
 
 # Set working directory
 WORKDIR /app
 
 # Install system dependencies
+# Playwright image already has browsers, but might need basic build tools for some python packages
 RUN apt-get update && apt-get install -y \
     gcc \
     build-essential \
@@ -14,7 +15,9 @@ RUN apt-get update && apt-get install -y \
 COPY requirements.txt .
 
 # Install Python dependencies
+# Use --no-cache-dir to keep image small
 RUN pip install --no-cache-dir -r requirements.txt
+RUN playwright install chromium
 
 # Copy application code
 COPY app ./app

app/config.py

@@ -78,8 +78,9 @@ class Settings(BaseSettings):
     APPWRITE_AI_COLLECTION_ID: str ="6985d84600311fce57c2"
     APPWRITE_DATA_COLLECTION_ID: str ="69845bcf00095c406439"
     APPWRITE_CLOUD_COLLECTION_ID: str ="cloud_articles"
-    APPWRITE_MAGAZINE_COLLECTION_ID: str ="69845cdd001712f4ac41"
-    APPWRITE_MEDIUM_COLLECTION_ID: str ="69845cf100332a456f74"
+    APPWRITE_MAGAZINE_COLLECTION_ID: str = "6798e285002a24aa3d63"
+    APPWRITE_MEDIUM_COLLECTION_ID: str = "679a0ec3001889753820"
+    APPWRITE_RESEARCH_COLLECTION_ID: str = "research_papers_v2"
     # Admin Alerting (Optional - Discord/Slack webhook URL)
     ADMIN_WEBHOOK_URL: Optional[str] = None
     

app/main.py

@@ -1,6 +1,19 @@
+import asyncio
+import sys
 from fastapi import FastAPI
 import warnings
 from fastapi.middleware.cors import CORSMiddleware
+
+# Windows-specific fix for Playwright + asyncio subprocesses
+if sys.platform == 'win32':
+    asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+    
+try:
+    import nest_asyncio
+    nest_asyncio.apply()
+except ImportError:
+    pass
+
 from contextlib import asynccontextmanager
 from app.config import settings
 # Suppress Pydantic V2 warnings from LangChain (known upstream issue)
@@ -22,27 +35,31 @@ from app.routes import news, search, analytics, subscription, admin, audio
 from app.services.scheduler import start_scheduler, shutdown_scheduler
 
 
+from app.services.browser_manager import browser_manager
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     """
     Application lifespan manager
     
     Handles startup and shutdown events for background tasks:
-    - Startup: Initialize and start APScheduler
-    - Shutdown: Gracefully stop all background jobs
+    - Startup: Initialize and start APScheduler, BrowserManager
+    - Shutdown: Gracefully stop all background jobs and BrowserManager
     """
-    # Startup: Start background scheduler
+    # Startup: Start background scheduler and browser
     print("=" * 60)
     print("🚀 Starting Segmento Pulse Backend...")
     start_scheduler()
+    await browser_manager.start()
     print("=" * 60)
     
     yield  # Application runs here
     
-    # Shutdown: Stop background scheduler
+    # Shutdown: Stop background scheduler and browser
     print("=" * 60)
     print("👋 Shutting down Segmento Pulse Backend...")
     shutdown_scheduler()
+    await browser_manager.shutdown()
     print("=" * 60)
 
 
@@ -72,6 +89,10 @@ app.include_router(subscription.router, tags=["Subscription"])
 app.include_router(admin.router, prefix="/api/admin", tags=["Admin"])
 app.include_router(audio.router, prefix="/api/audio", tags=["Audio"])
 
+# Phase 6: Research Papers
+from app.routes import research
+app.include_router(research.router, prefix="/api/research", tags=["Research"])
+
 # Phase 3: Engagement tracking
 from app.routes import engagement
 app.include_router(engagement.router, prefix="/api/engagement", tags=["Engagement"])

app/models.py

@@ -8,8 +8,9 @@ class Article(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
 
     title: str
+    id: Optional[str] = Field(None, alias="$id") # Appwrite ID
     description: Optional[str] = ""
-    url: HttpUrl
+    url: Optional[str] = None # Relaxed validation for compatibility
     # Direct mapping to DB fields (snake_case)
     image_url: Optional[str] = ""
     published_at: datetime

app/routes/audio.py

@@ -57,7 +57,7 @@ async def _find_article(appwrite, article_id: str, category: Optional[str] = Non
     # Try to find article in target collections
     for collection_id in target_collection_ids:
         try:
-            article = appwrite.tablesDB.get_row(
+            article = await appwrite.tablesDB.get_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=collection_id,
                 document_id=article_id
@@ -154,7 +154,7 @@ async def generate_audio_summary(request: AudioGenerationRequest):
                 "url_hash": url_hash  # Store full hash
             }
             
-            appwrite.tablesDB.create_row(
+            await appwrite.tablesDB.create_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=target_collection_id,
                 document_id=article_id,
@@ -162,7 +162,7 @@ async def generate_audio_summary(request: AudioGenerationRequest):
             )
             
             # Fetch it back
-            article = appwrite.tablesDB.get_row(
+            article = await appwrite.tablesDB.get_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=target_collection_id,
                 document_id=article_id
@@ -170,6 +170,7 @@ async def generate_audio_summary(request: AudioGenerationRequest):
             found_collection_id = target_collection_id
             print(f"✅ Created article in collection: {target_collection_id}")
 
+        
         # 2. Check if audio already exists
         if article.get('audio_url'):
             return AudioResponse(
@@ -179,8 +180,10 @@ async def generate_audio_summary(request: AudioGenerationRequest):
                 message="Audio already exists"
             )
             
+        from app.services.browser_manager import browser_manager
+
         # 3. Prepare text for summary
-        # FETCH FULL CONTENT using Trafilatura
+        # FETCH FULL CONTENT using Playwright (via BrowserManager) for SPA support
         import trafilatura
         
         # Determine URL to scrape
@@ -188,9 +191,20 @@ async def generate_audio_summary(request: AudioGenerationRequest):
         
         # Scrape
         print(f"Scraping content from: {target_view_url}")
-        downloaded = trafilatura.fetch_url(target_view_url)
-        extracted_text = trafilatura.extract(downloaded) if downloaded else None
         
+        # Use simple try-except loop for robustness, though BrowserManager handles most errors
+        extracted_text = None
+        try:
+            # 1. Fetch raw HTML using Headless Browser
+            raw_html = await browser_manager.get_content(target_view_url)
+            
+            # 2. Extract text from HTML
+            if raw_html:
+                extracted_text = trafilatura.extract(raw_html, include_comments=False)
+                
+        except Exception as e:
+            print(f"Scraping error: {e}")
+
         # Fallback to description if scraping fails
         if not extracted_text or len(extracted_text) < 100:
             print("Scraping failed or content too short, falling back to description")

app/routes/engagement.py

@@ -70,7 +70,8 @@ async def get_article_stats(article_id: str, category: Optional[str] = None):
             settings.APPWRITE_AI_COLLECTION_ID,
             settings.APPWRITE_DATA_COLLECTION_ID,
             settings.APPWRITE_MAGAZINE_COLLECTION_ID,
-            settings.APPWRITE_MEDIUM_COLLECTION_ID
+            settings.APPWRITE_MEDIUM_COLLECTION_ID,
+            settings.APPWRITE_RESEARCH_COLLECTION_ID
         ]
         
         for cid in fallback_collections:
@@ -82,7 +83,7 @@ async def get_article_stats(article_id: str, category: Optional[str] = None):
         
         for collection_id in target_collection_ids:
             try:
-                doc = appwrite_db.tablesDB.get_row(
+                doc = await appwrite_db.tablesDB.get_row(
                     database_id=settings.APPWRITE_DATABASE_ID,
                     collection_id=collection_id,
                     document_id=doc_id
@@ -106,7 +107,7 @@ async def get_article_stats(article_id: str, category: Optional[str] = None):
         return {
             "article_id": doc_id,
             "likes": doc.get('likes', 0),
-            "dislikes": doc.get('dislike', 0),
+            "dislikes": doc.get('dislikes') or doc.get('dislike', 0),
             "views": doc.get('views', 0),
             "success": True
         }
@@ -144,7 +145,7 @@ async def like_article(article_id: str, request: EngagementRequest = None):
         
         # 1. Try to get document from the TARGETED collection
         try:
-            doc = appwrite_db.tablesDB.get_row(
+            doc = await appwrite_db.tablesDB.get_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=target_collection_id,
                 document_id=doc_id
@@ -169,14 +170,14 @@ async def like_article(article_id: str, request: EngagementRequest = None):
                     
                     logger.info(f"📝 Creating new article with data: {new_doc}")
                     
-                    appwrite_db.tablesDB.create_row(
+                    await appwrite_db.tablesDB.create_row(
                         database_id=settings.APPWRITE_DATABASE_ID,
                         collection_id=target_collection_id,
                         document_id=doc_id,
                         data=new_doc
                     )
                     # Fetch the newly created doc
-                    doc = appwrite_db.tablesDB.get_row(
+                    doc = await appwrite_db.tablesDB.get_row(
                         database_id=settings.APPWRITE_DATABASE_ID,
                         collection_id=target_collection_id,
                         document_id=doc_id
@@ -194,7 +195,7 @@ async def like_article(article_id: str, request: EngagementRequest = None):
             
         new_likes = current_likes + 1
         
-        updated_doc = appwrite_db.tablesDB.update_row(
+        updated_doc = await appwrite_db.tablesDB.update_row(
             database_id=settings.APPWRITE_DATABASE_ID,
             collection_id=target_collection_id,
             document_id=doc_id,
@@ -234,7 +235,7 @@ async def dislike_article(article_id: str, request: EngagementRequest = None):
             target_collection_id = appwrite_db.get_collection_id(request.category)
         
         try:
-            doc = appwrite_db.tablesDB.get_row(
+            doc = await appwrite_db.tablesDB.get_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=target_collection_id,
                 document_id=doc_id
@@ -255,13 +256,13 @@ async def dislike_article(article_id: str, request: EngagementRequest = None):
                         "views": 0,
                         "category": request.category or "wildcard"
                     }
-                    appwrite_db.tablesDB.create_row(
+                    await appwrite_db.tablesDB.create_row(
                         database_id=settings.APPWRITE_DATABASE_ID,
                         collection_id=target_collection_id,
                         document_id=doc_id,
                         data=new_doc
                     )
-                    doc = appwrite_db.tablesDB.get_row(
+                    doc = await appwrite_db.tablesDB.get_row(
                         database_id=settings.APPWRITE_DATABASE_ID,
                         collection_id=target_collection_id,
                         document_id=doc_id
@@ -271,23 +272,33 @@ async def dislike_article(article_id: str, request: EngagementRequest = None):
             else:
                 raise HTTPException(status_code=404, detail=f"Article not found in {target_collection_id}")
         
-        current_dislikes = doc.get('dislike', 0)
+        current_dislikes = doc.get('dislikes') or doc.get('dislike', 0)
         if current_dislikes is None: current_dislikes = 0
             
         new_dislikes = current_dislikes + 1
         
-        updated_doc = appwrite_db.tablesDB.update_row(
+        # Schema Compat: Research uses 'dislikes' (plural), others use 'dislike' (singular)
+        update_data = {}
+        if target_collection_id == settings.APPWRITE_RESEARCH_COLLECTION_ID:
+            update_data = {"dislikes": new_dislikes}
+        else:
+            update_data = {"dislike": new_dislikes}
+
+        updated_doc = await appwrite_db.tablesDB.update_row(
             database_id=settings.APPWRITE_DATABASE_ID,
             collection_id=target_collection_id,
             document_id=doc_id,
-            data={"dislike": new_dislikes}
+            data=update_data
         )
         
+        # Return result (normalize key)
+        final_dislikes = updated_doc.get('dislikes') if 'dislikes' in updated_doc else updated_doc.get('dislike')
+        
         logger.info(f"👎 Article {doc_id[:8]}... disliked (total: {updated_doc['dislike']})")
         
         return {
             "article_id": doc_id,
-            "dislikes": updated_doc['dislike'],
+            "dislikes": final_dislikes,
             "success": True
         }
         
@@ -316,7 +327,7 @@ async def track_view(article_id: str, request: EngagementRequest = None):
             target_collection_id = appwrite_db.get_collection_id(request.category)
         
         try:
-            doc = appwrite_db.tablesDB.get_row(
+            doc = await appwrite_db.tablesDB.get_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=target_collection_id,
                 document_id=doc_id
@@ -336,13 +347,13 @@ async def track_view(article_id: str, request: EngagementRequest = None):
                         "views": 0,
                         "category": request.category or "wildcard"
                     }
-                    appwrite_db.tablesDB.create_row(
+                    await appwrite_db.tablesDB.create_row(
                         database_id=settings.APPWRITE_DATABASE_ID,
                         collection_id=target_collection_id,
                         document_id=doc_id,
                         data=new_doc
                     )
-                    doc = appwrite_db.tablesDB.get_row(
+                    doc = await appwrite_db.tablesDB.get_row(
                         database_id=settings.APPWRITE_DATABASE_ID,
                         collection_id=target_collection_id,
                         document_id=doc_id
@@ -359,7 +370,7 @@ async def track_view(article_id: str, request: EngagementRequest = None):
             
         new_views = current_views + 1
         
-        updated_doc = appwrite_db.tablesDB.update_row(
+        updated_doc = await appwrite_db.tablesDB.update_row(
             database_id=settings.APPWRITE_DATABASE_ID,
             collection_id=target_collection_id,
             document_id=doc_id,
@@ -414,11 +425,11 @@ async def get_trending_articles(
             collection_id = settings.APPWRITE_COLLECTION_ID
         
         # Query articles, sorted by views (descending)
-        response = appwrite_db.tablesDB.list_rows(
+        response = await appwrite_db.tablesDB.list_rows(
             database_id=settings.APPWRITE_DATABASE_ID,
             collection_id=collection_id,
             queries=[
-                Query.greater_than('publishedAt', cutoff),
+                Query.greater_than('published_at', cutoff),
                 Query.order_desc('views'),
                 Query.limit(limit)
             ]
@@ -431,7 +442,7 @@ async def get_trending_articles(
         for article in articles:
             views = article.get('views', 0)
             likes = article.get('likes', 0)
-            dislikes = article.get('dislike', 0)
+            dislikes = article.get('dislikes') or article.get('dislike', 0)
             article['engagement_score'] = views + (likes * 5) - (dislikes * 3)
         
         # Sort by engagement score
@@ -482,7 +493,7 @@ async def get_popular_cloud_articles(provider: Optional[str] = None, limit: int
         if provider:
             queries.insert(0, Query.equal('provider', provider))
         
-        response = appwrite_db.tablesDB.list_rows(
+        response = await appwrite_db.tablesDB.list_rows(
             database_id=settings.APPWRITE_DATABASE_ID,
             collection_id=settings.APPWRITE_CLOUD_COLLECTION_ID,
             queries=queries

app/routes/research.py

@@ -0,0 +1,59 @@
+
+from fastapi import APIRouter, HTTPException
+from typing import Dict, Any, Optional
+from app.services.appwrite_db import get_appwrite_db
+from app.config import settings
+import logging
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+@router.get("/{paper_id}")
+async def get_research_paper(paper_id: str):
+    """
+    Get a single research paper by ID.
+    """
+    try:
+        appwrite_db = get_appwrite_db()
+        
+        # Try to find by ID in research collection
+        try:
+            doc = await appwrite_db.tablesDB.get_row(
+                database_id=settings.APPWRITE_DATABASE_ID,
+                collection_id=settings.APPWRITE_RESEARCH_COLLECTION_ID,
+                document_id=paper_id
+            )
+            
+            # Helper to map fields
+            if doc:
+                return {
+                    "success": True,
+                    "paper": {
+                        "$id": doc.get('$id'),
+                        "title": doc.get('title'),
+                        "summary": doc.get('summary'),
+                        "authors": doc.get('authors'),
+                        "published_at": doc.get('published_at'),
+                        "pdf_url": doc.get('pdf_url'),
+                        "category": doc.get('category'),
+                        "likes": doc.get('likes', 0),
+                        "views": doc.get('views', 0),
+                        "text_summary": doc.get('summary'), # Compat
+                        "description": doc.get('summary'), # Compat
+                        "url": doc.get('pdf_url'), # Compat
+                        "image_url": doc.get('image_url'),
+                        "id": doc.get('$id'), # Compat
+                        "source": "ArXiv"
+                    }
+                }
+        except Exception as e:
+            logger.warning(f"Paper {paper_id} not found: {e}")
+            pass
+
+        raise HTTPException(status_code=404, detail="Research paper not found")
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error fetching paper {paper_id}: {e}")
+        raise HTTPException(status_code=500, detail=str(e))

app/services/appwrite_db.py

@@ -90,21 +90,21 @@ class AppwriteDatabase:
                 def __init__(self, db_service):
                     self.db = db_service
                 
-                def create_row(self, *args, **kwargs):
-                    return self.db.create_document(*args, **kwargs)
+                async def create_row(self, *args, **kwargs):
+                    return await asyncio.to_thread(self.db.create_document, *args, **kwargs)
                     
-                def get_row(self, *args, **kwargs):
-                    return self.db.get_document(*args, **kwargs)
+                async def get_row(self, *args, **kwargs):
+                    return await asyncio.to_thread(self.db.get_document, *args, **kwargs)
                 
-                def list_rows(self, *args, **kwargs):
-                    return self.db.list_documents(*args, **kwargs)
+                async def list_rows(self, *args, **kwargs):
+                    return await asyncio.to_thread(self.db.list_documents, *args, **kwargs)
 
-                def delete_row(self, *args, **kwargs):
+                async def delete_row(self, *args, **kwargs):
                      # Mapping delete_document -> delete_row if needed
-                    return self.db.delete_document(*args, **kwargs)
+                    return await asyncio.to_thread(self.db.delete_document, *args, **kwargs)
 
-                def update_row(self, *args, **kwargs):
-                    return self.db.update_document(*args, **kwargs)
+                async def update_row(self, *args, **kwargs):
+                    return await asyncio.to_thread(self.db.update_document, *args, **kwargs)
             
             self.tablesDB = TablesDBWrapper(self.databases)
             
@@ -140,7 +140,11 @@ class AppwriteDatabase:
         if cat.startswith('cloud-'):
             return settings.APPWRITE_CLOUD_COLLECTION_ID
             
-        # 3. Data Vertical (Security, Governance, etc.)
+        # 3. Research Vertical (New)
+        if cat == 'research' or cat.startswith('research-'):
+            return settings.APPWRITE_RESEARCH_COLLECTION_ID
+            
+        # 4. Data Vertical (Security, Governance, etc.)
         if cat.startswith('data-') or cat.startswith('business-') or cat == 'customer-data-platform':
             return settings.APPWRITE_DATA_COLLECTION_ID
             
@@ -193,29 +197,48 @@ class AppwriteDatabase:
                 'published_at',
                 'source',
                 'category',
-                'url_hash'
+                'url_hash',
+                'authors',   # Research specific
+                'pdf_url',   # Research specific
+                'summary'    # Research specific (mapped to description)
             ]
             
             # Query with projection
-            response = self.tablesDB.list_rows(
+            queries = [
+                Query.order_desc('published_at'),  # Uses index!
+                Query.limit(limit),
+                Query.offset(offset)
+            ]
+            
+            # Apply category filter ONLY if it's not the root 'research' category
+            # (Because 'research' collection only contains research papers, so no filter = All Research)
+            if category != 'research':
+                queries.insert(0, Query.equal('category', category))
+            
+            response = await self.tablesDB.list_rows(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=target_collection_id,
-                queries=[
-                    Query.equal('category', category),
-                    Query.order_desc('published_at'),  # Uses index!
-                    Query.limit(limit),
-                    Query.offset(offset)
-                ]
+                queries=queries
             )
             
             # Convert Appwrite documents to Article dictionaries
             articles = []
             for doc in response['documents']:
                 try:
+                    # Smart Mapping for Research Papers
+                    description = doc.get('description', '')
+                    if not description and doc.get('summary'):
+                         description = doc.get('summary')
+                         
+                    url = doc.get('url', '')
+                    if not url and doc.get('pdf_url'):
+                        url = doc.get('pdf_url')
+                        
                     article = {
+                        '$id': doc.get('$id'), # Ensure $id is passed!
                         'title': doc.get('title'),
-                        'description': doc.get('description', ''),
-                        'url': doc.get('url'),
+                        'description': description,
+                        'url': url,
                         'image_url': doc.get('image_url', ''),
                         'publishedAt': doc.get('published_at'),
                         'published_at': doc.get('published_at'), # Standard schema field
@@ -223,7 +246,9 @@ class AppwriteDatabase:
                         'category': doc.get('category'),
                         'likes': doc.get('likes', 0),
                         'dislikes': doc.get('dislike', 0),
-                        'views': doc.get('views', 0)
+                        'views': doc.get('views', 0),
+                        'author': doc.get('authors') # Map authors to author (singular for compat)
+                        # 'authors': doc.get('authors') # Keep plural if needed
                     }
                     articles.append(article)
                 except Exception as e:
@@ -280,7 +305,7 @@ class AppwriteDatabase:
 
             logger.info(f"🚀 [QUERY] Executing query on Collection: {target_collection_id}")
             
-            response = self.tablesDB.list_rows(
+            response = await self.tablesDB.list_rows(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=target_collection_id,
                 queries=queries
@@ -419,7 +444,7 @@ class AppwriteDatabase:
                     # Only the 'image' field uses legacy naming
 
                 # Try to create document
-                self.tablesDB.create_row(
+                await self.tablesDB.create_row(
                     database_id=settings.APPWRITE_DATABASE_ID,
                     collection_id=target_collection_id,
                     document_id=doc_id, # Truncated ID
@@ -488,7 +513,7 @@ class AppwriteDatabase:
             cutoff_date = (datetime.now() - timedelta(days=days)).isoformat()
             
             # Query old articles
-            response = self.tablesDB.list_rows(
+            response = await self.tablesDB.list_rows(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=settings.APPWRITE_COLLECTION_ID,
                 queries=[
@@ -500,7 +525,7 @@ class AppwriteDatabase:
             deleted_count = 0
             for doc in response['documents']:
                 try:
-                    self.tablesDB.delete_row(
+                    await self.tablesDB.delete_row(
                         database_id=settings.APPWRITE_DATABASE_ID,
                         collection_id=settings.APPWRITE_COLLECTION_ID,
                         document_id=doc['$id']
@@ -553,7 +578,7 @@ class AppwriteDatabase:
             # Using MD5 of email ensures idempotent writes (same email = same ID)
             doc_id = hashlib.md5(email.lower().encode()).hexdigest()
 
-            self.tablesDB.create_row(
+            await self.tablesDB.create_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=settings.APPWRITE_SUBSCRIBERS_COLLECTION_ID,
                 document_id=doc_id,
@@ -581,7 +606,7 @@ class AppwriteDatabase:
             return None
             
         try:
-            documents = self.tablesDB.list_rows(
+            documents = await self.tablesDB.list_rows(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=settings.APPWRITE_SUBSCRIBERS_COLLECTION_ID,
                 queries=[Query.equal("email", email)]
@@ -617,7 +642,7 @@ class AppwriteDatabase:
             if "Monthly" in preferences: data["sub_monthly"] = preferences["Monthly"]
             
             # Note: tablesDB wrapper now has update_row
-            self.tablesDB.update_row(
+            await self.tablesDB.update_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=settings.APPWRITE_SUBSCRIBERS_COLLECTION_ID,
                 document_id=doc_id,
@@ -633,7 +658,7 @@ class AppwriteDatabase:
     async def get_subscriber_by_token(self, token: str) -> Optional[Dict]:
         """Get subscriber by unsubscribe token"""
         try:
-            documents = self.tablesDB.list_rows(
+            documents = await self.tablesDB.list_rows(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=settings.APPWRITE_SUBSCRIBERS_COLLECTION_ID,
                 queries=[Query.equal("token", token)]
@@ -653,7 +678,7 @@ class AppwriteDatabase:
             return False
             
         try:
-            self.tablesDB.update_row(
+            await self.tablesDB.update_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=collection_id,
                 document_id=document_id,
@@ -692,7 +717,7 @@ class AppwriteDatabase:
                 
             data = {field: is_active}
             
-            self.tablesDB.update_row(
+            await self.tablesDB.update_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=settings.APPWRITE_SUBSCRIBERS_COLLECTION_ID,
                 document_id=subscriber['$id'],
@@ -719,7 +744,7 @@ class AppwriteDatabase:
             
             data = {"isActive": subscribed}
             
-            self.tablesDB.update_row(
+            await self.tablesDB.update_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=settings.APPWRITE_SUBSCRIBERS_COLLECTION_ID,
                 document_id=subscriber['$id'],
@@ -750,7 +775,7 @@ class AppwriteDatabase:
             # Store in UTC ISO format
             utc_now = datetime.now(pytz.UTC).isoformat()
             
-            self.tablesDB.update_row(
+            await self.tablesDB.update_row(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=settings.APPWRITE_SUBSCRIBERS_COLLECTION_ID,
                 document_id=subscriber['$id'],
@@ -794,7 +819,7 @@ class AppwriteDatabase:
             # 1. Must be globally active (isActive=true)
             # 2. Must be subscribed to specific preference (sub_X=true)
             
-            documents = self.tablesDB.list_rows(
+            documents = await self.tablesDB.list_rows(
                 database_id=settings.APPWRITE_DATABASE_ID,
                 collection_id=settings.APPWRITE_SUBSCRIBERS_COLLECTION_ID,
                 queries=[

app/services/audio_service.py

@@ -5,20 +5,20 @@ import hashlib
 from typing import Optional, Dict
 from datetime import datetime
 import edge_tts
-from groq import AsyncGroq
+from groq import Groq
 from app.services.appwrite_db import get_appwrite_db
 from app.config import settings
 
 class AudioService:
     def __init__(self):
-        self.groq_client = AsyncGroq(api_key=settings.GROQ_API_KEY)
-        self.voice = "en-US-AndrewNeural"  # Professional male voice
-        # self.voice = "en-US-AvaNeural" # Professional female voice
+        # Use Sync client to avoid 'unknown async library' errors with anyio/Proactor on Windows
+        self.groq_client = Groq(api_key=settings.GROQ_API_KEY)
+        self.voice = "en-US-AndrewNeural"
         
-    async def generate_summary(self, content: str) -> str:
-        """Generate a concise audio-friendly summary using Groq"""
+    def _generate_summary_sync(self, content: str) -> str:
+        """Synchronous wrapper for Groq API"""
         try:
-            chat_completion = await self.groq_client.chat.completions.create(
+            chat_completion = self.groq_client.chat.completions.create(
                 messages=[
                     {
                         "role": "system",
@@ -29,21 +29,68 @@ class AudioService:
                         "content": content,
                     }
                 ],
-                model="llama-3.3-70b-versatile", # Default efficient model
+                model="llama-3.3-70b-versatile",
                 temperature=0.5,
                 max_tokens=150,
             )
             return chat_completion.choices[0].message.content.strip()
+        except Exception as e:
+            print(f"Error in Groq Sync API: {e}")
+            raise e
+
+    async def generate_summary(self, content: str) -> str:
+        """Generate a concise audio-friendly summary using Groq (Threaded)"""
+        try:
+            # Run blocking sync IO in a separate thread to keep event loop free
+            return await asyncio.to_thread(self._generate_summary_sync, content)
         except Exception as e:
             print(f"Error generating summary: {e}")
             return ""
 
-    async def generate_audio(self, text: str, output_path: str) -> bool:
-        """Generate audio file from text using Edge TTS"""
+    def _generate_audio_subprocess(self, text: str, output_path: str) -> bool:
+        """Helper to run edge-tts in a separate process for stability"""
+        import subprocess
+        import sys
+        import tempfile
+        
+        temp_file_path = None
         try:
-            communicate = edge_tts.Communicate(text, self.voice)
-            await communicate.save(output_path)
+            # Create temp file with utf-8 encoding
+            with tempfile.NamedTemporaryFile(mode='w', delete=False, encoding='utf-8') as f:
+                f.write(text)
+                temp_file_path = f.name
+                
+            # Construct command: python -m edge_tts --file <temp> --write-media <out> --voice <voice>
+            cmd = [
+                sys.executable, "-m", "edge_tts",
+                "--file", temp_file_path,
+                "--write-media", output_path,
+                "--voice", self.voice
+            ]
+            
+            # Run blocking subprocess (safe in thread)
+            result = subprocess.run(cmd, check=True, capture_output=True, text=True)
             return True
+            
+        except subprocess.CalledProcessError as e:
+            print(f"Error running edge-tts subprocess: {e.stderr}")
+            return False
+        except Exception as e:
+            print(f"General error in audio subprocess: {e}")
+            return False
+        finally:
+            # Cleanup temp file
+            if temp_file_path and os.path.exists(temp_file_path):
+                try:
+                    os.unlink(temp_file_path)
+                except:
+                    pass
+
+    async def generate_audio(self, text: str, output_path: str) -> bool:
+        """Generate audio file from text using Edge TTS (Subprocess)"""
+        try:
+            # Run the subprocess wrapper in a thread to keep main loop free
+            return await asyncio.to_thread(self._generate_audio_subprocess, text, output_path)
         except Exception as e:
             print(f"Error generating audio: {e}")
             return False
@@ -62,7 +109,9 @@ class AudioService:
             # Use InputFile for file upload
             from appwrite.input_file import InputFile
             
-            result = appwrite.storage.create_file(
+            # Run blocking storage upload in a thread
+            result = await asyncio.to_thread(
+                appwrite.storage.create_file,
                 bucket_id=bucket_id,
                 file_id='unique()',
                 file=InputFile.from_path(file_path)

app/services/browser_manager.py

@@ -0,0 +1,132 @@
+import asyncio
+import logging
+from typing import Optional
+from playwright.async_api import async_playwright, Browser, Playwright, BrowserContext
+import random
+import sys
+
+# Critical Fix for Windows: Force ProactorEventLoop for subprocess support
+if sys.platform == 'win32':
+    asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# List of realistic User-Agents
+USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
+]
+
+class BrowserManager:
+    _instance = None
+    
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(BrowserManager, cls).__new__(cls)
+            cls._instance._initialized = False
+        return cls._instance
+    
+    def __init__(self):
+        if self._initialized:
+            return
+            
+        self.playwright: Optional[Playwright] = None
+        self.browser: Optional[Browser] = None
+        # Limit concurrent scraping operations to prevent OOM
+        # 3 concurrent tabs is a safe starting point for a typical container (1-2GB RAM)
+        self.semaphore = asyncio.Semaphore(3) 
+        self._initialized = True
+        logger.info("BrowserManager initialized (Semaphore: 3)")
+
+    async def start(self):
+        """Initialize the global browser instance"""
+        if self.browser:
+            return
+
+        try:
+            logger.info("Starting Playwright...")
+            self.playwright = await async_playwright().start()
+            self.browser = await self.playwright.chromium.launch(
+                headless=True,
+                # Add arguments to improve stability in container environments
+                args=[
+                    '--no-sandbox',
+                    '--disable-setuid-sandbox',
+                    '--disable-dev-shm-usage', # Overcome limited /dev/shm size
+                    '--disable-gpu' # Not needed for headless
+                ]
+            )
+            logger.info("Global Browser Instance Started successfully")
+        except Exception as e:
+            logger.error(f"Failed to start Playwright: {e}")
+            raise
+
+    async def shutdown(self):
+        """Gracefully close the global browser instance"""
+        logger.info("Shutting down BrowserManager...")
+        if self.browser:
+            await self.browser.close()
+            self.browser = None
+        
+        if self.playwright:
+            await self.playwright.stop()
+            self.playwright = None
+            
+        logger.info("BrowserManager shutdown complete")
+
+    async def get_content(self, url: str) -> Optional[str]:
+        """
+        Fetch dynamic content using a fresh context from the shared browser.
+        Controlled by semaphore to prevention resource exhaustion.
+        """
+        if not self.browser:
+            logger.error("Browser not initialized! Call start() first.")
+            return None
+
+        async with self.semaphore:
+            context: Optional[BrowserContext] = None
+            page = None
+            try:
+                # Create a lightweight context (incognito-like)
+                # Randomize user agent for basic anti-bot evasion
+                context = await self.browser.new_context(
+                    user_agent=random.choice(USER_AGENTS),
+                    viewport={'width': 1920, 'height': 1080},
+                    java_script_enabled=True
+                )
+                
+                page = await context.new_page()
+                
+                logger.info(f"Navigating to {url}")
+                # "domcontentloaded" is faster than "networkidle" and usually sufficient for text
+                await page.goto(url, wait_until="domcontentloaded", timeout=15000)
+                
+                # Wait a bit for JS hydration (React/Next.js)
+                await page.wait_for_timeout(2000)
+                
+                content = await page.content()
+                logger.info(f"Successfully scraped {len(content)} bytes from {url}")
+                return content
+                
+            except Exception as e:
+                logger.error(f"Scraping failed for {url}: {e}")
+                return None
+                
+            finally:
+                # CRITICAL: Always close context to free memory
+                if page:
+                    try:
+                        await page.close()
+                    except:
+                        pass
+                        
+                if context:
+                    try:
+                        await context.close()
+                    except:
+                        pass
+
+# Global Singleton Instance
+browser_manager = BrowserManager()

app/services/research_aggregator.py

@@ -0,0 +1,203 @@
+
+import asyncio
+import arxiv
+import logging
+from datetime import datetime
+from typing import List, Dict, Any
+from appwrite.query import Query
+
+from app.config import settings
+from app.services.appwrite_db import get_appwrite_db
+
+# Configure logger
+logger = logging.getLogger(__name__)
+
+# Category Mapping: ArXiv -> Segmento Pulse
+CATEGORY_MAPPING = {
+    # AI & Machine Learning
+    "cs.AI": "research-ai",
+    "cs.LG": "research-ml",  # Machine Learning
+    "cs.CL": "research-ai",  # Computation and Language (NLP)
+    "cs.CV": "research-ai",  # Computer Vision
+    "cs.NE": "research-ml",  # Neural and Evolutionary Computing
+    
+    # Cloud & Distributed Computing
+    "cs.DC": "research-cloud", # Distributed, Parallel, and Cluster Computing
+    "cs.OS": "research-cloud", # Operating Systems
+    "cs.NI": "research-cloud", # Networking and Internet Architecture
+    
+    # Data & Databases
+    "cs.DB": "research-data",  # Databases
+    "cs.DS": "research-data",  # Data Structures and Algorithms
+    "cs.IR": "research-data",  # Information Retrieval
+    "cs.CR": "research-data",  # Cryptography and Security (Data Security)
+}
+
+# Reverse mapping for display/debugging
+INTERNAL_TO_DISPLAY = {
+    "research-ai": "Artificial Intelligence",
+    "research-ml": "Machine Learning",
+    "research-cloud": "Cloud Computing",
+    "research-data": "Data Engineering"
+}
+
+class ResearchAggregator:
+    """
+    Fetches research papers from ArXiv and stores them in Appwrite.
+    """
+    def __init__(self):
+        self.client = arxiv.Client(
+            page_size=20,
+            delay_seconds=3.0,
+            num_retries=3
+        )
+
+    async def fetch_and_process_daily_papers(self):
+        """
+        Main entry point: Fetches papers for all mapped categories.
+        """
+        logger.info("🔬 [RESEARCH AGGREGATOR] Starting daily fetch...")
+        
+        total_fetched = 0
+        total_saved = 0
+        
+        # Group ArXiv categories by our internal buckets to query efficiently
+        # Actually, query arXiv by category groups
+        # We can query multiple categories at once: 'cat:cs.AI OR cat:cs.LG'
+        
+        # 1. Build Query Strings
+        # AI Group
+        ai_query = "cat:cs.AI OR cat:cs.LG OR cat:cs.CL OR cat:cs.CV OR cat:cs.NE"
+        # Cloud Group
+        cloud_query = "cat:cs.DC OR cat:cs.OS OR cat:cs.NI"
+        # Data Group
+        data_query = "cat:cs.DB OR cat:cs.DS OR cat:cs.IR OR cat:cs.CR"
+        
+        queries = [
+            ("AI/ML", ai_query),
+            ("Cloud", cloud_query),
+            ("Data", data_query)
+        ]
+        
+        for group_name, query_str in queries:
+            logger.info(f"   🔍 Querying ArXiv for {group_name}...")
+            
+            # Construct search
+            search = arxiv.Search(
+                query=query_str,
+                max_results=30, # Limit per group to avoid spam
+                sort_by=arxiv.SortCriterion.SubmittedDate,
+                sort_order=arxiv.SortOrder.Descending
+            )
+            
+            # Execute sync generator in async context (blocking? ArXiv lib is sync)
+            # We should ideally run this in a thread executor if it blocks too long, 
+            # but for 30 items it's okay for background job.
+            
+            results = list(self.client.results(search))
+            logger.info(f"   found {len(results)} papers for {group_name}")
+            
+            for paper in results:
+                total_fetched += 1
+                processed_paper = self._process_paper(paper)
+                if processed_paper:
+                   saved = await self._save_paper(processed_paper)
+                   if saved:
+                       total_saved += 1
+                       
+        logger.info(f"✅ [RESEARCH AGGREGATOR] Completed. Fetched: {total_fetched}, Saved: {total_saved}")
+        return total_saved
+
+    def _process_paper(self, paper: arxiv.Result) -> Dict[str, Any]:
+        """
+        Transforms ArXiv result into our Appwrite Schema.
+        """
+        # 1. Determine Primary Category
+        # ArXiv results have .categories list. We take the first one that matches our mapping.
+        primary_cat = paper.categories[0] 
+        internal_cat = CATEGORY_MAPPING.get(primary_cat)
+        
+        if not internal_cat:
+            # Fallback: check other categories
+            for cat in paper.categories:
+                if cat in CATEGORY_MAPPING:
+                    internal_cat = CATEGORY_MAPPING[cat]
+                    primary_cat = cat
+                    break
+        
+        if not internal_cat:
+            return None # Skip if not in our scope
+            
+        # 2. Format Data
+        return {
+            "paper_id": self._get_short_id(paper.entry_id),
+            "title": paper.title.replace("\n", " ").strip(),
+            "summary": paper.summary.replace("\n", " ").strip(),
+            "authors": [a.name for a in paper.authors],
+            "published_at": paper.published.isoformat(),
+            "pdf_url": paper.pdf_url,
+            "url": paper.pdf_url, # COMPATIBILITY: Map pdf_url to url for frontend/models
+            "category": internal_cat,     # research-ai
+            "original_category": primary_cat, # cs.AI
+            "sub_category": INTERNAL_TO_DISPLAY.get(internal_cat, "Research"), # Friendly name
+            "source": "arXiv"
+        }
+
+    def _get_short_id(self, entry_id: str) -> str:
+        # ArXiv IDs are like http://arxiv.org/abs/2101.12345v1
+        # We want "2101.12345v1"
+        return entry_id.split("/")[-1]
+
+    async def _save_paper(self, paper_data: Dict[str, Any]) -> bool:
+        """
+        Saves to Appwrite if not exists.
+        """
+        appwrite = get_appwrite_db()
+        if not appwrite.initialized:
+            logger.error("Appwrite not initialized")
+            return False
+            
+        try:
+            # 1. Create (Atomic)
+            # We rely on the unique index on paper_id to throw 409 Conflict if exists.
+            # This avoids the "Check-Then-Act" race condition.
+
+            # 2. Create
+            # We need to flatten authors list to string because Appwrite String array 
+            # logic depends on how we created schema. 
+            # Wait, `authors` attribute in script was `type: string, size: 5000`. 
+            # It's a single string, not array.
+            # So we join them.
+            paper_data['authors'] = ", ".join(paper_data['authors'])
+            
+            await appwrite.tablesDB.create_row(
+                database_id=settings.APPWRITE_DATABASE_ID,
+                collection_id=settings.APPWRITE_RESEARCH_COLLECTION_ID,
+                document_id="unique()",
+                data=paper_data
+            )
+            logger.info(f"   💾 Saved: {paper_data['title'][:50]}...")
+            return True
+            
+        except Exception as e:
+            # Check for 409 Conflict (Appwrite throws Exception with message)
+            if "Document already exists" in str(e) or "409" in str(e):
+                 logger.debug(f"   ⏭️  Skipping duplicate (Atomic): {paper_data['paper_id']}")
+                 return False
+            
+            logger.error(f"   ❌ Error saving paper {paper_data['paper_id']}: {e}")
+            return False
+
+# Standalone run for testing
+if __name__ == "__main__":
+    import sys
+    import os
+    
+    # Add project root to path
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../')))
+    
+    async def run():
+        agg = ResearchAggregator()
+        await agg.fetch_and_process_daily_papers()
+    
+    asyncio.run(run())

app/services/scheduler.py

@@ -14,6 +14,7 @@ from app.services.news_aggregator import NewsAggregator
 from app.services.appwrite_db import get_appwrite_db
 from app.services.cache_service import CacheService
 from app.services.adaptive_scheduler import get_adaptive_scheduler, AdaptiveScheduler
+from app.services.research_aggregator import ResearchAggregator
 from app.config import settings
 
 # Setup logging
@@ -210,6 +211,27 @@ async def fetch_all_news():
         adaptive.print_summary()
 
 
+async def fetch_daily_research():
+    """
+    Background Job: Fetch Research Papers from ArXiv
+    Runs daily at 02:00 IST
+    """
+    logger.info("═" * 80)
+    logger.info("🔬 [RESEARCH FETCHER] Starting daily research fetch...")
+    logger.info("🕐 Start Time: %s", datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
+    logger.info("═" * 80)
+    
+    try:
+        aggregator = ResearchAggregator()
+        saved_count = await aggregator.fetch_and_process_daily_papers()
+        logger.info(f"✅ [RESEARCH FETCHER] Completed. Saved {saved_count} new papers.")
+        
+    except Exception as e:
+        logger.error(f"❌ [RESEARCH FETCHER] Failed: {e}", exc_info=True)
+    
+    logger.info("═" * 80)
+
+
 async def fetch_and_validate_category(category: str) -> tuple:
     """
     Fetch and validate articles for a single category
@@ -342,7 +364,7 @@ async def cleanup_old_news():
                 # -------------------------------------------------------------
                 # 1. SMART CHECK: "Hey collection, do you have old data?"
                 # -------------------------------------------------------------
-                check_response = appwrite_db.tablesDB.list_rows(
+                check_response = await appwrite_db.tablesDB.list_rows(
                     database_id=settings.APPWRITE_DATABASE_ID,
                     collection_id=collection_id,
                     queries=[
@@ -364,7 +386,7 @@ async def cleanup_old_news():
                 
                 while True:
                     # Query old articles (Batch of 500)
-                    response = appwrite_db.tablesDB.list_rows(
+                    response = await appwrite_db.tablesDB.list_rows(
                         database_id=settings.APPWRITE_DATABASE_ID,
                         collection_id=collection_id,
                         queries=[
@@ -386,7 +408,7 @@ async def cleanup_old_news():
                         try:
                             # This deletes the FULL DOCUMENT (Row) including all attributes
                             # (published_at, url, image, likes, views, dislikes, etc.)
-                            appwrite_db.tablesDB.delete_row(
+                            await appwrite_db.tablesDB.delete_row(
                                 database_id=settings.APPWRITE_DATABASE_ID,
                                 collection_id=collection_id,
                                 document_id=doc['$id']
@@ -521,6 +543,17 @@ def start_scheduler():
     logger.info("")
     logger.info(f"✅ Job #{job_counter} Registered: 📊 Monthly Newsletter")
     
+    # Research Papers Job (Daily at 02:00 IST)
+    scheduler.add_job(
+        fetch_daily_research,
+        trigger=CronTrigger(hour=2, minute=0, timezone=IST),
+        id='fetch_research_papers',
+        name='Research Fetcher (Daily 02:00 IST)',
+        replace_existing=True
+    )
+    logger.info("")
+    logger.info(f"✅ Job #{job_counter + 1} Registered: 🔬 Research Fetcher")
+    
     # Start the scheduler
     logger.info("")
     logger.info("🚀 Starting scheduler engine...")

data/velocity_tracking.json

@@ -2,209 +2,209 @@
   "ai": {
     "interval": 15,
     "history": [
-      10,
-      9,
       26,
       0,
-      0
+      0,
+      6,
+      9
     ],
-    "last_fetch": "2026-02-09T15:13:49.323857",
-    "total_fetches": 19,
-    "total_articles": 268
+    "last_fetch": "2026-02-14T13:58:05.405121",
+    "total_fetches": 21,
+    "total_articles": 283
   },
   "data-security": {
     "interval": 15,
     "history": [
-      9,
-      9,
       22,
       0,
-      0
+      0,
+      3,
+      2
     ],
-    "last_fetch": "2026-02-09T15:13:49.328546",
-    "total_fetches": 19,
-    "total_articles": 181
+    "last_fetch": "2026-02-14T13:58:05.410615",
+    "total_fetches": 21,
+    "total_articles": 186
   },
   "data-governance": {
-    "interval": 15,
+    "interval": 60,
     "history": [
-      19,
-      9,
       19,
       0,
-      0
+      0,
+      3,
+      2
     ],
-    "last_fetch": "2026-02-09T15:13:49.332767",
-    "total_fetches": 19,
-    "total_articles": 173
+    "last_fetch": "2026-02-14T13:58:05.416621",
+    "total_fetches": 21,
+    "total_articles": 178
   },
   "data-privacy": {
     "interval": 15,
     "history": [
-      7,
-      7,
       23,
       0,
-      0
+      0,
+      5,
+      5
     ],
-    "last_fetch": "2026-02-09T15:13:49.336592",
-    "total_fetches": 19,
-    "total_articles": 200
+    "last_fetch": "2026-02-14T13:58:05.421392",
+    "total_fetches": 21,
+    "total_articles": 210
   },
   "data-engineering": {
-    "interval": 15,
+    "interval": 60,
     "history": [
-      14,
-      14,
       14,
       0,
-      0
+      0,
+      2,
+      5
     ],
-    "last_fetch": "2026-02-09T15:13:49.340993",
-    "total_fetches": 19,
-    "total_articles": 184
+    "last_fetch": "2026-02-14T13:58:05.424148",
+    "total_fetches": 21,
+    "total_articles": 191
   },
   "data-management": {
     "interval": 15,
     "history": [
-      19,
-      10,
       10,
       0,
-      0
+      0,
+      10,
+      10
     ],
-    "last_fetch": "2026-02-09T15:13:49.344310",
-    "total_fetches": 19,
-    "total_articles": 251
+    "last_fetch": "2026-02-14T13:58:05.430647",
+    "total_fetches": 21,
+    "total_articles": 271
   },
   "business-intelligence": {
     "interval": 15,
     "history": [
-      9,
-      9,
       23,
       0,
-      0
+      0,
+      8,
+      8
     ],
-    "last_fetch": "2026-02-09T15:13:49.347805",
-    "total_fetches": 19,
-    "total_articles": 239
+    "last_fetch": "2026-02-14T13:58:05.435766",
+    "total_fetches": 21,
+    "total_articles": 255
   },
   "business-analytics": {
     "interval": 15,
     "history": [
-      19,
-      5,
       20,
       0,
-      0
+      0,
+      6,
+      6
     ],
-    "last_fetch": "2026-02-09T15:13:49.351590",
-    "total_fetches": 19,
-    "total_articles": 191
+    "last_fetch": "2026-02-14T13:58:05.438090",
+    "total_fetches": 21,
+    "total_articles": 203
   },
   "customer-data-platform": {
     "interval": 15,
     "history": [
-      9,
-      9,
       26,
       0,
-      0
+      0,
+      9,
+      9
     ],
-    "last_fetch": "2026-02-09T15:13:49.355413",
-    "total_fetches": 19,
-    "total_articles": 229
+    "last_fetch": "2026-02-14T13:58:05.440124",
+    "total_fetches": 21,
+    "total_articles": 247
   },
   "data-centers": {
     "interval": 15,
     "history": [
-      27,
-      17,
       27,
       0,
-      0
+      0,
+      6,
+      6
     ],
-    "last_fetch": "2026-02-09T15:13:49.358619",
-    "total_fetches": 19,
-    "total_articles": 306
+    "last_fetch": "2026-02-14T13:58:05.443427",
+    "total_fetches": 21,
+    "total_articles": 318
   },
   "cloud-computing": {
     "interval": 15,
     "history": [
-      24,
-      24,
       24,
       0,
-      0
+      0,
+      5,
+      7
     ],
-    "last_fetch": "2026-02-09T15:13:49.361966",
-    "total_fetches": 19,
-    "total_articles": 388
+    "last_fetch": "2026-02-14T13:58:05.448267",
+    "total_fetches": 21,
+    "total_articles": 400
   },
   "magazines": {
     "interval": 60,
     "history": [
-      10,
-      2,
       0,
       0,
-      0
+      0,
+      6,
+      5
     ],
-    "last_fetch": "2026-02-09T15:13:49.365217",
-    "total_fetches": 19,
-    "total_articles": 95
+    "last_fetch": "2026-02-14T13:58:05.451427",
+    "total_fetches": 21,
+    "total_articles": 106
   },
   "data-laws": {
     "interval": 15,
     "history": [
-      10,
-      10,
       29,
       0,
-      0
+      0,
+      9,
+      9
     ],
-    "last_fetch": "2026-02-09T15:13:49.368458",
-    "total_fetches": 19,
-    "total_articles": 309
+    "last_fetch": "2026-02-14T13:58:05.455106",
+    "total_fetches": 21,
+    "total_articles": 327
   },
   "cloud-aws": {
-    "interval": 5,
+    "interval": 15,
     "history": [
-      29,
-      29,
       29,
       0,
-      0
+      0,
+      10,
+      10
     ],
-    "last_fetch": "2026-02-09T15:13:49.371139",
-    "total_fetches": 19,
-    "total_articles": 494
+    "last_fetch": "2026-02-14T13:58:05.456886",
+    "total_fetches": 21,
+    "total_articles": 514
   },
   "cloud-azure": {
     "interval": 15,
     "history": [
-      20,
-      20,
       20,
       0,
-      0
+      0,
+      10,
+      10
     ],
-    "last_fetch": "2026-02-09T15:13:49.374325",
-    "total_fetches": 19,
-    "total_articles": 340
+    "last_fetch": "2026-02-14T13:58:05.459585",
+    "total_fetches": 21,
+    "total_articles": 360
   },
   "cloud-gcp": {
-    "interval": 15,
+    "interval": 60,
     "history": [
       20,
-      20,
-      20,
+      0,
+      0,
       0,
       0
     ],
-    "last_fetch": "2026-02-09T15:13:49.377372",
-    "total_fetches": 19,
+    "last_fetch": "2026-02-14T13:58:05.464655",
+    "total_fetches": 21,
     "total_articles": 380
   },
   "cloud-oracle": {
@@ -216,8 +216,8 @@
       0,
       0
     ],
-    "last_fetch": "2026-02-09T15:13:49.380479",
-    "total_fetches": 19,
+    "last_fetch": "2026-02-14T13:58:05.467651",
+    "total_fetches": 21,
     "total_articles": 20
   },
   "cloud-ibm": {
@@ -229,8 +229,8 @@
       0,
       0
     ],
-    "last_fetch": "2026-02-09T15:13:49.384721",
-    "total_fetches": 19,
+    "last_fetch": "2026-02-14T13:58:05.471190",
+    "total_fetches": 21,
     "total_articles": 20
   },
   "cloud-alibaba": {
@@ -242,8 +242,8 @@
       0,
       0
     ],
-    "last_fetch": "2026-02-09T15:13:49.387837",
-    "total_fetches": 19,
+    "last_fetch": "2026-02-14T13:58:05.472734",
+    "total_fetches": 21,
     "total_articles": 20
   },
   "cloud-digitalocean": {
@@ -255,8 +255,8 @@
       0,
       0
     ],
-    "last_fetch": "2026-02-09T15:13:49.392634",
-    "total_fetches": 19,
+    "last_fetch": "2026-02-14T13:58:05.474379",
+    "total_fetches": 21,
     "total_articles": 20
   },
   "cloud-huawei": {
@@ -268,21 +268,21 @@
       0,
       0
     ],
-    "last_fetch": "2026-02-09T15:13:49.396316",
-    "total_fetches": 19,
+    "last_fetch": "2026-02-14T13:58:05.478287",
+    "total_fetches": 21,
     "total_articles": 20
   },
   "cloud-cloudflare": {
-    "interval": 15,
+    "interval": 60,
     "history": [
       20,
-      20,
-      20,
+      0,
+      0,
       0,
       0
     ],
-    "last_fetch": "2026-02-09T15:13:49.400555",
-    "total_fetches": 19,
+    "last_fetch": "2026-02-14T13:58:05.483222",
+    "total_fetches": 21,
     "total_articles": 360
   }
 }

run.py

@@ -0,0 +1,24 @@
+import uvicorn
+import asyncio
+import sys
+import os
+
+if __name__ == "__main__":
+    # Force WindowsProactorEventLoopPolicy on Windows
+    # This must be done before any asyncio loop is created
+    if sys.platform == 'win32':
+        asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+
+    # Run Uvicorn programmatically
+    # DISABLE RELOAD to fix Windows Asyncio Subprocess issue
+    # Playwright on Windows requires the main thread's event loop to be Proactor,
+    # and Uvicorn's reloader messes this up.
+    uvicorn.run(
+        "app.main:app",
+        host="127.0.0.1",
+        port=8000,
+        reload=False, 
+        workers=1,
+        loop="asyncio",
+        log_level="info"
+    )

scripts/add_missing_research_cols.py

@@ -0,0 +1,71 @@
+
+import os
+import sys
+import time
+from dotenv import load_dotenv
+from appwrite.client import Client
+from appwrite.services.databases import Databases
+
+load_dotenv()
+
+APPWRITE_ENDPOINT = os.getenv("APPWRITE_ENDPOINT")
+APPWRITE_PROJECT_ID = os.getenv("APPWRITE_PROJECT_ID")
+APPWRITE_API_KEY = os.getenv("APPWRITE_API_KEY")
+APPWRITE_DATABASE_ID = os.getenv("APPWRITE_DATABASE_ID")
+RESEARCH_COLLECTION_ID = os.getenv("APPWRITE_RESEARCH_COLLECTION_ID")
+
+client = Client()
+client.set_endpoint(APPWRITE_ENDPOINT)
+client.set_project(APPWRITE_PROJECT_ID)
+client.set_key(APPWRITE_API_KEY)
+
+databases = Databases(client)
+
+def wait_for_attribute(collection_id, key):
+    print(f"   ⏳ Waiting for attribute '{key}'...", end="", flush=True)
+    for _ in range(30):
+        try:
+            response = databases.list_attributes(APPWRITE_DATABASE_ID, collection_id)
+            attrs = response.get('attributes', [])
+            target = next((a for a in attrs if a['key'] == key), None)
+            if target and target['status'] == 'available':
+                print(" ✅ Ready.")
+                return True
+            time.sleep(2)
+            print(".", end="", flush=True)
+        except Exception:
+            time.sleep(2)
+    print(" ❌ Timeout.")
+    return False
+
+def add_missing():
+    print(f"🔧 Adding missing attributes to {RESEARCH_COLLECTION_ID}...")
+    
+    # 1. original_category
+    print(" -> original_category")
+    try:
+        databases.create_string_attribute(APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "original_category", 50, False)
+    except Exception as e:
+        if "already exists" not in str(e): print(f"Error: {e}")
+    wait_for_attribute(RESEARCH_COLLECTION_ID, "original_category")
+
+    # 2. sub_category
+    print(" -> sub_category")
+    try:
+        databases.create_string_attribute(APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "sub_category", 100, False)
+    except Exception as e:
+        if "already exists" not in str(e): print(f"Error: {e}")
+    wait_for_attribute(RESEARCH_COLLECTION_ID, "sub_category")
+
+    # 3. source
+    print(" -> source")
+    try:
+        databases.create_string_attribute(APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "source", 50, False)
+    except Exception as e:
+        if "already exists" not in str(e): print(f"Error: {e}")
+    wait_for_attribute(RESEARCH_COLLECTION_ID, "source")
+
+    print("\n✅ Schema Update Complete.")
+
+if __name__ == "__main__":
+    add_missing()

scripts/add_url_to_research.py

@@ -0,0 +1,102 @@
+
+import os
+import sys
+import time
+from dotenv import load_dotenv
+from appwrite.client import Client
+from appwrite.services.databases import Databases
+
+load_dotenv()
+
+APPWRITE_ENDPOINT = os.getenv("APPWRITE_ENDPOINT")
+APPWRITE_PROJECT_ID = os.getenv("APPWRITE_PROJECT_ID")
+APPWRITE_API_KEY = os.getenv("APPWRITE_API_KEY")
+APPWRITE_DATABASE_ID = os.getenv("APPWRITE_DATABASE_ID")
+RESEARCH_COLLECTION_ID = os.getenv("APPWRITE_RESEARCH_COLLECTION_ID")
+
+client = Client()
+client.set_endpoint(APPWRITE_ENDPOINT)
+client.set_project(APPWRITE_PROJECT_ID)
+client.set_key(APPWRITE_API_KEY)
+
+databases = Databases(client)
+
+def wait_for_attribute(collection_id, key):
+    print(f"   ⏳ Waiting for attribute '{key}'...", end="", flush=True)
+    for _ in range(30):
+        try:
+            response = databases.list_attributes(APPWRITE_DATABASE_ID, collection_id)
+            attrs = response.get('attributes', [])
+            target = next((a for a in attrs if a['key'] == key), None)
+            if target and target['status'] == 'available':
+                print(" ✅ Ready.")
+                return True
+            time.sleep(2)
+            print(".", end="", flush=True)
+        except Exception:
+            time.sleep(2)
+    print(" ❌ Timeout.")
+    return False
+
+def add_url_and_backfill():
+    print(f"🔧 Adding 'url' attribute to {RESEARCH_COLLECTION_ID}...")
+    
+    # 1. Create Attribute
+    try:
+        # url (string, 2000 chars, required=False)
+        databases.create_url_attribute(APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "url", False)
+    except Exception as e:
+        if "already exists" not in str(e): print(f"Error creating attribute: {e}")
+    
+    if wait_for_attribute(RESEARCH_COLLECTION_ID, "url"):
+        print("✅ Attribute 'url' created.")
+        
+        # 2. Backfill existing documents
+        print("🔄 Backfilling 'url' from 'pdf_url'...")
+        try:
+            # 1. List all documents (cursor pagination if many, but assuming few for now)
+            has_more = True
+            cursor = None
+            total_updated = 0
+            
+            while has_more:
+                from appwrite.query import Query
+                q = [Query.limit(100)]
+                if cursor:
+                    q.append(Query.cursor_after(cursor))
+
+                response = databases.list_documents(
+                    database_id=APPWRITE_DATABASE_ID,
+                    collection_id=RESEARCH_COLLECTION_ID,
+                    queries=q
+                )
+                
+                docs = response.get('documents', [])
+                if not docs:
+                    break
+                    
+                for doc in docs:
+                    if not doc.get('url') and doc.get('pdf_url'):
+                        try:
+                            databases.update_document(
+                                database_id=APPWRITE_DATABASE_ID,
+                                collection_id=RESEARCH_COLLECTION_ID,
+                                document_id=doc['$id'],
+                                data={'url': doc['pdf_url']}
+                            )
+                            total_updated += 1
+                            print(f"   updated {doc['$id']}")
+                        except Exception as e:
+                            print(f"   failed to update {doc['$id']}: {e}")
+                
+                cursor = docs[-1]['$id']
+                if len(docs) < 100:
+                    has_more = False
+            
+            print(f"✅ Backfill complete. Updated {total_updated} documents.")
+            
+        except Exception as e:
+            print(f"❌ Backfill failed: {e}")
+
+if __name__ == "__main__":
+    add_url_and_backfill()

scripts/init_research_collection.py

@@ -0,0 +1,169 @@
+import os
+import sys
+import warnings
+from dotenv import load_dotenv
+from appwrite.client import Client
+from appwrite.services.databases import Databases
+
+# Suppress deprecation warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+# Load environment variables
+load_dotenv()
+
+# Configuration
+APPWRITE_ENDPOINT = os.getenv("APPWRITE_ENDPOINT", "https://cloud.appwrite.io/v1")
+APPWRITE_PROJECT_ID = os.getenv("APPWRITE_PROJECT_ID")
+APPWRITE_API_KEY = os.getenv("APPWRITE_API_KEY")
+APPWRITE_DATABASE_ID = os.getenv("APPWRITE_DATABASE_ID")
+RESEARCH_COLLECTION_ID = os.getenv("APPWRITE_RESEARCH_COLLECTION_ID", "69845c19002c864d4d3f")
+
+if not all([APPWRITE_ENDPOINT, APPWRITE_PROJECT_ID, APPWRITE_API_KEY, APPWRITE_DATABASE_ID]):
+    print("❌ Missing environment variables. Please check .env file.")
+    sys.exit(1)
+
+# Initialize Appwrite Client
+client = Client()
+client.set_endpoint(APPWRITE_ENDPOINT)
+client.set_project(APPWRITE_PROJECT_ID)
+client.set_key(APPWRITE_API_KEY)
+
+databases = Databases(client)
+
+def init_research_schema():
+    print(f"🔬 Initializing Schema for Collection: {RESEARCH_COLLECTION_ID}")
+    
+    # 1. Verify Collection Exists
+    try:
+        databases.get_collection(APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID)
+        print("✅ Collection exists.")
+    except Exception as e:
+        print(f"❌ Collection not found: {e}")
+        return
+
+    # 2. Define Required Attributes
+    required_attributes = [
+        {"key": "paper_id", "type": "string", "size": 255, "required": True},
+        {"key": "title", "type": "string", "size": 500, "required": True},
+        {"key": "summary", "type": "string", "size": 5000, "required": False}, # Abstract
+        {"key": "authors", "type": "string", "size": 5000, "required": False},
+        {"key": "published_at", "type": "datetime", "required": True},
+        {"key": "pdf_url", "type": "url", "required": True},
+        {"key": "category", "type": "string", "size": 255, "required": True}, # Internal ID (research-ai)
+        {"key": "sub_category", "type": "string", "size": 255, "required": False}, # New strict sub-category
+        {"key": "original_category", "type": "string", "size": 255, "required": True}, # ArXiv ID (cs.AI)
+        {"key": "likes", "type": "integer", "required": False, "default": 0},
+        {"key": "dislike", "type": "integer", "required": False, "default": 0}, # Note: 'dislike' singular to match news
+        {"key": "views", "type": "integer", "required": False, "default": 0},
+    ]
+
+    # 3. Check and Create Attributes
+    import time
+    
+    try:
+        attrs = databases.list_attributes(APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID)
+        existing_attributes = {attr['key']: attr for attr in attrs['attributes']}
+        existing_keys = list(existing_attributes.keys())
+        print(f"Existing Attributes: {existing_keys}")
+        
+        for attr in required_attributes:
+            key = attr['key']
+            if key not in existing_keys:
+                print(f"⚙️ Creating attribute: {key}...")
+                try:
+                    if attr['type'] == "string":
+                        databases.create_string_attribute(
+                            database_id=APPWRITE_DATABASE_ID, 
+                            collection_id=RESEARCH_COLLECTION_ID, 
+                            key=key, 
+                            size=attr['size'], 
+                            required=attr['required'], 
+                            default=attr.get('default')
+                        )
+                    elif attr['type'] == "datetime":
+                        databases.create_datetime_attribute(
+                            database_id=APPWRITE_DATABASE_ID, 
+                            collection_id=RESEARCH_COLLECTION_ID, 
+                            key=key, 
+                            required=attr['required'], 
+                            default=attr.get('default')
+                        )
+                    elif attr['type'] == "url":
+                        databases.create_url_attribute(
+                            database_id=APPWRITE_DATABASE_ID, 
+                            collection_id=RESEARCH_COLLECTION_ID, 
+                            key=key, 
+                            required=attr['required'], 
+                            default=attr.get('default')
+                        )
+                    elif attr['type'] == "integer":
+                        databases.create_integer_attribute(
+                            database_id=APPWRITE_DATABASE_ID, 
+                            collection_id=RESEARCH_COLLECTION_ID, 
+                            key=key, 
+                            required=attr['required'], 
+                            min=0, 
+                            max=None, 
+                            default=attr.get('default')
+                        )
+                    print(f"   ✅ Request sent for: {key}")
+                except Exception as attr_error:
+                    print(f"   ��� Failed to create {key}: {attr_error}")
+            else:
+                print(f"   🔹 Exists: {key}")
+        
+        # 3.5 WAIT FOR ATTRIBUTES TO BE AVAILABLE
+        print("\n⏳ Waiting for attributes to become 'available'...")
+        pending_attrs = [attr['key'] for attr in required_attributes]
+        max_retries = 30 # 30 * 2 = 60 seconds
+        
+        for key in pending_attrs:
+            for attempt in range(max_retries):
+                try:
+                    # Generic get_attribute doesn't exist mainly, need typed check or list loop
+                    # robust way: list all and check specific
+                    curr_attrs = databases.list_attributes(APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID)['attributes']
+                    target = next((a for a in curr_attrs if a['key'] == key), None)
+                    
+                    if target and target['status'] == 'available':
+                        print(f"   ✅ {key} is available.")
+                        break
+                    elif target and target['status'] == 'failed':
+                        print(f"   ❌ {key} creation FAILED in Appwrite.")
+                        break
+                    else:
+                         if attempt % 5 == 0:
+                            print(f"   ... waiting for {key} (attempt {attempt+1}/{max_retries})")
+                         time.sleep(2)
+                except Exception as e:
+                    print(f"Error checking status for {key}: {e}")
+                    time.sleep(2)
+            else:
+                 print(f"   ⚠️ Timeout waiting for {key} to be available.")
+
+        # 4. Create Index on paper_id (if not exists)
+        # Only try to create index if paper_id is available
+        print("\n⚙️ Checking/Creating index on paper_id...")
+        try:
+            databases.create_index(
+                database_id=APPWRITE_DATABASE_ID,
+                collection_id=RESEARCH_COLLECTION_ID,
+                key="unique_paper_id",
+                type="unique",
+                attributes=["paper_id"]
+            )
+            print("   ✅ Index created.")
+        except Exception as e:
+            # If error contains "already exists" or 409, it's fine
+            if "already exists" in str(e) or "409" in str(e):
+                print("   🔹 Index already exists.")
+            elif "attribute not found" in str(e).lower() or "processing" in str(e).lower():
+                 print(f"   ❌ Index creation failed: Attributes still processing or missing. ({e})")
+            else:
+                print(f"   ⚠️  Could not create index: {e}")
+
+    except Exception as e:
+        print(f"❌ Error during schema initialization: {e}")
+
+if __name__ == "__main__":
+    init_research_schema()

scripts/manual_research_schema.py

@@ -0,0 +1,120 @@
+
+import os
+import sys
+import warnings
+from dotenv import load_dotenv
+from appwrite.client import Client
+from appwrite.services.databases import Databases
+from appwrite.id import ID
+
+# Suppress warnings
+warnings.filterwarnings("ignore")
+
+# Load environment variables
+load_dotenv()
+
+APPWRITE_ENDPOINT = os.getenv("APPWRITE_ENDPOINT")
+APPWRITE_PROJECT_ID = os.getenv("APPWRITE_PROJECT_ID")
+APPWRITE_API_KEY = os.getenv("APPWRITE_API_KEY")
+APPWRITE_DATABASE_ID = os.getenv("APPWRITE_DATABASE_ID")
+RESEARCH_COLLECTION_ID = os.getenv("APPWRITE_RESEARCH_COLLECTION_ID")
+
+if not all([APPWRITE_ENDPOINT, APPWRITE_PROJECT_ID, APPWRITE_API_KEY, APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID]):
+    print("❌ Missing environment variables. Check .env file.")
+    sys.exit(1)
+
+client = Client()
+client.set_endpoint(APPWRITE_ENDPOINT)
+client.set_project(APPWRITE_PROJECT_ID)
+client.set_key(APPWRITE_API_KEY)
+
+databases = Databases(client)
+
+def create_attribute_safe(func, *args, **kwargs):
+    try:
+        func(*args, **kwargs)
+        print(f"   ✅ Created attribute/index.")
+    except Exception as e:
+        if "already exists" in str(e):
+            print(f"   🔹 Already exists.")
+        else:
+            print(f"   ❌ Error: {e}")
+
+def manual_setup():
+    print(f"🔬 Manual Schema Setup for Collection: {RESEARCH_COLLECTION_ID}")
+    
+    # 1. Attributes
+    print("\n1. creating Attributes...")
+    
+    print(" - paper_id (string, 100 char, required)")
+    create_attribute_safe(databases.create_string_attribute, APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "paper_id", 100, True)
+    
+    print(" - title (string, 500 char, required)")
+    create_attribute_safe(databases.create_string_attribute, APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "title", 500, True)
+    
+    print(" - summary (string, 5000 char, required)")
+    create_attribute_safe(databases.create_string_attribute, APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "summary", 5000, True)
+    
+    print(" - authors (string, 5000 char, optional - storing as JSON string or comma separated)")
+    create_attribute_safe(databases.create_string_attribute, APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "authors", 5000, False)
+    
+    print(" - published_at (datetime, required)")
+    create_attribute_safe(databases.create_datetime_attribute, APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "published_at", True)
+    
+    print(" - pdf_url (url, required - using string 2000)")
+    create_attribute_safe(databases.create_url_attribute, APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "pdf_url", True)
+    
+    print(" - category (string, 50 char, required)")
+    create_attribute_safe(databases.create_string_attribute, APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "category", 50, True)
+    
+    # Engagement stats
+    print(" - likes (integer, min 0, default 0)")
+    create_attribute_safe(databases.create_integer_attribute, APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "likes", False, 0, 2147483647, 0)
+
+    print(" - dislikes (integer, min 0, default 0)")
+    create_attribute_safe(databases.create_integer_attribute, APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "dislikes", False, 0, 2147483647, 0)
+    
+    print(" - views (integer, min 0, default 0)")
+    create_attribute_safe(databases.create_integer_attribute, APPWRITE_DATABASE_ID, RESEARCH_COLLECTION_ID, "views", False, 0, 2147483647, 0)
+
+    # 2. Indexes
+    print("\n2. Creating Indexes...")
+    
+    print(" - paper_id (unique)")
+    create_attribute_safe(
+        databases.create_index, 
+        APPWRITE_DATABASE_ID, 
+        RESEARCH_COLLECTION_ID, 
+        "unique_paper_id", 
+        "unique", 
+        ["paper_id"],
+        ["ASC"]
+    )
+    
+    print(" - published_at (key)")
+    create_attribute_safe(
+        databases.create_index, 
+        APPWRITE_DATABASE_ID, 
+        RESEARCH_COLLECTION_ID, 
+        "idx_published_at", 
+        "key", 
+        ["published_at"],
+        ["DESC"]
+    )
+
+    print(" - category (key)")
+    create_attribute_safe(
+        databases.create_index, 
+        APPWRITE_DATABASE_ID, 
+        RESEARCH_COLLECTION_ID, 
+        "idx_category", 
+        "key", 
+        ["category"],
+        ["ASC"]
+    )
+
+    print("\n✅ Setup commands sent. Attribute creation is ASYNC in Appwrite.")
+    print("   Wait ~30-60 seconds before running ingestion.")
+
+if __name__ == "__main__":
+    manual_setup()

scripts/setup_research_v2.py

@@ -0,0 +1,214 @@
+
+import os
+import sys
+import time
+import warnings
+from dotenv import load_dotenv
+from appwrite.client import Client
+from appwrite.services.databases import Databases
+
+# Suppress warnings
+warnings.filterwarnings("ignore")
+
+# Load environment variables
+load_dotenv()
+
+APPWRITE_ENDPOINT = os.getenv("APPWRITE_ENDPOINT")
+APPWRITE_PROJECT_ID = os.getenv("APPWRITE_PROJECT_ID")
+APPWRITE_API_KEY = os.getenv("APPWRITE_API_KEY")
+APPWRITE_DATABASE_ID = os.getenv("APPWRITE_DATABASE_ID")
+
+if not all([APPWRITE_ENDPOINT, APPWRITE_PROJECT_ID, APPWRITE_API_KEY, APPWRITE_DATABASE_ID]):
+    print("❌ Missing environment variables. Check .env file.")
+    sys.exit(1)
+
+client = Client()
+client.set_endpoint(APPWRITE_ENDPOINT)
+client.set_project(APPWRITE_PROJECT_ID)
+client.set_key(APPWRITE_API_KEY)
+
+databases = Databases(client)
+
+def wait_for_attribute(collection_id, key):
+    """Polls Appwrite until the attribute status is 'available'."""
+    print(f"   ⏳ Waiting for attribute '{key}' to be available...", end="", flush=True)
+    retries = 0
+    max_retries = 60 # 2 minutes
+    
+    while retries < max_retries:
+        try:
+            # Check attribute status by listing or getting
+            # The SDK doesn't have 'get_attribute' easily for all types, so we list
+            response = databases.list_attributes(APPWRITE_DATABASE_ID, collection_id)
+            attrs = response.get('attributes', [])
+            
+            target = next((a for a in attrs if a['key'] == key), None)
+            
+            if target:
+                if target['status'] == 'available':
+                    print(" ✅ Ready.")
+                    return True
+                elif target['status'] == 'failed':
+                    print(" ❌ Failed.")
+                    return False
+            
+            time.sleep(2)
+            print(".", end="", flush=True)
+            retries += 1
+        except Exception as e:
+            print(f" Error: {e}")
+            time.sleep(2)
+            retries += 1
+            
+    print(" ❌ Timeout.")
+    return False
+
+def wait_for_index(collection_id, key):
+    """Polls Appwrite until the index status is 'available'."""
+    print(f"   ⏳ Waiting for index '{key}' to be available...", end="", flush=True)
+    retries = 0
+    max_retries = 60 
+    
+    while retries < max_retries:
+        try:
+            response = databases.list_indexes(APPWRITE_DATABASE_ID, collection_id)
+            indexes = response.get('indexes', [])
+            
+            target = next((i for i in indexes if i['key'] == key), None)
+            
+            if target:
+                if target['status'] == 'available':
+                    print(" ✅ Ready.")
+                    return True
+                elif target['status'] == 'failed':
+                    print(" ❌ Failed.")
+                    return False
+            
+            time.sleep(2)
+            print(".", end="", flush=True)
+            retries += 1
+        except Exception as e:
+            print(f" Error: {e}")
+            time.sleep(2)
+            retries += 1
+
+    print(" ❌ Timeout.")
+    return False
+
+def setup_v2(collection_id):
+    print(f"🚀 Starting Robust Schema Setup for: {collection_id}")
+    
+    # Define Attributes
+    # (func, key, size/opts, required, default, array)
+    # Simplified structure for the loop
+    
+    # 1. Attributes
+    print("\n📦 Creating Attributes (Synchronous Mode)...")
+    
+    try:
+        # paper_id
+        print(" -> paper_id")
+        try:
+            databases.create_string_attribute(APPWRITE_DATABASE_ID, collection_id, "paper_id", 100, True)
+        except Exception as e:
+             if "already exists" not in str(e): print(f"Error: {e}")
+        wait_for_attribute(collection_id, "paper_id")
+
+        # title
+        print(" -> title")
+        try:
+            databases.create_string_attribute(APPWRITE_DATABASE_ID, collection_id, "title", 500, True)
+        except Exception as e:
+             if "already exists" not in str(e): print(f"Error: {e}")
+        wait_for_attribute(collection_id, "title")
+
+        # summary
+        print(" -> summary")
+        try:
+            databases.create_string_attribute(APPWRITE_DATABASE_ID, collection_id, "summary", 5000, True)
+        except Exception as e:
+             if "already exists" not in str(e): print(f"Error: {e}")
+        wait_for_attribute(collection_id, "summary")
+
+        # authors
+        print(" -> authors")
+        try:
+            databases.create_string_attribute(APPWRITE_DATABASE_ID, collection_id, "authors", 5000, False)
+        except Exception as e:
+             if "already exists" not in str(e): print(f"Error: {e}")
+        wait_for_attribute(collection_id, "authors")
+
+        # published_at
+        print(" -> published_at")
+        try:
+            databases.create_datetime_attribute(APPWRITE_DATABASE_ID, collection_id, "published_at", True)
+        except Exception as e:
+             if "already exists" not in str(e): print(f"Error: {e}")
+        wait_for_attribute(collection_id, "published_at")
+
+        # pdf_url
+        print(" -> pdf_url")
+        try:
+            databases.create_url_attribute(APPWRITE_DATABASE_ID, collection_id, "pdf_url", True)
+        except Exception as e:
+             if "already exists" not in str(e): print(f"Error: {e}")
+        wait_for_attribute(collection_id, "pdf_url")
+
+        # category
+        print(" -> category")
+        try:
+            databases.create_string_attribute(APPWRITE_DATABASE_ID, collection_id, "category", 50, True)
+        except Exception as e:
+             if "already exists" not in str(e): print(f"Error: {e}")
+        wait_for_attribute(collection_id, "category")
+        
+        # Stats
+        for stat in ["likes", "dislikes", "views"]:
+            print(f" -> {stat}")
+            try:
+                databases.create_integer_attribute(APPWRITE_DATABASE_ID, collection_id, stat, False, 0, 2147483647, 0)
+            except Exception as e:
+                if "already exists" not in str(e): print(f"Error: {e}")
+            wait_for_attribute(collection_id, stat)
+
+    except Exception as e:
+        print(f"❌ Critical Error during attribute creation: {e}")
+        return
+
+    # 2. Indexes
+    print("\n🗂️ Creating Indexes (Now that attributes are ready)...")
+    
+    # unique_paper_id
+    print(" -> unique_paper_id")
+    try:
+        databases.create_index(APPWRITE_DATABASE_ID, collection_id, "unique_paper_id", "unique", ["paper_id"], ["ASC"])
+    except Exception as e:
+        if "already exists" not in str(e): print(f"Error: {e}")
+    wait_for_index(collection_id, "unique_paper_id")
+
+    # idx_published_at
+    print(" -> idx_published_at")
+    try:
+        databases.create_index(APPWRITE_DATABASE_ID, collection_id, "idx_published_at", "key", ["published_at"], ["DESC"])
+    except Exception as e:
+        if "already exists" not in str(e): print(f"Error: {e}")
+    wait_for_index(collection_id, "idx_published_at")
+
+    # idx_category
+    print(" -> idx_category")
+    try:
+        databases.create_index(APPWRITE_DATABASE_ID, collection_id, "idx_category", "key", ["category"], ["ASC"])
+    except Exception as e:
+        if "already exists" not in str(e): print(f"Error: {e}")
+    wait_for_index(collection_id, "idx_category")
+
+    print("\n✅ Verification Complete: All attributes and indexes are AVAILABLE.")
+    print("   You may now run the ingestion script.")
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python scripts/setup_research_v2.py <NEW_COLLECTION_ID>")
+        sys.exit(1)
+    
+    col_id = sys.argv[1]
+    setup_v2(col_id)

scripts/test_research_flow.py

@@ -0,0 +1,103 @@
+
+import asyncio
+import os
+import sys
+import httpx
+from dotenv import load_dotenv
+
+# Add project root to path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../')))
+
+from app.config import settings
+from app.services.appwrite_db import get_appwrite_db
+from app.services.research_aggregator import ResearchAggregator
+
+load_dotenv()
+
+async def test_flow():
+    print("🧪 Starting End-to-End Research Verification...")
+    print(f"   Target Collection: {settings.APPWRITE_RESEARCH_COLLECTION_ID}")
+    
+    # 1. Simulate Ingestion (Fetch 1 paper from ArXiv)
+    print("\n1. Testing Ingestion (Dry Run logic)...")
+    agg = ResearchAggregator()
+    
+    # We'll mock the fetch to avoid hitting ArXiv rate limits or waiting
+    # We just want to test the _save_paper logic with the new schema.
+    
+    test_paper = {
+        "paper_id": "test.12345",
+        "title": "Test Paper for Verification",
+        "summary": "This is a test summary for verification purposes.",
+        "authors": ["Test Author"],
+        "published_at": "2023-10-27T00:00:00+00:00",
+        "pdf_url": "http://arxiv.org/pdf/test.12345",
+        "category": "research-ai",
+        "original_category": "cs.AI",
+        "sub_category": "Artificial Intelligence", # Friendly name
+        "source": "arXiv"
+    }
+    
+    print(f"   Attempting to save test paper: {test_paper['paper_id']}")
+    saved = await agg._save_paper(test_paper)
+    
+    if saved:
+        print("   ✅ Ingestion Successful (Paper Saved/Created).")
+    else:
+        print("   ⚠️  Ingestion Skipped (Likely Duplicate or Error).")
+
+    # 2. Verify Data in DB
+    print("\n2. Verifying Data in Appwrite...")
+    appwrite = get_appwrite_db()
+    try:
+        # We need to find the document ID of the paper we just saved (or existing one)
+        # We can query by paper_id
+        from appwrite.query import Query
+        response = await appwrite.tablesDB.list_rows(
+            database_id=settings.APPWRITE_DATABASE_ID,
+            collection_id=settings.APPWRITE_RESEARCH_COLLECTION_ID,
+            queries=[Query.equal("paper_id", "test.12345")]
+        )
+        
+        if response['total'] == 0:
+            print("   ❌ Error: Paper not found in DB after ingestion.")
+            return
+
+        doc = response['documents'][0]
+        doc_id = doc['$id']
+        print(f"   ✅ Paper found! Doc ID: {doc_id}")
+        print(f"   Current Stats -> Likes: {doc.get('likes')}, Views: {doc.get('views')}")
+        
+    except Exception as e:
+        print(f"   ❌ Error verifying data: {e}")
+        return
+
+    # 3. Test Engagement (Like)
+    # This simulates what the Frontend does (POST /api/engagement/articles/{id}/like)
+    # We will use the internal router function logic or just call DB update manually to verify schema.
+    # Calling DB manually is better to verify the *schema* accepts updates.
+    
+    print("\n3. Testing Stats Update (Simulation)...")
+    try:
+        current_likes = doc.get('likes', 0) or 0
+        new_likes = current_likes + 1
+        
+        updated_doc = await appwrite.tablesDB.update_row(
+            database_id=settings.APPWRITE_DATABASE_ID,
+            collection_id=settings.APPWRITE_RESEARCH_COLLECTION_ID,
+            document_id=doc_id,
+            data={"likes": new_likes}
+        )
+        
+        if updated_doc['likes'] == new_likes:
+            print(f"   ✅ Stats Update Successful! Likes: {current_likes} -> {updated_doc['likes']}")
+        else:
+            print(f"   ❌ Stats Update Mismatch: Expected {new_likes}, got {updated_doc['likes']}")
+
+    except Exception as e:
+        print(f"   ❌ Error updating stats: {e}")
+        
+    print("\n🎉 Verification Complete!")
+
+if __name__ == "__main__":
+    asyncio.run(test_flow())

scripts/verify_research_columns.py

@@ -0,0 +1,58 @@
+
+import os
+import sys
+from dotenv import load_dotenv
+from appwrite.client import Client
+from appwrite.services.databases import Databases
+
+load_dotenv()
+
+APPWRITE_ENDPOINT = os.getenv("APPWRITE_ENDPOINT")
+APPWRITE_PROJECT_ID = os.getenv("APPWRITE_PROJECT_ID")
+APPWRITE_API_KEY = os.getenv("APPWRITE_API_KEY")
+APPWRITE_DATABASE_ID = os.getenv("APPWRITE_DATABASE_ID")
+RESEARCH_COLLECTION_ID = os.getenv("APPWRITE_RESEARCH_COLLECTION_ID")
+
+client = Client()
+client.set_endpoint(APPWRITE_ENDPOINT)
+client.set_project(APPWRITE_PROJECT_ID)
+client.set_key(APPWRITE_API_KEY)
+
+databases = Databases(client)
+
+def list_attributes():
+    print(f"📊 Checking Attributes for Collection: {RESEARCH_COLLECTION_ID}")
+    try:
+        # Fetch list of attributes
+        # Note: Appwrite returns a list of attribute objects
+        # We need to handle pagination if there are many, but usually < 25
+        response = databases.list_attributes(
+            database_id=APPWRITE_DATABASE_ID,
+            collection_id=RESEARCH_COLLECTION_ID
+        )
+        
+        attributes = response.get('attributes', [])
+        print(f"   Found {len(attributes)} attributes.")
+        
+        found_attrs = set()
+        for attr in attributes:
+            key = attr.get('key')
+            status = attr.get('status')
+            typ = attr.get('type')
+            print(f"   - {key} ({typ}) [Status: {status}]")
+            found_attrs.add(key)
+            
+        # Check against expected
+        expected = ['paper_id', 'title', 'summary', 'authors', 'published_at', 'pdf_url', 'category', 'likes', 'dislikes', 'views']
+        missing = [x for x in expected if x not in found_attrs]
+        
+        if missing:
+            print(f"\n❌ MISSING Attributes: {missing}")
+        else:
+            print(f"\n✅ All expected attributes exist.")
+            
+    except Exception as e:
+        print(f"❌ Error fetching attributes: {e}")
+
+if __name__ == "__main__":
+    list_attributes()

scripts/verify_research_data.py

@@ -0,0 +1,36 @@
+import asyncio
+from app.services.appwrite_db import get_appwrite_db
+from app.config import settings
+
+async def verify_research():
+    db = get_appwrite_db()
+    if not db.initialized:
+        print("DB not initialized")
+        return
+
+    collection_id = settings.APPWRITE_RESEARCH_COLLECTION_ID
+    print(f"Research Collection ID: {collection_id}")
+
+    try:
+        # List documents
+        docs = await db.tablesDB.list_rows(
+            database_id=settings.APPWRITE_DATABASE_ID,
+            collection_id=collection_id,
+            queries=[]
+        )
+        print(f"Total Documents: {docs['total']}")
+        if docs['documents']:
+            print("Sample Document:", docs['documents'][0])
+        else:
+            print("No documents found.")
+            
+        # Test get_articles logic
+        articles = await db.get_articles('research', limit=5)
+        print(f"get_articles('research') returned: {len(articles)}")
+    except Exception as e:
+        print(f"Error: {e}")
+
+if __name__ == "__main__":
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.run_until_complete(verify_research())

scripts/verify_research_indexes.py

@@ -0,0 +1,90 @@
+
+import os
+import sys
+import asyncio
+from dotenv import load_dotenv
+from appwrite.client import Client
+from appwrite.services.databases import Databases
+from appwrite.query import Query
+
+load_dotenv()
+
+APPWRITE_ENDPOINT = os.getenv("APPWRITE_ENDPOINT")
+APPWRITE_PROJECT_ID = os.getenv("APPWRITE_PROJECT_ID")
+APPWRITE_API_KEY = os.getenv("APPWRITE_API_KEY")
+APPWRITE_DATABASE_ID = os.getenv("APPWRITE_DATABASE_ID")
+RESEARCH_COLLECTION_ID = os.getenv("APPWRITE_RESEARCH_COLLECTION_ID")
+
+client = Client()
+client.set_endpoint(APPWRITE_ENDPOINT)
+client.set_project(APPWRITE_PROJECT_ID)
+client.set_key(APPWRITE_API_KEY)
+
+databases = Databases(client)
+
+async def probe_index(attr, query_type):
+    print(f"🕵️ Probing index for '{attr}'...")
+    try:
+        # We don't need async here actually, the python SDK is sync unless we use the async client, 
+        # but my environment context shows async usage in services. 
+        # The standard appwrite python SDK is synchronous (CHECK: 'appwrite' package).
+        # Wait, the code I viewed in `research_aggregator.py` imports `arxiv` but uses `app.services.appwrite_db` handling stuff.
+        # Standard `appwrite` package on PyPI is synchronous.
+        # But `appwrite_db.py` wraps it? No, checking `appwrite_db.py`: `from appwrite.services.databases import Databases`.
+        # Wait, step 768 `appwrite_db.py` has `await self.tablesDB.list_rows`.
+        # If the user is using `appwrite` python package < 4.0 it might be sync, or if they operate it differently.
+        # BUT, to be safe and simple, I will use the SYNC calls here as it's a script.
+        
+        # Testing Sort
+        q = [Query.order_desc(attr), Query.limit(1)]
+        databases.list_documents(
+            database_id=APPWRITE_DATABASE_ID,
+            collection_id=RESEARCH_COLLECTION_ID,
+            queries=q
+        )
+        print(f"   ✅ Index '{attr}' appears ACTIVE (Sort query worked).")
+        return True
+    except Exception as e:
+        if "Index not found" in str(e) or "missing index" in str(e).lower():
+            print(f"   ❌ Index '{attr}' is MISSING or INVALID.")
+            print(f"      Error: {e}")
+            return False
+        else:
+            print(f"   ⚠️  Query failed (Unrelated?): {e}")
+            return False
+
+def verify():
+    print(f"🧪 Verifying Indexes for {RESEARCH_COLLECTION_ID}")
+    
+    # 1. Check 'published_at' (Sortable)
+    probe_index("published_at", "order_desc")
+    
+    # 2. Check 'category' (Filterable/Sortable)
+    # To check filter index, we try to filter.
+    print(f"🕵️ Probing index for 'category' (Filter)...")
+    try:
+        databases.list_documents(
+             database_id=APPWRITE_DATABASE_ID,
+             collection_id=RESEARCH_COLLECTION_ID,
+             queries=[Query.equal("category", "research-ai"), Query.limit(1)]
+        )
+        print(f"   ✅ Index 'category' appears ACTIVE (Filter query worked).")
+    except Exception as e:
+        print(f"   ❌ Index 'category' is MISSING/INVALID: {e}")
+
+    # 3. Check 'paper_id' (Unique)
+    # Hard to test unique without creating, but we can try to filter by it if it's a key.
+    # Usually unique indexes also allow filtering.
+    print(f"🕵️ Probing index for 'paper_id' (Filter)...")
+    try:
+        databases.list_documents(
+             database_id=APPWRITE_DATABASE_ID,
+             collection_id=RESEARCH_COLLECTION_ID,
+             queries=[Query.equal("paper_id", "test"), Query.limit(1)]
+        )
+        print(f"   ✅ Index 'paper_id' appears ACTIVE (Filter query worked).")
+    except Exception as e:
+        print(f"   ❌ Index 'paper_id' is MISSING/INVALID: {e}")
+
+if __name__ == "__main__":
+    verify()

tools/verify_browser.py

@@ -0,0 +1,33 @@
+import asyncio
+import logging
+import sys
+import os
+
+# Add project root to path
+sys.path.append(os.getcwd())
+
+from app.services.browser_manager import browser_manager
+
+logging.basicConfig(level=logging.INFO)
+
+async def test_browser_manager():
+    print("Testing BrowserManager...")
+    try:
+        await browser_manager.start()
+        
+        url = "https://example.com"
+        print(f"Fetching {url}...")
+        content = await browser_manager.get_content(url)
+        
+        if content and "Example Domain" in content:
+            print("✅ Successfully fetched content!")
+        else:
+            print("❌ Failed to fetch content or content mismatch.")
+            
+    except Exception as e:
+        print(f"❌ Error: {e}")
+    finally:
+        await browser_manager.shutdown()
+
+if __name__ == "__main__":
+    asyncio.run(test_browser_manager())