Fixed WebScraper and standardized URLs

app/core/settings.py

@@ -24,7 +24,7 @@ class Settings(BaseSettings):
     
     # API Config
     API_HOST: str = "0.0.0.0"
-    API_PORT: int = 8000
+    PORT: int = 8000  # Standard Render/Cloud Run env var
     LOG_LEVEL: str = "INFO"
     TIMEOUT_SECONDS: int = 120
     

app/memory/cache.py

@@ -2,6 +2,7 @@ import json
 import logging
 import os
 from typing import Any, Dict, Optional
+from app.core.settings import settings
 
 import redis
 from redis.exceptions import RedisError
@@ -23,7 +24,7 @@ class CacheManager:
             redis_url: Redis connection string (used if pool not provided).
             connection_pool: Existing Redis connection pool.
         """
-        self.redis_url = redis_url or os.getenv("REDIS_URL", "redis://localhost:6379/0")
+        self.redis_url = redis_url or settings.REDIS_URL
         self.redis_client = None
         
         try:

app/memory/database.py

@@ -1,7 +1,7 @@
 import logging
 import os
-from datetime import datetime, timezone
 from typing import Any, Dict, Optional, List
+from app.core.settings import settings
 
 import pymongo
 from pymongo import IndexModel, ASCENDING
@@ -24,7 +24,7 @@ class DatabaseManager:
             mongo_uri: MongoDB connection string.
             client: Existing PyMongo client (shared pool).
         """
-        self.mongo_uri = mongo_uri or os.getenv("MONGO_URI", "mongodb://localhost:27017/")
+        self.mongo_uri = mongo_uri or settings.MONGO_URI
         self.client = None
         self.db = None
         self.collection = None
@@ -45,9 +45,11 @@ class DatabaseManager:
             self.client.server_info()
             
             # Setup DB and collection
-            db_name = "mathminds_ai"
+            db_name = settings.MONGO_DB_NAME
             try:
-                uri_db = pymongo.uri_parser.parse_uri(self.mongo_uri).get('database')
+                # If URI contains a DB name, it will override the setting default
+                parsed_uri = pymongo.uri_parser.parse_uri(self.mongo_uri)
+                uri_db = parsed_uri.get('database')
                 if uri_db:
                     db_name = uri_db
             except Exception:

app/worker/celery_app.py

@@ -0,0 +1,25 @@
+from celery import Celery
+import os
+from app.core.settings import settings
+
+# Initialize Celery
+celery_app = Celery(
+    "mathminds",
+    broker=settings.REDIS_URL,
+    backend=settings.REDIS_URL,
+    include=["app.worker.tasks"]
+)
+
+# Optional configuration
+celery_app.conf.update(
+    task_serializer="json",
+    accept_content=["json"],
+    result_serializer="json",
+    timezone="UTC",
+    enable_utc=True,
+    task_track_started=True,
+    task_time_limit=300, # 5 minutes max
+)
+
+if __name__ == "__main__":
+    celery_app.start()

app/worker/tasks.py

@@ -0,0 +1,22 @@
+import logging
+from app.worker.celery_app import celery_app
+from app.tools.web_scraper import run_playwright_sync
+
+logger = logging.getLogger(__name__)
+
+@celery_app.task(name="app.worker.tasks.scrape_task", bind=True)
+def scrape_task(self, query: str, headless: bool = True, extraction_focus: str = None):
+    """
+    Celery task for web scraping.
+    """
+    logger.info(f"Task {self.request.id} started for query: {query}")
+    try:
+        result = run_playwright_sync(query, headless, extraction_focus)
+        return result
+    except Exception as e:
+        logger.error(f"Task failed: {e}")
+        return {
+            "source": "web_scraper",
+            "error": str(e),
+            "status": "error"
+        }

check_redis.py

@@ -0,0 +1,18 @@
+import redis
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+redis_url = os.getenv("REDIS_URL", "redis://localhost:6379/0")
+
+def check_redis():
+    print(f"Checking Redis at: {redis_url}")
+    try:
+        r = redis.from_url(redis_url)
+        r.ping()
+        print("✅ Redis is UP!")
+    except Exception as e:
+        print(f"❌ Redis is DOWN or unreachable: {e}")
+
+if __name__ == "__main__":
+    check_redis()

debug_celery_worker.py

@@ -0,0 +1,49 @@
+import asyncio
+import logging
+import sys
+import os
+
+# Add the current directory to sys.path so we can import 'app'
+sys.path.append(os.getcwd())
+
+from app.worker.tasks import scrape_task
+import time
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+async def debug_scrape():
+    print("Triggering Celery Scrape Task...")
+    query = "gold rate in india today"
+    
+    try:
+        # Dispatch task
+        result = scrape_task.delay(query)
+        print(f"Task ID: {result.id}")
+        
+        # Wait for result
+        start_time = time.time()
+        max_wait = 60 # seconds
+        
+        while time.time() - start_time < max_wait:
+            if result.ready():
+                print("Task Ready!")
+                print("Result Status:", result.status)
+                # Safely handle potential encoding issues when printing to console
+                try:
+                    res_content = str(result.result)
+                    print("Result Content (partial):", res_content[:200].encode('ascii', 'ignore').decode('ascii'))
+                except Exception as e:
+                    print(f"Result received, but print failed: {e}")
+                return
+            
+            print(f"Waiting... (status: {result.status})")
+            await asyncio.sleep(2)
+            
+        print("Task timed out. Is the worker running?")
+        
+    except Exception as e:
+        print(f"Dispatch failed: {e}")
+
+if __name__ == "__main__":
+    asyncio.run(debug_scrape())

frontend/Dockerfile

@@ -0,0 +1,33 @@
+# Use official Python runtime as a parent image
+FROM python:3.12-slim
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV PORT=8501
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies (curl for health checks)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies
+# We reuse the root requirements.txt for simplicity, or we could have a specific one
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of the application
+COPY . .
+
+# Create a non-root user and switch to it for security
+RUN useradd -m appuser && chown -R appuser /app
+USER appuser
+
+# Expose Streamlit's default port
+EXPOSE 8501
+
+# Command to run the Streamlit app
+CMD streamlit run frontend/app.py --server.port=$PORT --server.address=0.0.0.0

frontend/app.py

@@ -107,8 +107,8 @@ st.markdown("""
 # ====================================================
 # Config
 # ====================================================
-BASE_API_URL = os.getenv("API_BASE_URL", "http://localhost:8000")
-API_URL = f"{BASE_API_URL}/solve"
+BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:8000")
+API_URL = f"{BACKEND_URL}/solve"
 
 
 # ====================================================
@@ -154,7 +154,7 @@ def load_sessions():
     """Fetch THIS user's chat sessions from the backend and populate state."""
     try:
         headers = get_auth_headers()
-        response = requests.get(f"{BASE_API_URL}/chat/sessions", headers=headers, timeout=30)
+        response = requests.get(f"{BACKEND_URL}/chat/sessions", headers=headers, timeout=30)
         if response.status_code == 200:
             st.session_state.chat_sessions = response.json()
             # Mark that we've successfully loaded data for this specific user
@@ -197,7 +197,7 @@ def load_messages(session_id):
     try:
         headers = get_auth_headers()
         response = requests.get(
-            f"{BASE_API_URL}/chat/sessions/{session_id}/messages",
+            f"{BACKEND_URL}/chat/sessions/{session_id}/messages",
             headers=headers, timeout=30
         )
         if response.status_code == 200:
@@ -232,7 +232,7 @@ def add_message(role, content, sent_to_api=False, **kwargs):
 def new_chat():
     try:
         headers = get_auth_headers()
-        response = requests.post(f"{BASE_API_URL}/chat/sessions", headers=headers, timeout=30)
+        response = requests.post(f"{BACKEND_URL}/chat/sessions", headers=headers, timeout=30)
         if response.status_code == 200:
             new_s = response.json()
             st.session_state.active_session_id = new_s["session_id"]
@@ -248,7 +248,7 @@ def new_chat():
 def delete_chat(sid):
     try:
         headers = get_auth_headers()
-        response = requests.delete(f"{BASE_API_URL}/chat/sessions/{sid}", headers=headers, timeout=30)
+        response = requests.delete(f"{BACKEND_URL}/chat/sessions/{sid}", headers=headers, timeout=30)
         if response.status_code == 200:
             if st.session_state.active_session_id == sid:
                 st.session_state.active_session_id = None
@@ -265,7 +265,7 @@ def rename_chat(sid, new_title):
     try:
         headers = get_auth_headers()
         response = requests.patch(
-            f"{BASE_API_URL}/chat/sessions/{sid}",
+            f"{BACKEND_URL}/chat/sessions/{sid}",
             headers=headers, json={"title": new_title}, timeout=30
         )
         if response.status_code == 200:
@@ -387,7 +387,7 @@ def profile_interface():
 
     if "profile_data" not in st.session_state:
         try:
-            r = requests.get(f"{BASE_API_URL}/users/profile", headers=headers, timeout=30)
+            r = requests.get(f"{BACKEND_URL}/users/profile", headers=headers, timeout=30)
             st.session_state.profile_data = r.json() if r.status_code == 200 else {}
         except Exception:
             st.session_state.profile_data = {}
@@ -410,7 +410,7 @@ def profile_interface():
         if st.form_submit_button("Save Profile", use_container_width=True, type="primary"):
             payload = {"display_name": display_name, "math_level": math_level, "interests": interests}
             try:
-                r = requests.post(f"{BASE_API_URL}/users/profile", json=payload, headers=headers)
+                r = requests.post(f"{BACKEND_URL}/users/profile", json=payload, headers=headers)
                 if r.status_code == 200:
                     st.success("Profile updated!")
                     st.session_state.profile_data = payload

frontend/firebase_utils.py

@@ -0,0 +1,60 @@
+import requests
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+FIREBASE_WEB_API_KEY = os.getenv("FIREBASE_WEB_API_KEY")
+
+def sign_in_with_email(email, password):
+    """
+    Signs in a user using Firebase Auth REST API.
+    Returns (id_token, local_id, email, error_message)
+    """
+    if not FIREBASE_WEB_API_KEY:
+        return None, None, None, "FIREBASE_WEB_API_KEY is not set in .env"
+
+    url = f"https://identitytoolkit.googleapis.com/v1/accounts:signInWithPassword?key={FIREBASE_WEB_API_KEY}"
+    payload = {
+        "email": email,
+        "password": password,
+        "returnSecureToken": True
+    }
+    
+    try:
+        response = requests.post(url, json=payload)
+        data = response.json()
+        
+        if response.status_code == 200:
+            return data["idToken"], data["localId"], data["email"], None
+        else:
+            error_msg = data.get("error", {}).get("message", "Unknown error")
+            return None, None, None, error_msg
+    except Exception as e:
+        return None, None, None, str(e)
+
+def sign_up_with_email(email, password):
+    """
+    Registers a new user using Firebase Auth REST API.
+    """
+    if not FIREBASE_WEB_API_KEY:
+        return None, None, None, "FIREBASE_WEB_API_KEY is not set in .env"
+
+    url = f"https://identitytoolkit.googleapis.com/v1/accounts:signUp?key={FIREBASE_WEB_API_KEY}"
+    payload = {
+        "email": email,
+        "password": password,
+        "returnSecureToken": True
+    }
+    
+    try:
+        response = requests.post(url, json=payload)
+        data = response.json()
+        
+        if response.status_code == 200:
+            return data["idToken"], data["localId"], data["email"], None
+        else:
+            error_msg = data.get("error", {}).get("message", "Unknown error")
+            return None, None, None, error_msg
+    except Exception as e:
+        return None, None, None, str(e)

test_scraper_local.py

@@ -0,0 +1,33 @@
+import logging
+import sys
+import os
+
+# Add the current directory to sys.path so we can import 'app'
+sys.path.append(os.getcwd())
+
+from app.tools.web_scraper import run_playwright_sync
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def test_direct_scrape():
+    print("--- Testing run_playwright_sync DIRECTLY (Subprocess-safe) ---")
+    query = "current gold rate in mumbai"
+    
+    try:
+        # We run it synchronously as it's designed
+        result = run_playwright_sync(query, headless=True)
+        
+        print("\n[RESULT]")
+        if result.get("status") == "success":
+            print(f"URL: {result.get('url')}")
+            print(f"Content Length: {len(result.get('content', ''))}")
+            print(f"Sample: {result.get('content')[:500]}...")
+        else:
+            print(f"Error: {result.get('error')}")
+            
+    except Exception as e:
+        print(f"Crashed: {e}")
+
+if __name__ == "__main__":
+    test_direct_scrape()