from agents import Agent, OpenAIChatCompletionsModel, AsyncOpenAI, Runner, RunConfig import os from bson import ObjectId from dotenv import load_dotenv, find_dotenv from reports.crawl import crawl import json from reports.generate_report import generate_report from reports.email_sender import send_email_with_pdf, send_ssl_alert_email from reports.ssl_checker import check_ssl_certificate_async import datetime from reports.database import db from reports.gemini_token_tracker import track_token_usage load_dotenv(find_dotenv()) external_client = AsyncOpenAI( api_key=os.getenv("GEMINI_API_KEY"), base_url="https://generativelanguage.googleapis.com/v1beta/openai/", ) llm_model = OpenAIChatCompletionsModel( model="gemini-2.5-flash", openai_client=external_client ) config = RunConfig( model=llm_model, model_provider=external_client, tracing_disabled=True ) instructions = """ You are a professional Web SEO Audit Agent built to analyze individual page crawl data and provide precise, actionable, and technically accurate SEO recommendations. You will receive crawl data for a single web page, including meta tags, schema details, broken links, images, and structure. Your goal is to generate a **structured, concise, and page-specific SEO report**. ### Your Output Must Be a Valid JSON Object and Include: 1. **seo_summary** - Give a concise summary of the SEO health of this specific page. - Do not mention total number of pages or crawl scope. Focus only on this page. 2. **detected_issues** - List clear technical or SEO issues (if any). - If no issues exist, say: “No major SEO issues detected.” 3. **ai_recommendations** - Suggest clear, technical, and developer-friendly improvements for each issue. - Be specific about what to fix and how. - If no issues exist, say: “No action needed. Page is well optimized.” 4. **structured_data_schema_feedback** - Mention what schema types were found. - Suggest relevant additional schemas (if beneficial). 5. **optimization_suggestions** - Include best practices related to this specific page only (if applicable). 6. **priority_level** - Assign High, Medium, or Low to each recommendation (if any). ### Important Rules: - ❌ Do NOT mention how many pages were crawled. - ❌ Do NOT mention crawl depth, website size, or accessibility of other pages. - ❌ Do NOT add assumptions beyond the provided crawl data. - ✅ Focus only on the current page’s content and structure. - ✅ Keep tone professional, precise, and data-driven. - ✅ Output must always be a single JSON object. ### Example Output: { "seo_summary": "Page is mostly optimized but missing a canonical tag.", "detected_issues": ["Missing canonical tag", "2 broken links found"], "ai_recommendations": [ {"action": "Add a canonical tag for this page", "priority": "High"}, {"action": "Fix 2 broken links to improve crawlability", "priority": "High"} ], "structured_data_schema_feedback": { "existing": ["Organization"], "suggested": ["BreadcrumbList"] }, "optimization_suggestions": [ "Optimize large images for faster load times.", "Add descriptive alt text for all images." ], "priority_level": { "canonical_tag": "High", "broken_links": "High", "image_optimization": "Medium" } } """ agent: Agent = Agent( name="Web SEO Audit Agent", instructions=instructions, model=llm_model ) async def create_report(website_url: str, user_info: dict = None, user_email: str = None, plan: str = "free",report_id=None): try: # If user_info is provided, extract user_email and plan from it if user_info: if user_email is None: # Only extract from user_info if not provided separately user_email = user_info.get("email") # Update plan from user_info if available user_plan = user_info.get("plan") if user_plan: plan = user_plan # If user_info is not provided but user_email is, fetch user details from database elif user_email: user = await db["users"].find_one({"email": user_email}) if user: # Remove sensitive information user_info = {k: v for k, v in user.items() if k not in ["password", "otp_hash", "otp_expires", "reset_otp_hash", "reset_otp_expires"]} # Extract plan from user info if available user_plan = user_info.get("plan") if user_plan: plan = user_plan crawl_result = await crawl(website_url=website_url, plan=plan) # Handle the new return format from crawl function if isinstance(crawl_result, dict) and "status" in crawl_result: if crawl_result["status"] == "error": print(f"X Crawl failed: {crawl_result['message']}") await db["reports"].update_one( {"_id": ObjectId(report_id)}, {"$set": { "status": "failed", "error_message": crawl_result["message"], "updated_at": datetime.datetime.utcnow() }} ) return {"status": "error", "message": crawl_result["message"], "data": None} elif crawl_result["status"] == "success": crawl_data = crawl_result["data"] else: print(f"X Crawl returned unexpected status: {crawl_result}") await db["reports"].update_one( {"_id": ObjectId(report_id)}, {"$set": { "status": "failed", "error_message": crawl_result["message"], "updated_at": datetime.datetime.utcnow() }} ) return {"status": "error", "message": "Crawl returned unexpected status.", "data": None} else: # Fallback for old format (backward compatibility) crawl_data = crawl_result if not crawl_data or not isinstance(crawl_data, list): print(f"X Crawl failed or returned invalid data: {crawl_data}") await db["reports"].update_one( {"_id": ObjectId(report_id)}, {"$set": { "status": "failed", "error_message": "Crawl returned invalid data", "updated_at": datetime.datetime.utcnow() }} ) return {"status": "error", "message": "Crawl failed or returned invalid data.", "data": None} # ============================================ # RATE LIMIT PROTECTION: Limit pages per audit # ============================================ # Check remaining daily requests (20 requests/day hard limit) from reports.gemini_token_tracker import get_current_usage try: usage = await get_current_usage() requests_today = usage.get("requests_count", 0) remaining_requests = max(0, 20 - requests_today) if remaining_requests < len(crawl_data): print(f"⚠️ Low on daily requests: {remaining_requests} remaining. Limiting to {remaining_requests} pages.") crawl_data = crawl_data[:remaining_requests] except Exception as e: print(f"⚠️ Could not check request count: {e}") # ============================================ page_reports = [] total_tokens_used = 0 api_error_occurred = False api_error_message = None for page_data in crawl_data: url = page_data.get("url") if isinstance(page_data, dict) else None try: # Agent Layer for suggestion ai_result = await Runner.run( agent, input=json.dumps({"page_data": page_data}, ensure_ascii=False, indent=2), run_config=config ) # Extract token usage from AI result if available page_tokens = 0 if hasattr(ai_result, 'usage') and ai_result.usage: page_tokens = ai_result.usage.total_tokens or 0 total_tokens_used += page_tokens print(f"📊 Page {url}: {page_tokens} tokens used") # Convert string output into JSON object try: parsed_result = json.loads(ai_result.final_output.strip("`json\n").strip("`")) except json.JSONDecodeError: print(f"! Warning: Could not parse AI result for {url}. Storing raw string instead.") parsed_result = {"raw_output": ai_result.final_output} page_reports.append({ "url": url, "ai_result": parsed_result, "tokens_used": page_tokens }) except Exception as e: # Track API errors (including rate limits) api_error_occurred = True api_error_message = str(e) print(f"❌ API error for {url}: {e}") # Check if it's a rate limit error (429) if "429" in str(e) or "RESOURCE_EXHAUSTED" in str(e) or "exceeded your current quota" in str(e): print(f"⚠️ RATE LIMIT HIT: Gemini API quota exceeded!") # Track this as a failed request for monitoring await track_token_usage( tokens=0, # No tokens consumed on failed request endpoint="seo_audit_failed_rate_limit", user_email=user_email ) break # Stop processing more pages # Track total token usage for this audit (only if we had successful API calls) if total_tokens_used > 0: print(f"📊 Total tokens used for audit: {total_tokens_used}") await track_token_usage( tokens=total_tokens_used, endpoint="seo_audit", user_email=user_email ) elif api_error_occurred: # Log that audit completed with errors print(f"⚠️ Audit completed with API errors. No tokens tracked.") # print("AI Recommendation: ",page_reports) # Now generate final structured report # First, check SSL certificate ssl_info = await check_ssl_certificate_async(website_url) report_result = await generate_report( website=website_url, crawl_data=crawl_data, ai_recommendation=page_reports, user_info=user_info, plan_type=plan, ssl_info=ssl_info # Pass SSL info to PDF generator ) # Handle the new return format from generate_report function if isinstance(report_result, dict) and "status" in report_result: if report_result["status"] == "error": print(f"X Report generation failed: {report_result['message']}") await db["reports"].update_one( {"_id": ObjectId(report_id)}, {"$set": { "status": "failed", "error_message": report_result["message"], "updated_at": datetime.datetime.utcnow() }} ) return {"status": "error", "message": report_result["message"], "data": None} elif report_result["status"] == "success": report = report_result else: print(f"X Report generation returned unexpected status: {report_result}") return {"status": "error", "message": "Report generation returned unexpected status.", "data": None} else: # Fallback for old format (backward compatibility) report = {"data": report_result} user_id = user_info.get("_id") if user_info else None # print("Storing report in MongoDB...") report_data = report["data"] # Attach user reference report_data["user_id"] = ObjectId(user_id) # Add useful metadata report_data["website"] = website_url report_data["created_at"] = datetime.datetime.utcnow() report_data["status"] = "completed" # Update the existing "in-progress" report await db["reports"].update_one( {"_id": ObjectId(report_id)}, # pass report_id from /audit {"$set": { "status": "completed", "pdf": report_data.get("pdf"), # PDF bytes "score": report_data.get("score"), "issues": report_data.get("issues"), "ai_result": report_data.get("ai_result"), "updated_at": datetime.datetime.utcnow() }} ) pdf_data = report["data"]["pdf"] # Create a meaningful filename safe_site = website_url.replace("https://", "").replace("http://", "").replace("/", "_") timestamp = datetime.datetime.now().strftime("%Y-%m-%d") pdf_filename = f"SEO_Report_{safe_site}_{timestamp}.pdf" # Generate professional HTML email template from reports.email_sender import get_email_html_template # Extract username from user_info username = user_info.get("username") if user_info else None # Get backend URL from environment variable for email logo backend_url = os.getenv("BACKEND_URL", "http://localhost:8000") email_html = get_email_html_template( website_url=website_url, username=username, backend_url=backend_url ) # Send the PDF report to the user's email email_result = send_email_with_pdf( to_email=user_email, subject=f"Your SEO Audit Report for {website_url}", message=email_html, pdf_data=pdf_data, pdf_filename=pdf_filename ) # Update the existing "in-progress" report with email delivery status await db["reports"].update_one( {"_id": ObjectId(report_id)}, {"$set": { "status": "completed", "pdf": report_data.get("pdf"), # PDF bytes "score": report_data.get("score"), "issues": report_data.get("issues"), "ai_result": report_data.get("ai_result"), "email_sent": email_result["success"], "email_delivery_status": email_result["status"], "email_sent_at": datetime.datetime.utcnow() if email_result["success"] else None, "email_error": email_result.get("error"), "ssl_info": ssl_info, # Store SSL info in report for reference "updated_at": datetime.datetime.utcnow() }} ) # ============ SSL ALERT EMAIL ============ # Send SSL alert email if certificate is expiring within 7 days or already expired username = user_info.get("username") if user_info else None backend_url = os.getenv("BACKEND_URL", "http://localhost:8000") if ssl_info: is_expired = ssl_info.get("is_expired", False) days_until_expiry = ssl_info.get("days_until_expiry") # Send alert if expired OR expiring within 7 days should_send_alert = ( (is_expired) or (days_until_expiry is not None and 0 <= days_until_expiry <= 7) ) if should_send_alert: print(f"🔒 SSL Alert: Sending SSL alert email for {website_url} (expired: {is_expired}, days remaining: {days_until_expiry})") ssl_alert_result = send_ssl_alert_email( to_email=user_email, website_url=website_url, username=username, ssl_info=ssl_info, backend_url=backend_url ) # Update report with SSL alert status await db["reports"].update_one( {"_id": ObjectId(report_id)}, {"$set": { "ssl_alert_sent": ssl_alert_result["success"], "ssl_alert_sent_at": datetime.datetime.utcnow() if ssl_alert_result["success"] else None, "ssl_alert_error": ssl_alert_result.get("error"), "ssl_status": { "has_ssl": ssl_info.get("has_ssl"), "ssl_valid": ssl_info.get("ssl_valid"), "is_expired": ssl_info.get("is_expired"), "days_until_expiry": ssl_info.get("days_until_expiry"), "expiry_date": ssl_info.get("expiry_date"), "issuer": ssl_info.get("issuer") } }} ) if ssl_alert_result["success"]: print("✅ SSL alert email sent successfully") else: print(f"❌ Failed to send SSL alert email: {ssl_alert_result.get('error')}") else: # SSL is healthy, just store the info await db["reports"].update_one( {"_id": ObjectId(report_id)}, {"$set": { "ssl_status": { "has_ssl": ssl_info.get("has_ssl"), "ssl_valid": ssl_info.get("ssl_valid"), "is_expired": ssl_info.get("is_expired"), "days_until_expiry": ssl_info.get("days_until_expiry"), "expiry_date": ssl_info.get("expiry_date"), "issuer": ssl_info.get("issuer") } }} ) print(f"✓ SSL certificate healthy ({days_until_expiry} days remaining)") # ============ BUILD RESPONSE WITH SSL STATUS ============ ssl_status_response = None if ssl_info: ssl_status_response = { "has_ssl": ssl_info.get("has_ssl", False), "ssl_valid": ssl_info.get("ssl_valid", False), "is_expired": ssl_info.get("is_expired", False), "days_until_expiry": ssl_info.get("days_until_expiry"), "expiry_date": ssl_info.get("expiry_date").isoformat() if ssl_info.get("expiry_date") else None, "issuer": ssl_info.get("issuer", "Unknown"), "alert_sent": ssl_info.get("is_expired", False) or (ssl_info.get("days_until_expiry") is not None and 0 <= ssl_info.get("days_until_expiry") <= 7) } print("Report stored successfully with ID:") return { "status": "success", "data": { "message": "Report generated and stored successfully", "ssl_status": ssl_status_response } } except Exception as e: print("Unexpected error:", str(e)) await db["reports"].update_one( {"_id": ObjectId(report_id)}, {"$set": { "status": "failed", "error_message": str(e), "updated_at": datetime.datetime.utcnow() }} ) return { "status": "error", "message": f"Report generation failed: {str(e)}", "data": None } # ---------------- Run Locally ---------------- if __name__ == "__main__": import asyncio asyncio.run(create_report(website_url="https://blueverse.ae", user_email="subhancontact2@gmail.com", plan="free"))