v3

modules/__init__.py

@@ -32,6 +32,8 @@ from .inputs import (
     # objectives,
 )
 
+from .serper_env import SERPER_API_KEYS, get_valid_serper_key,get_serper_credits
+
 # from .directory import (
 #     output_dir,
 #     keywords_dir,

modules/serper_env.py

@@ -0,0 +1,40 @@
+import http.client
+import json
+import os
+import requests
+SERPER_API_KEYS = [
+    os.getenv("SERPER_API_KEY_1", ""),
+    os.getenv("SERPER_API_KEY_2", ""),
+    os.getenv("SERPER_API_KEY_3", ""),
+]
+
+CURRENT_SERPER_INDEX = 0
+
+
+def get_serper_credits(api_key: str) -> int:
+    try:
+        conn = http.client.HTTPSConnection("google.serper.dev")
+        headers = {"X-API-KEY": api_key}
+        conn.request("GET", "/credits", headers=headers)
+        res = conn.getresponse()
+        data = json.loads(res.read().decode("utf-8"))
+        return int(data.get("credits", 0))
+    except:
+        return 0
+
+
+def get_valid_serper_key(min_credits=100):
+    global CURRENT_SERPER_INDEX
+
+    for idx, key in enumerate(SERPER_API_KEYS):
+        if not key:
+            continue
+        credits = get_serper_credits(key)
+        if credits >= min_credits:
+            CURRENT_SERPER_INDEX = idx
+            return key, credits, CURRENT_SERPER_INDEX
+
+    raise Exception(
+        "❌ SERPER credits exhausted on all APIs. Please contact the developer. "
+        "❌ يجب التواصل مع المطور لإضافة API Keys جديدة لSERPER"
+    )

routers/content_generation_route.py

@@ -15,6 +15,8 @@ from core import generate_course_content, convert_dict_to_html
 from modules import (
     llm,
     inputs,
+    get_valid_serper_key,
+    get_serper_credits,
 )
 from tools import scrape_course
 from schemas import (
@@ -45,21 +47,37 @@ def run_keywords(outlines: CourseContent):
 
 
 @router.post("/scrape")
-def scrape_endpoint(course:CourseWithQueries):
+def scrape_endpoint(course: CourseWithQueries):
     try:
-        # نحول Pydantic model لـ dict بنفس الشكل اللي الفانكشن مستنياها
+        # 1️⃣ اختار API Key صالح
+        api_key, credits_before, CURRENT_SERPER_INDEX = get_valid_serper_key(
+            min_credits=100
+        )
+
+        # 2️⃣ شغّل السكربنج
         course_dict = course.dict()
         summary, raw = scrape_course(course_dict)
+
+        # 3️⃣ احسب الكريديتس بعد التشغيل
+        credits_after = get_serper_credits(api_key)
+
         return {
             "message": "Queries Searched and Scraped Well 🚀",
+            "serper": {
+                "api_index": CURRENT_SERPER_INDEX,
+                "credits_before": credits_before,
+                "credits_after": credits_after,
+                "credits_used": credits_before - credits_after,
+            },
             "summary": summary,
             "raw": raw,
         }
 
-        return {"message": "Queries Searched and Scraped Well 🚀", "summary": summary, "raw": raw}
     except Exception as e:
-        # في البروداكشن خليه logging و متطلعش الرسالة الخام للعميل
-        raise HTTPException(status_code=500, detail=str(e))
+        return {
+            "error": True,
+            "message": str(e),
+        }
 
 
 @router.post("/content_generation")

tools/searching_scraping_queries.py

@@ -8,11 +8,11 @@ import requests
 from bs4 import BeautifulSoup
 from PyPDF2 import PdfReader
 from concurrent.futures import ThreadPoolExecutor
-
+from modules import SERPER_API_KEYS
 # -------- CONFIG ----------
 import os
 
-SERPER_API_KEY = os.getenv("SERPER_API_KEY", "")
+# SERPER_API_KEY = os.getenv("SERPER_API_KEY", "")
 BAD_KEYWORDS = [
     "facebook",
     "youtube",
@@ -88,19 +88,21 @@ def scrape_page(url, retries=RETRIES, limit=SCRAPE_CHAR_LIMIT):
     return ""
 
 
-def search_serper(query, max_results=SERP_MAX_RESULTS):
-    if not SERPER_API_KEY:
-        # في البروداكشن خليه raise Exception أحسن
+def search_serper(query, CURRENT_SERPER_INDEX, max_results=SERP_MAX_RESULTS):
+    api_key = SERPER_API_KEYS[CURRENT_SERPER_INDEX]
+    if not api_key:
         return []
 
     try:
         conn = http.client.HTTPSConnection("google.serper.dev")
         payload = json.dumps({"q": query, "page": 1})
-        headers = {"X-API-KEY": SERPER_API_KEY, "Content-Type": "application/json"}
+        headers = {
+            "X-API-KEY": api_key,
+            "Content-Type": "application/json",
+        }
         conn.request("POST", "/search", payload, headers)
         res = conn.getresponse()
-        data = res.read()
-        response_json = json.loads(data.decode("utf-8"))
+        response_json = json.loads(res.read().decode("utf-8"))
     except:
         return []
 
@@ -157,7 +159,7 @@ def safe_append(summary_result, raw_result, final_output_summary, final_output_r
         )
 
 
-def scrape_course(course: dict):
+def scrape_course(course: dict,CURRENT_SERPER_INDEX):
     """
     course هنا هو نفس الستركشر اللي كان جاي من JSON file
     ويرجع dicts: (final_output_summary, final_output_raw)
@@ -217,7 +219,7 @@ def scrape_course(course: dict):
                     print(
                         f"\n🔍 [SEARCH] U{unit_idx} T{topic_idx} S{sub_idx} — Query: {query_text}"
                     )
-                    results = search_serper(query_text, max_results=SERP_MAX_RESULTS)
+                    results = search_serper(query_text, CURRENT_SERPER_INDEX, max_results=SERP_MAX_RESULTS)
 
                     for r in results:
                         with lock: