Create main.py

main.py

@@ -0,0 +1,111 @@
+from flask import Flask, request, jsonify
+import os
+import json
+import time
+import subprocess
+import nest_asyncio
+from scrapegraphai.graphs import SearchGraph
+from flask_cors import CORS, cross_origin
+
+# Ensure Playwright installs required browsers and dependencies
+subprocess.run(["playwright", "install"])
+nest_asyncio.apply()
+
+app = Flask(__name__)
+CORS(app)
+
+# Set your Google API key as an environment variable.
+GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
+if not GOOGLE_API_KEY:
+    raise ValueError("GOOGLE_API_KEY environment variable is not set.")
+
+graph_config = {
+    "llm": {
+        "api_key": GOOGLE_API_KEY,
+        "model": "google_genai/gemini-2.0-flash-lite",
+    },
+    "max_results": 8,
+    "verbose": True,
+    "headless": True
+}
+
+def get_data(search_term):
+    """
+    Run the SearchGraph for a given search term.
+    If a rate-limit error (202) occurs, wait 10 seconds and retry.
+    """
+    full_prompt = (
+        f"search for {search_term} grants\n\n"
+        "List me all grants or funds with:\n"
+        "- Grant name/title\n"
+        "- Short summary \n"
+        "- Funding organization\n"
+        "- Grant value (numeric only)\n"
+        "- Application deadline\n"
+        "- Eligible countries\n"
+        "- Sector/field\n"
+        "- Eligibility criteria\n"
+        "Return in JSON format."
+    )
+    try:
+        search_graph = SearchGraph(prompt=full_prompt, config=graph_config)
+        result = search_graph.run()
+        if not result or not result.get("grants"):
+            return {"error": f"No results returned for '{search_term}'. Please try again with a different search term."}
+        return result
+    except Exception as e:
+        err_str = str(e)
+        if "202" in err_str:
+            time.sleep(10)
+            try:
+                search_graph = SearchGraph(prompt=full_prompt, config=graph_config)
+                result = search_graph.run()
+                if not result or not result.get("grants"):
+                    return {"error": f"No results returned for '{search_term}' after retry. Please try again with a different search term."}
+                return result
+            except Exception as e2:
+                return {"error": f"Retry failed for '{search_term}': {str(e2)}. Please try again later."}
+        else:
+            return {"error": f"An error occurred for '{search_term}': {str(e)}. Please try again."}
+
+def process_multiple_search_terms(search_terms):
+    """
+    Process multiple search terms and aggregate results.
+    Returns a dictionary with a 'grants' key containing combined results.
+    """
+    all_data = {"grants": []}
+    for term in search_terms:
+        term = term.strip()
+        if not term:
+            continue
+        result = get_data(term)
+        if result and result.get("grants"):
+            all_data["grants"].extend(result["grants"])
+    return all_data
+
+@app.route("/scrape", methods=["POST"])
+def scrape():
+    """
+    Endpoint to scrape grant opportunities.
+    Expects a JSON body with the key 'search_terms' (a string with newline-separated search terms
+    or a list of strings). Returns JSON with the aggregated results.
+    """
+    data = request.get_json()
+    if not data or "search_terms" not in data:
+        return jsonify({"error": "Request must include 'search_terms' key."}), 400
+
+    search_terms = data["search_terms"]
+    # If search_terms is a string, split it by newlines.
+    if isinstance(search_terms, str):
+        search_terms = [s.strip() for s in search_terms.split("\n") if s.strip()]
+    elif not isinstance(search_terms, list):
+        return jsonify({"error": "'search_terms' must be a string or list of strings."}), 400
+
+    if not search_terms:
+        return jsonify({"error": "No valid search terms provided."}), 400
+
+    result = process_multiple_search_terms(search_terms)
+    return jsonify(result), 200
+
+if __name__ == "__main__":
+    app.run(debug=True, host="0.0.0.0", port=7860)