Update main.py

main.py

@@ -2,18 +2,11 @@ from flask import Flask, request, jsonify
 import os
 import json
 import time
-import subprocess
-import nest_asyncio
-import requests  # For API fallback call to Supadata
-from scrapegraphai.graphs import SearchGraph
 from flask_cors import CORS
 from google import genai
 from google.genai import types
-from supadata import Supadata, SupadataError
-
-# Ensure Playwright installs required browsers and dependencies
-subprocess.run(["playwright", "install"])
-nest_asyncio.apply()
+from exa_py import Exa
+from linkup import LinkupClient
 
 app = Flask(__name__)
 CORS(app)
@@ -23,63 +16,104 @@ GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY")
 if not GOOGLE_API_KEY:
     raise ValueError("GOOGLE_API_KEY environment variable is not set.")
 
-SUPADATA_API_KEY = os.environ.get("SUPADATA")
-if not SUPADATA_API_KEY:
-    raise ValueError("SUPADATA_API_KEY environment variable is not set.")
+EXA_API_KEY = os.environ.get("EXA_API_KEY")
+if not EXA_API_KEY:
+    raise ValueError("EXA_API_KEY environment variable is not set.")
 
-# Initialize Supadata client
-supadata = Supadata(api_key=SUPADATA_API_KEY)
+LINKUP_API_KEY = os.environ.get("LINKUP_API_KEY")
+if not LINKUP_API_KEY:
+    raise ValueError("LINKUP_API_KEY environment variable is not set.")
 
-graph_config = {
-    "llm": {
-        "api_key": GOOGLE_API_KEY,
-        "model": "google_genai/gemini-2.0-flash-lite",
-    },
-    "max_results": 8,
-    "verbose": True,
-    "headless": True
-}
+# Initialize clients
+exa = Exa(api_key=EXA_API_KEY)
+linkup_client = LinkupClient(api_key=LINKUP_API_KEY)
 
 
 def get_data(search_term):
     """
-    Run the SearchGraph for a given search term.
-    If a rate-limit error (202) occurs, wait 10 seconds and retry.
+    Run the Linkup deep search for a given search term.
+    If a rate-limit error occurs, wait 10 seconds and retry.
     """
-    full_prompt = (
-        f"search for {search_term} grants\n\n"
-        "List me all grants or funds with:\n"
-        "- Grant name/title\n"
-        "- Short summary \n"
-        "- Funding organization\n"
-        "- Grant value (numeric only)\n"
-        "- Application deadline\n"
-        "- Eligible countries\n"
-        "- Sector/field\n"
-        "- Eligibility criteria\n"
-        "- link URL\n"
-        "Return in JSON format."
-    )
+    full_query = f"{search_term} grants funding opportunities"
 
     print("\n=== DEBUG: Start get_data() ===")
     print(f"Search Term: {search_term}")
-    print(f"Full Prompt:\n{full_prompt}\n")
+    print(f"Full Query: {full_query}\n")
 
     try:
-        search_graph = SearchGraph(prompt=full_prompt, config=graph_config)
-        result = search_graph.run()
-        print("\n=== DEBUG: Raw result from search_graph.run() ===")
-        print(result)
+        response = linkup_client.search(
+            query=full_query,
+            depth="deep",
+            output_type="sourcedAnswer",
+            include_images=False,
+        )
+        
+        print("\n=== DEBUG: Raw result from linkup search ===")
+        print(response)
         print("===========================================")
 
-        # Ensure result is in dictionary format
-        if isinstance(result, str):
-            try:
-                result = json.loads(result)
-                print("DEBUG: Successfully parsed JSON string into dictionary.")
-            except json.JSONDecodeError:
-                print("ERROR: Failed to parse JSON from search result.")
-                return {"error": "Failed to parse JSON from search result."}
+        # Extract the answer content from Linkup response
+        content = ""
+        if hasattr(response, 'answer'):
+            content = response.answer
+        elif isinstance(response, dict) and 'answer' in response:
+            content = response['answer']
+        else:
+            content = str(response)
+
+        # Process the content with Gemini AI to extract structured grant data
+        structured_prompt = (
+            f"Based on the following search results about {search_term} grants, "
+            "extract and structure grant information with:\n"
+            "- Grant name/title\n"
+            "- Short summary \n"
+            "- Funding organization\n"
+            "- Grant value (numeric only)\n"
+            "- Application deadline\n"
+            "- Eligible countries\n"
+            "- Sector/field\n"
+            "- Eligibility criteria\n"
+            "- link URL\n"
+            "Return in JSON format with a 'grants' array.\n\n"
+            f"Search results: {content}"
+        )
+
+        client = genai.Client(api_key=GOOGLE_API_KEY)
+        gemini_response = client.models.generate_content(
+            model="models/gemini-2.0-flash-lite",
+            contents=f"{structured_prompt}, return the json string and nothing else"
+        )
+        
+        gemini_text = gemini_response.text
+        print(f"DEBUG: Gemini response: {gemini_text}")
+
+        # Parse JSON from Gemini response
+        try:
+            # Try to find JSON in the response
+            start_index = gemini_text.find('{')
+            if start_index == -1:
+                start_index = gemini_text.find('[')
+            
+            if start_index != -1:
+                if gemini_text[start_index] == '{':
+                    end_index = gemini_text.rfind('}') + 1
+                else:
+                    end_index = gemini_text.rfind(']') + 1
+                
+                json_string = gemini_text[start_index:end_index]
+                result = json.loads(json_string)
+                
+                # Ensure result has grants array
+                if isinstance(result, list):
+                    result = {"grants": result}
+                elif isinstance(result, dict) and "grants" not in result:
+                    # If it's a dict but no grants key, assume it's a single grant
+                    result = {"grants": [result]}
+            else:
+                result = {"grants": []}
+        except json.JSONDecodeError as je:
+            print(f"ERROR: Failed to parse JSON from Gemini response: {je}")
+            result = {"grants": []}
 
         if not result or "grants" not in result or not result["grants"]:
             print(f"DEBUG: No grants found for '{search_term}'.")
@@ -92,20 +126,81 @@ def get_data(search_term):
         err_str = str(e)
         print(f"ERROR: Exception occurred - {err_str}")
 
-        if "202" in err_str:
-            print("DEBUG: Rate limit (202) detected. Retrying in 10 seconds...")
+        # Check for rate limiting or similar errors
+        if "rate" in err_str.lower() or "limit" in err_str.lower():
+            print("DEBUG: Rate limit detected. Retrying in 10 seconds...")
             time.sleep(10)
             try:
-                search_graph = SearchGraph(prompt=full_prompt, config=graph_config)
-                result = search_graph.run()
-                print("\n=== DEBUG: Retrying search_graph.run() ===")
-                print(result)
-                print("===========================================")
+                response = linkup_client.search(
+                    query=full_query,
+                    depth="deep",
+                    output_type="sourcedAnswer",
+                    include_images=False,
+                )
+                
+                # Process retry response similar to above
+                content = ""
+                if hasattr(response, 'answer'):
+                    content = response.answer
+                elif isinstance(response, dict) and 'answer' in response:
+                    content = response['answer']
+                else:
+                    content = str(response)
+
+                structured_prompt = (
+                    f"Based on the following search results about {search_term} grants, "
+                    "extract and structure grant information with:\n"
+                    "- Grant name/title\n"
+                    "- Short summary \n"
+                    "- Funding organization\n"
+                    "- Grant value (numeric only)\n"
+                    "- Application deadline\n"
+                    "- Eligible countries\n"
+                    "- Sector/field\n"
+                    "- Eligibility criteria\n"
+                    "- link URL\n"
+                    "Return in JSON format with a 'grants' array.\n\n"
+                    f"Search results: {content}"
+                )
+
+                client = genai.Client(api_key=GOOGLE_API_KEY)
+                gemini_response = client.models.generate_content(
+                    model="models/gemini-2.0-flash-lite",
+                    contents=f"{structured_prompt}, return the json string and nothing else"
+                )
+                
+                gemini_text = gemini_response.text
+                
+                try:
+                    start_index = gemini_text.find('{')
+                    if start_index == -1:
+                        start_index = gemini_text.find('[')
+                    
+                    if start_index != -1:
+                        if gemini_text[start_index] == '{':
+                            end_index = gemini_text.rfind('}') + 1
+                        else:
+                            end_index = gemini_text.rfind(']') + 1
+                        
+                        json_string = gemini_text[start_index:end_index]
+                        result = json.loads(json_string)
+                        
+                        if isinstance(result, list):
+                            result = {"grants": result}
+                        elif isinstance(result, dict) and "grants" not in result:
+                            result = {"grants": [result]}
+                    else:
+                        result = {"grants": []}
+                except json.JSONDecodeError:
+                    result = {"grants": []}
+
                 if not result or "grants" not in result or not result["grants"]:
                     print(f"DEBUG: No grants found after retry for '{search_term}'.")
                     return {"error": f"No results returned for '{search_term}' after retry. Please try again with a different search term."}
+                
                 print("DEBUG: Grants found on retry, returning results.")
                 return result
+                
             except Exception as e2:
                 print(f"ERROR: Retry failed - {str(e2)}")
                 return {"error": f"Retry failed for '{search_term}': {str(e2)}. Please try again later."}
@@ -155,80 +250,105 @@ def scrape():
 
 def get_data_from_url(url):
     """
-    Scrape the provided URL using Supadata. If it fails, fall back to the Supadata API.
+    Scrape the provided URL using Exa API.
     Extract grant data using Gemini AI.
     """
-    page_content = None  # Placeholder for storing scraped page content
+    print(f"\n=== DEBUG: Start get_data_from_url() ===")
+    print(f"URL: {url}")
 
-    # Step 1: Attempt Supadata's built-in scraper
     try:
-        web_content = supadata.web.scrape(url)
-        page_content = web_content.content
-    except TypeError as te:
-        if "unexpected keyword argument 'type'" in str(te):
-            print("Falling back to Supadata API due to unexpected keyword 'type' error.")
+        # Use Exa to get content from URL
+        result = exa.get_contents(
+            [url],
+            text=True
+        )
+        
+        print("\n=== DEBUG: Raw result from Exa ===")
+        print(result)
+        print("=====================================")
+
+        # Extract text content from Exa response
+        page_content = ""
+        if hasattr(result, 'results') and result.results:
+            page_content = result.results[0].text if hasattr(result.results[0], 'text') else str(result.results[0])
+        elif isinstance(result, dict) and 'results' in result and result['results']:
+            page_content = result['results'][0].get('text', str(result['results'][0]))
         else:
-            print(f"Unexpected error in Supadata scrape: {te}")
+            page_content = str(result)
 
-    # Step 2: If Supadata's built-in scraper fails, use Supadata API
-    if not page_content:
+        if not page_content:
+            print("ERROR: No content extracted from URL")
+            return {}
+
+        print(f"DEBUG: Extracted content length: {len(page_content)}")
+
+        # Process content with Gemini AI
+        full_prompt = (
+            "Extract the following grant data from the provided web content. "
+            "- Grant name/title\n"
+            "- Short summary\n"
+            "- Funding organization\n"
+            "- Grant value (numeric only)\n"
+            "- Application deadline\n"
+            "- Eligible countries\n"
+            "- Sector/field\n"
+            "- Eligibility criteria\n"
+            "Return in JSON format with a 'grants' array.\n\n"
+            f"Web content: {page_content[:10000]}"  # Limit content to avoid token limits
+        )
+        
+        client = genai.Client(api_key=GOOGLE_API_KEY)
+        gemini_response = client.models.generate_content(
+            model="models/gemini-2.0-flash-lite",
+            contents=f"{full_prompt}, return the json string and nothing else"
+        )
+        
+        response_text = gemini_response.text
+        print(f"DEBUG: Gemini response: {response_text}")
+
+        # Extract JSON output from Gemini
         try:
-            api_url = "https://api.supadata.ai/v1/web/scrape"
-            headers = {"X-API-Key": SUPADATA_API_KEY}
-            response = requests.get(api_url, headers=headers, params={"url": url})
-            if response.status_code == 200:
-                page_content = response.json().get("content", "")
+            start_index = response_text.find('[')
+            if start_index == -1:
+                start_index = response_text.find('{')
+            
+            if start_index != -1:
+                if response_text[start_index] == '[':
+                    end_index = response_text.rfind(']') + 1
+                else:
+                    end_index = response_text.rfind('}') + 1
+                
+                json_string = response_text[start_index:end_index]
+                parsed_result = json.loads(json_string)
+                
+                # Ensure JSON is wrapped correctly
+                if isinstance(parsed_result, list):
+                    parsed_result = {"grants": parsed_result}
+                elif isinstance(parsed_result, dict) and "grants" not in parsed_result:
+                    # If it's a dict but no grants key, assume it's a single grant
+                    parsed_result = {"grants": [parsed_result]}
             else:
-                print(f"Supadata API failed with status {response.status_code}")
-                return {}
-        except Exception as e:
-            print(f"Error calling Supadata API: {e}")
+                parsed_result = {"grants": []}
+                
+        except Exception as parse_error:
+            print(f"Error parsing JSON from Gemini model response: {parse_error}")
+            print(f"Response: {response_text}")
             return {}
 
-    # Pass content to Gemini AI
-    full_prompt = (
-        "Extract the following grant data from the provided web content. "
-        "- Grant name/title\n"
-        "- Short summary\n"
-        "- Funding organization\n"
-        "- Grant value (numeric only)\n"
-        "- Application deadline\n"
-        "- Eligible countries\n"
-        "- Sector/field\n"
-        "- Eligibility criteria\n"
-        "Return in JSON format.\n\n"
-        f"Web content: {page_content}"
-    )
-    
-    client = genai.Client(api_key=GOOGLE_API_KEY)
-    new_answer = client.models.generate_content(
-        model="models/gemini-2.0-flash-lite",
-        contents=f"{full_prompt}, return the json string and nothing else"
-    )
-    
-    response = new_answer.text
-
-    # Extract JSON output from Gemini
-    try:
-        start_index = response.find('[')
-        end_index = response.rfind(']') + 1
-        json_string = response[start_index:end_index]
-        result = json.loads(json_string)
-    except Exception as parse_error:
-        print(f"Error parsing JSON from Gemini model response. Response: {response}")
-        return {}
+        if not parsed_result.get("grants"):
+            print("No grant opportunities found in the scraped URL.")
+            return {}
 
-    # Ensure JSON is wrapped correctly
-    if isinstance(result, list):
-        result = {"grants": result}
+        print(f"DEBUG: Found {len(parsed_result['grants'])} grants")
+        if parsed_result['grants']:
+            print(f"First grant opportunity: {parsed_result['grants'][0]}")
+        
+        return parsed_result
 
-    if not result.get("grants"):
-        print("No grant opportunities found in the scraped URL.")
+    except Exception as e:
+        print(f"ERROR: Exception in get_data_from_url: {str(e)}")
         return {}
 
-    print(f"First grant opportunity: {result['grants'][0]}")
-    return result
-
 
 @app.route("/scrape_url", methods=["POST"])
 def scrape_url():
@@ -250,4 +370,4 @@ def scrape_url():
 
 
 if __name__ == "__main__":
-    app.run(debug=True, host="0.0.0.0", port=7860)
+    app.run(debug=True, host="0.0.0.0", port=7860)