Update main.py

main.py

@@ -21,6 +21,39 @@ app.add_middleware(
 
 @app.get("/get_scraped_data")
 async def get_data(url: str):
+    import requests
+    from bs4 import BeautifulSoup
+    
+    # URL of the page to scrape
+    #url = "https://www.imf.org/en/News/Articles/2024/03/21/pr2494-sri-lanka-imf-staff-level-agreement-for-second-review-sla"
+    url = url
+    
+    # Send a GET request to the URL
+    response = requests.get(url)
+    
+    # Check if the request was successful
+    if response.status_code == 200:
+        # Parse the page content
+        soup = BeautifulSoup(response.content, 'html.parser')
+    
+        # Extract all text content (paragraphs, headers, etc.)
+        elements = soup.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'])
+        body_text = "\n".join([element.get_text().strip() for element in elements])
+    
+        # Extract all links
+        links = []
+        for a_tag in soup.find_all('a', href=True):
+            links.append(a_tag['href'])
+    
+        # Print the extracted information
+        print("Body Text:")
+        print(body_text)
+        print("\nLinks:")
+        for link in links:
+            print(link)
+    else:
+        print("Failed to retrieve the webpage")
+    return "done"
     try:
         data = await Scraper.scrape(url)
         return data