File size: 399 Bytes
1505bbf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
import requests
from bs4 import BeautifulSoup

def scrape_url(url):

    headers = {"User-Agent":"Mozilla/5.0"}

    r = requests.get(url, headers=headers)

    soup = BeautifulSoup(r.text,"html.parser")

    elements = soup.find_all(["h1","h2","h3","p","li"])

    text = " ".join(
        el.get_text(strip=True)
        for el in elements
        if el.get_text(strip=True)
    )

    return text