Update scraper.py
Browse files- scraper.py +2 -5
scraper.py
CHANGED
|
@@ -25,11 +25,8 @@ class Scraper:
|
|
| 25 |
|
| 26 |
# Get page content (paragraphs, headers)
|
| 27 |
page_content = await page.evaluate("""() => {
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
tag: element.tagName,
|
| 31 |
-
text: element.innerText
|
| 32 |
-
}));
|
| 33 |
}""")
|
| 34 |
|
| 35 |
# Print the results
|
|
|
|
| 25 |
|
| 26 |
# Get page content (paragraphs, headers)
|
| 27 |
page_content = await page.evaluate("""() => {
|
| 28 |
+
let elements = Array.from(document.querySelectorAll('p, h1, h2, h3, h4, h5, h6'));
|
| 29 |
+
return elements.map(element => element.innerText).join('\\n');
|
|
|
|
|
|
|
|
|
|
| 30 |
}""")
|
| 31 |
|
| 32 |
# Print the results
|