chatbot4 / quotes_spider.py
ashutoshsharma58's picture
Create quotes_spider.py
2210390 verified
raw
history blame contribute delete
787 Bytes
import scrapy
from scrapy.crawler import CrawlerProcess
class QuotesSpider(scrapy.Spider):
name = "quotes"
start_urls = ['https://quotes.toscrape.com/']
def parse(self, response):
quotes = []
for quote in response.css('div.quote'):
quotes.append({
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
})
return quotes
def run_spider():
process = CrawlerProcess(settings={
"FEEDS": {
"quotes.json": {"format": "json"},
},
})
process.crawl(QuotesSpider)
process.start()
with open('quotes.json', 'r') as file:
return file.read()