Spaces:
Sleeping
Sleeping
File size: 1,486 Bytes
9f2df60 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | import pytest, os
from src.utils.logging import init_logging
from src.scraping.scraper import Scraper
from src.scraping.url_normalizer import UrlNormalizer
from src.config import config
pytestmark = [pytest.mark.network, pytest.mark.integration]
class TestHappyPath:
def test_happy_path(self):
init_logging()
target_url = 'https://emba.unisg.ch/'
url_filename = UrlNormalizer().url_to_filename(target_url)
scraper = Scraper(scrape_all=True)
results = scraper.scrape_target(target_url)
logs_file_path = os.path.join(config.paths.LOGS, 'scraping.log')
raw_html_file_path = os.path.join(config.paths.RAW_HTML_OUTPUT, url_filename + '.html')
extracted_text_file_path = os.path.join(config.paths.EXTRACTED_TEXT_OUTPUT, url_filename + '.txt')
# Test that the logs are being written
assert os.path.exists(logs_file_path)
with open(logs_file_path, 'r') as f:
assert len(f.read()) > 0
# Test whether the raw HTML was extracted and saved
assert os.path.exists(raw_html_file_path)
with open(raw_html_file_path, 'r') as f:
assert len(f.read()) > 0
# Test whether the text was extracted and saved
assert os.path.exists(extracted_text_file_path)
with open(raw_html_file_path, 'r') as f:
assert len(f.read()) > 0
if __name__ == '__main__':
pytest.main([__file__, '-v'])
|