Spaces:

lenson78
/

Scrapling

Paused

Karim shoair commited on Sep 14, 2025

Commit

88842b1

1 Parent(s): 76ae95e

tests: Multiple changes to tests

- Remove all tests for the old encoding logic
- Stop using the `nopecha` test page to test Cloudflare solver
- Remove useless tests like testing for infinite timeout
- Fixes to make the code compatible with new changes

Files changed (7) hide show

tests/cli/test_cli.py +1 -0
tests/fetchers/async/test_camoufox.py +2 -12
tests/fetchers/async/test_dynamic.py +0 -6
tests/fetchers/sync/test_camoufox.py +0 -4
tests/fetchers/test_response_handling.py +0 -15
tests/fetchers/test_utils.py +1 -7
tests/parser/test_parser_advanced.py +1 -1

tests/cli/test_cli.py CHANGED Viewed

@@ -14,6 +14,7 @@ def configure_selector_mock():
     """Helper function to create a properly configured Selector mock"""
     mock_response = MagicMock(spec=Selector)
     mock_response.body = "<html><body>Test content</body></html>"
     mock_response.get_all_text.return_value = "Test content"
     mock_response.css_first.return_value = mock_response
     mock_response.css.return_value = [mock_response]

     """Helper function to create a properly configured Selector mock"""
     mock_response = MagicMock(spec=Selector)
     mock_response.body = "<html><body>Test content</body></html>"
+    mock_response.encoding = "utf-8"
     mock_response.get_all_text.return_value = "Test content"
     mock_response.css_first.return_value = mock_response
     mock_response.css.return_value = [mock_response]

tests/fetchers/async/test_camoufox.py CHANGED Viewed

@@ -1,3 +1,4 @@
 import pytest
 import pytest_httpbin
@@ -23,14 +24,9 @@ class TestStealthyFetcher:
             "basic_url": f"{url}/get",
             "html_url": f"{url}/html",
             "delayed_url": f"{url}/delay/10",  # 10 Seconds delay response
-            "cookies_url": f"{url}/cookies/set/test/value",
-            "cloudflare_url": "https://nopecha.com/demo/cloudflare",  # Interactive turnstile page
         }
-    async def test_cloudflare_fetch(self, fetcher, urls):
-        """Test if Cloudflare bypass is working"""
-        assert (await fetcher.async_fetch(urls["cloudflare_url"], solve_cloudflare=True)).status == 200
     async def test_basic_fetch(self, fetcher, urls):
         """Test doing a basic fetch request with multiple statuses"""
         assert (await fetcher.async_fetch(urls["status_200"])).status == 200
@@ -86,9 +82,3 @@ class TestStealthyFetcher:
             **kwargs
         )
         assert response.status == 200
-    async def test_infinite_timeout(self, fetcher, urls):
-        """Test if infinite timeout breaks the code or not"""
-        assert (
-            await fetcher.async_fetch(urls["delayed_url"], timeout=0)
-        ).status == 200

+from playwright._impl._errors import TimeoutError
 import pytest
 import pytest_httpbin
             "basic_url": f"{url}/get",
             "html_url": f"{url}/html",
             "delayed_url": f"{url}/delay/10",  # 10 Seconds delay response
+            "cookies_url": f"{url}/cookies/set/test/value"
         }
     async def test_basic_fetch(self, fetcher, urls):
         """Test doing a basic fetch request with multiple statuses"""
         assert (await fetcher.async_fetch(urls["status_200"])).status == 200
             **kwargs
         )
         assert response.status == 200

tests/fetchers/async/test_dynamic.py CHANGED Viewed

@@ -90,9 +90,3 @@ class TestDynamicFetcherAsync:
         with pytest.raises(Exception):
             await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah")
-    @pytest.mark.asyncio
-    async def test_infinite_timeout(self, fetcher, urls):
-        """Test if infinite timeout breaks the code or not"""
-        response = await fetcher.async_fetch(urls["delayed_url"], timeout=0)
-        assert response.status == 200


90
91	with pytest.raises(Exception):
92	await fetcher.async_fetch(urls["html_url"], cdp_url="ws://blahblah")

tests/fetchers/sync/test_camoufox.py CHANGED Viewed

@@ -77,7 +77,3 @@ class TestStealthyFetcher:
             **kwargs
         )
         assert response.status == 200
-    def test_infinite_timeout(self, fetcher):
-        """Test if infinite timeout breaks the code or not"""
-        assert fetcher.fetch(self.delayed_url, timeout=0).status == 200

             **kwargs
         )
         assert response.status == 200

tests/fetchers/test_response_handling.py CHANGED Viewed

@@ -1,7 +1,6 @@
 from unittest.mock import Mock
 from scrapling.parser import Selector
-from scrapling.engines.toolbelt.custom import ResponseEncoding
 from scrapling.engines.toolbelt.convertor import ResponseFactory, Response
@@ -32,20 +31,6 @@ class TestResponseFactory:
         assert response.url == "https://example.com"
         assert isinstance(response, Response)
-    def test_response_encoding_edge_cases(self):
-        """Test response encoding handling"""
-        # Test various content types
-        test_cases = [
-            (None, "utf-8"),
-            ("", "utf-8"),
-            ("text/html; charset=invalid", "utf-8"),
-            ("application/octet-stream", "utf-8"),
-        ]
-        for content_type, expected in test_cases:
-            encoding = ResponseEncoding.get_value(content_type)
-            assert encoding == expected
     def test_response_history_processing(self):
         """Test processing response history"""
         # Mock responses with redirects

 from unittest.mock import Mock
 from scrapling.parser import Selector
 from scrapling.engines.toolbelt.convertor import ResponseFactory, Response
         assert response.url == "https://example.com"
         assert isinstance(response, Response)
     def test_response_history_processing(self):
         """Test processing response history"""
         # Mock responses with redirects

tests/fetchers/test_utils.py CHANGED Viewed

@@ -1,7 +1,7 @@
 import pytest
 from pathlib import Path
-from scrapling.engines.toolbelt.custom import ResponseEncoding, StatusText, Response
 from scrapling.engines.toolbelt.navigation import (
     construct_proxy_dict,
     js_bypass_path
@@ -131,12 +131,6 @@ def status_map():
     }
-def test_parsing_content_type(content_type_map):
-    """Test if parsing different types of 'content-type' returns the expected result"""
-    for header_value, expected_encoding in content_type_map.items():
-        assert ResponseEncoding.get_value(header_value) == expected_encoding
 def test_parsing_response_status(status_map):
     """Test if using different http responses' status codes returns the expected result"""
     for status_code, expected_status_text in status_map.items():

 import pytest
 from pathlib import Path
+from scrapling.engines.toolbelt.custom import StatusText, Response
 from scrapling.engines.toolbelt.navigation import (
     construct_proxy_dict,
     js_bypass_path
     }
 def test_parsing_response_status(status_map):
     """Test if using different http responses' status codes returns the expected result"""
     for status_code, expected_status_text in status_map.items():

tests/parser/test_parser_advanced.py CHANGED Viewed

@@ -99,7 +99,7 @@ class TestAdvancedSelectors:
             keep_comments=False,
             keep_cdata=False
         )
-        content = page.body
         assert "Comment" not in content
     def test_advanced_xpath_variables(self, complex_html):

             keep_comments=False,
             keep_cdata=False
         )
+        content = page.html_content
         assert "Comment" not in content
     def test_advanced_xpath_variables(self, complex_html):