Karim shoair commited on
Commit
65cee4e
·
1 Parent(s): d4c3d51

test: add tests for the new `impersonate` logic

Browse files
tests/cli/test_cli.py CHANGED
@@ -193,3 +193,51 @@ class TestCLI:
193
  [html_url, 'output.invalid']
194
  )
195
  # Should handle the error gracefully
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
193
  [html_url, 'output.invalid']
194
  )
195
  # Should handle the error gracefully
196
+
197
+ def test_impersonate_comma_separated(self, runner, tmp_path, html_url):
198
+ """Test that comma-separated impersonate values are parsed correctly"""
199
+ output_file = tmp_path / "output.md"
200
+
201
+ with patch('scrapling.fetchers.Fetcher.get') as mock_get:
202
+ mock_response = configure_selector_mock()
203
+ mock_response.status = 200
204
+ mock_get.return_value = mock_response
205
+
206
+ result = runner.invoke(
207
+ get,
208
+ [
209
+ html_url,
210
+ str(output_file),
211
+ '--impersonate', 'chrome,firefox,safari'
212
+ ]
213
+ )
214
+ assert result.exit_code == 0
215
+
216
+ # Verify that the impersonate argument was converted to a list
217
+ call_kwargs = mock_get.call_args[1]
218
+ assert isinstance(call_kwargs['impersonate'], list)
219
+ assert call_kwargs['impersonate'] == ['chrome', 'firefox', 'safari']
220
+
221
+ def test_impersonate_single_browser(self, runner, tmp_path, html_url):
222
+ """Test that single impersonate value remains as string"""
223
+ output_file = tmp_path / "output.md"
224
+
225
+ with patch('scrapling.fetchers.Fetcher.get') as mock_get:
226
+ mock_response = configure_selector_mock()
227
+ mock_response.status = 200
228
+ mock_get.return_value = mock_response
229
+
230
+ result = runner.invoke(
231
+ get,
232
+ [
233
+ html_url,
234
+ str(output_file),
235
+ '--impersonate', 'chrome'
236
+ ]
237
+ )
238
+ assert result.exit_code == 0
239
+
240
+ # Verify that the impersonate argument remains a string
241
+ call_kwargs = mock_get.call_args[1]
242
+ assert isinstance(call_kwargs['impersonate'], str)
243
+ assert call_kwargs['impersonate'] == 'chrome'
tests/fetchers/test_impersonate_list.py ADDED
@@ -0,0 +1,153 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Test suite for list-based impersonate parameter functionality."""
2
+ import pytest
3
+ import pytest_httpbin
4
+ from unittest.mock import patch, MagicMock
5
+
6
+ from scrapling import Fetcher
7
+ from scrapling.fetchers import FetcherSession
8
+ from scrapling.engines.static import _select_random_browser
9
+
10
+
11
+ class TestRandomBrowserSelection:
12
+ """Test the random browser selection helper function."""
13
+
14
+ def test_select_random_browser_with_single_string(self):
15
+ """Test that single browser string is returned as-is."""
16
+ result = _select_random_browser("chrome")
17
+ assert result == "chrome"
18
+
19
+ def test_select_random_browser_with_none(self):
20
+ """Test that None is returned as-is."""
21
+ result = _select_random_browser(None)
22
+ assert result is None
23
+
24
+ def test_select_random_browser_with_list(self):
25
+ """Test that a browser is randomly selected from a list."""
26
+ browsers = ["chrome", "firefox", "safari"]
27
+ result = _select_random_browser(browsers)
28
+ assert result in browsers
29
+
30
+ def test_select_random_browser_with_empty_list(self):
31
+ """Test that empty list returns None."""
32
+ result = _select_random_browser([])
33
+ assert result is None
34
+
35
+ def test_select_random_browser_with_single_item_list(self):
36
+ """Test that single-item list returns that item."""
37
+ result = _select_random_browser(["chrome"])
38
+ assert result == "chrome"
39
+
40
+
41
+ @pytest_httpbin.use_class_based_httpbin
42
+ class TestFetcherWithImpersonateList:
43
+ """Test Fetcher with list-based impersonate parameter."""
44
+
45
+ @pytest.fixture(autouse=True)
46
+ def setup_urls(self, httpbin):
47
+ """Fixture to set up URLs for testing."""
48
+ self.basic_url = f"{httpbin.url}/get"
49
+
50
+ def test_get_with_impersonate_list(self):
51
+ """Test that GET request works with impersonate as a list."""
52
+ browsers = ["chrome", "firefox"]
53
+ response = Fetcher.get(self.basic_url, impersonate=browsers)
54
+ assert response.status == 200
55
+
56
+ def test_get_with_single_impersonate(self):
57
+ """Test that GET request still works with single browser string."""
58
+ response = Fetcher.get(self.basic_url, impersonate="chrome")
59
+ assert response.status == 200
60
+
61
+ def test_post_with_impersonate_list(self):
62
+ """Test that POST request works with impersonate as a list."""
63
+ browsers = ["chrome", "firefox"]
64
+ post_url = self.basic_url.replace("/get", "/post")
65
+ response = Fetcher.post(post_url, data={"key": "value"}, impersonate=browsers)
66
+ assert response.status == 200
67
+
68
+ def test_put_with_impersonate_list(self):
69
+ """Test that PUT request works with impersonate as a list."""
70
+ browsers = ["chrome", "safari"]
71
+ put_url = self.basic_url.replace("/get", "/put")
72
+ response = Fetcher.put(put_url, data={"key": "value"}, impersonate=browsers)
73
+ assert response.status == 200
74
+
75
+ def test_delete_with_impersonate_list(self):
76
+ """Test that DELETE request works with impersonate as a list."""
77
+ browsers = ["chrome", "edge"]
78
+ delete_url = self.basic_url.replace("/get", "/delete")
79
+ response = Fetcher.delete(delete_url, impersonate=browsers)
80
+ assert response.status == 200
81
+
82
+
83
+ @pytest_httpbin.use_class_based_httpbin
84
+ class TestFetcherSessionWithImpersonateList:
85
+ """Test FetcherSession with list-based impersonate parameter."""
86
+
87
+ @pytest.fixture(autouse=True)
88
+ def setup_urls(self, httpbin):
89
+ """Fixture to set up URLs for testing."""
90
+ self.basic_url = f"{httpbin.url}/get"
91
+
92
+ def test_session_init_with_impersonate_list(self):
93
+ """Test that FetcherSession can be initialized with impersonate as a list."""
94
+ browsers = ["chrome", "firefox", "safari"]
95
+ session = FetcherSession(impersonate=browsers)
96
+ assert session._default_impersonate == browsers
97
+
98
+ def test_session_request_with_impersonate_list(self):
99
+ """Test that session request works with impersonate as a list."""
100
+ browsers = ["chrome", "firefox"]
101
+ with FetcherSession(impersonate=browsers) as session:
102
+ response = session.get(self.basic_url)
103
+ assert response.status == 200
104
+
105
+ def test_session_multiple_requests_with_impersonate_list(self):
106
+ """Test that multiple requests in a session work with impersonate list."""
107
+ browsers = ["chrome110", "chrome120", "chrome131"]
108
+ with FetcherSession(impersonate=browsers) as session:
109
+ response1 = session.get(self.basic_url)
110
+ response2 = session.get(self.basic_url)
111
+ assert response1.status == 200
112
+ assert response2.status == 200
113
+
114
+ def test_session_request_level_impersonate_override(self):
115
+ """Test that request-level impersonate overrides session-level."""
116
+ session_browsers = ["chrome", "firefox"]
117
+ request_browser = "safari"
118
+
119
+ with FetcherSession(impersonate=session_browsers) as session:
120
+ response = session.get(self.basic_url, impersonate=request_browser)
121
+ assert response.status == 200
122
+
123
+ def test_session_request_level_impersonate_list_override(self):
124
+ """Test that request-level impersonate list overrides session-level."""
125
+ session_browsers = ["chrome", "firefox"]
126
+ request_browsers = ["safari", "edge"]
127
+
128
+ with FetcherSession(impersonate=session_browsers) as session:
129
+ response = session.get(self.basic_url, impersonate=request_browsers)
130
+ assert response.status == 200
131
+
132
+
133
+ class TestImpersonateTypeValidation:
134
+ """Test type validation for impersonate parameter."""
135
+
136
+ def test_impersonate_accepts_string(self):
137
+ """Test that impersonate accepts string type."""
138
+ # This should not raise any type errors
139
+ session = FetcherSession(impersonate="chrome")
140
+ assert session._default_impersonate == "chrome"
141
+
142
+ def test_impersonate_accepts_list(self):
143
+ """Test that impersonate accepts list type."""
144
+ # This should not raise any type errors
145
+ browsers = ["chrome", "firefox"]
146
+ session = FetcherSession(impersonate=browsers)
147
+ assert session._default_impersonate == browsers
148
+
149
+ def test_impersonate_accepts_none(self):
150
+ """Test that impersonate accepts None."""
151
+ # This should not raise any type errors
152
+ session = FetcherSession(impersonate=None)
153
+ assert session._default_impersonate is None