Karim shoair commited on
Commit ·
f62e5ea
1
Parent(s): 1d1bcaf
fix: remove old proxy logic
Browse filesWhen I was caching it with the rest of the data
scrapling/engines/_browsers/_validators.py
CHANGED
|
@@ -96,7 +96,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
|
|
| 96 |
if self.page_action and not callable(self.page_action):
|
| 97 |
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
|
| 98 |
if self.proxy:
|
| 99 |
-
self.proxy = construct_proxy_dict(self.proxy
|
| 100 |
if self.cdp_url:
|
| 101 |
cdp_msg = _is_invalid_cdp_url(self.cdp_url)
|
| 102 |
if cdp_msg:
|
|
|
|
| 96 |
if self.page_action and not callable(self.page_action):
|
| 97 |
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
|
| 98 |
if self.proxy:
|
| 99 |
+
self.proxy = construct_proxy_dict(self.proxy)
|
| 100 |
if self.cdp_url:
|
| 101 |
cdp_msg = _is_invalid_cdp_url(self.cdp_url)
|
| 102 |
if cdp_msg:
|
scrapling/engines/toolbelt/navigation.py
CHANGED
|
@@ -49,20 +49,11 @@ async def async_intercept_route(route: async_Route):
|
|
| 49 |
await route.continue_()
|
| 50 |
|
| 51 |
|
| 52 |
-
|
| 53 |
-
def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: Literal[True]) -> Tuple: ...
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
@overload
|
| 57 |
-
def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: Literal[False] = False) -> Dict: ...
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: bool = False) -> Dict | Tuple:
|
| 61 |
"""Validate a proxy and return it in the acceptable format for Playwright
|
| 62 |
Reference: https://playwright.dev/python/docs/network#http-proxy
|
| 63 |
|
| 64 |
:param proxy_string: A string or a dictionary representation of the proxy.
|
| 65 |
-
:param as_tuple: Return the proxy dictionary as a tuple to be cachable
|
| 66 |
:return:
|
| 67 |
"""
|
| 68 |
if isinstance(proxy_string, str):
|
|
@@ -78,7 +69,7 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: b
|
|
| 78 |
}
|
| 79 |
if proxy.port:
|
| 80 |
result["server"] += f":{proxy.port}"
|
| 81 |
-
return
|
| 82 |
except ValueError:
|
| 83 |
# Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
|
| 84 |
raise ValueError("The proxy argument's string is in invalid format!")
|
|
@@ -87,7 +78,7 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: b
|
|
| 87 |
try:
|
| 88 |
validated = convert(proxy_string, ProxyDict)
|
| 89 |
result_dict = structs.asdict(validated)
|
| 90 |
-
return
|
| 91 |
except ValidationError as e:
|
| 92 |
raise TypeError(f"Invalid proxy dictionary: {e}")
|
| 93 |
|
|
|
|
| 49 |
await route.continue_()
|
| 50 |
|
| 51 |
|
| 52 |
+
def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple) -> Dict:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
"""Validate a proxy and return it in the acceptable format for Playwright
|
| 54 |
Reference: https://playwright.dev/python/docs/network#http-proxy
|
| 55 |
|
| 56 |
:param proxy_string: A string or a dictionary representation of the proxy.
|
|
|
|
| 57 |
:return:
|
| 58 |
"""
|
| 59 |
if isinstance(proxy_string, str):
|
|
|
|
| 69 |
}
|
| 70 |
if proxy.port:
|
| 71 |
result["server"] += f":{proxy.port}"
|
| 72 |
+
return result
|
| 73 |
except ValueError:
|
| 74 |
# Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
|
| 75 |
raise ValueError("The proxy argument's string is in invalid format!")
|
|
|
|
| 78 |
try:
|
| 79 |
validated = convert(proxy_string, ProxyDict)
|
| 80 |
result_dict = structs.asdict(validated)
|
| 81 |
+
return result_dict
|
| 82 |
except ValidationError as e:
|
| 83 |
raise TypeError(f"Invalid proxy dictionary: {e}")
|
| 84 |
|