Karim shoair commited on
Commit
f62e5ea
·
1 Parent(s): 1d1bcaf

fix: remove old proxy logic

Browse files

When I was caching it with the rest of the data

scrapling/engines/_browsers/_validators.py CHANGED
@@ -96,7 +96,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
96
  if self.page_action and not callable(self.page_action):
97
  raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
98
  if self.proxy:
99
- self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
100
  if self.cdp_url:
101
  cdp_msg = _is_invalid_cdp_url(self.cdp_url)
102
  if cdp_msg:
 
96
  if self.page_action and not callable(self.page_action):
97
  raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
98
  if self.proxy:
99
+ self.proxy = construct_proxy_dict(self.proxy)
100
  if self.cdp_url:
101
  cdp_msg = _is_invalid_cdp_url(self.cdp_url)
102
  if cdp_msg:
scrapling/engines/toolbelt/navigation.py CHANGED
@@ -49,20 +49,11 @@ async def async_intercept_route(route: async_Route):
49
  await route.continue_()
50
 
51
 
52
- @overload
53
- def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: Literal[True]) -> Tuple: ...
54
-
55
-
56
- @overload
57
- def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: Literal[False] = False) -> Dict: ...
58
-
59
-
60
- def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: bool = False) -> Dict | Tuple:
61
  """Validate a proxy and return it in the acceptable format for Playwright
62
  Reference: https://playwright.dev/python/docs/network#http-proxy
63
 
64
  :param proxy_string: A string or a dictionary representation of the proxy.
65
- :param as_tuple: Return the proxy dictionary as a tuple to be cachable
66
  :return:
67
  """
68
  if isinstance(proxy_string, str):
@@ -78,7 +69,7 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: b
78
  }
79
  if proxy.port:
80
  result["server"] += f":{proxy.port}"
81
- return tuple(result.items()) if as_tuple else result
82
  except ValueError:
83
  # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
84
  raise ValueError("The proxy argument's string is in invalid format!")
@@ -87,7 +78,7 @@ def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple, as_tuple: b
87
  try:
88
  validated = convert(proxy_string, ProxyDict)
89
  result_dict = structs.asdict(validated)
90
- return tuple(result_dict.items()) if as_tuple else result_dict
91
  except ValidationError as e:
92
  raise TypeError(f"Invalid proxy dictionary: {e}")
93
 
 
49
  await route.continue_()
50
 
51
 
52
+ def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple) -> Dict:
 
 
 
 
 
 
 
 
53
  """Validate a proxy and return it in the acceptable format for Playwright
54
  Reference: https://playwright.dev/python/docs/network#http-proxy
55
 
56
  :param proxy_string: A string or a dictionary representation of the proxy.
 
57
  :return:
58
  """
59
  if isinstance(proxy_string, str):
 
69
  }
70
  if proxy.port:
71
  result["server"] += f":{proxy.port}"
72
+ return result
73
  except ValueError:
74
  # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
75
  raise ValueError("The proxy argument's string is in invalid format!")
 
78
  try:
79
  validated = convert(proxy_string, ProxyDict)
80
  result_dict = structs.asdict(validated)
81
+ return result_dict
82
  except ValidationError as e:
83
  raise TypeError(f"Invalid proxy dictionary: {e}")
84