Karim shoair commited on
Commit
9ac26c8
·
1 Parent(s): 0cae90d

refactor: cleaner code for the fetcher's lazy loader

Browse files
Files changed (1) hide show
  1. scrapling/__init__.py +16 -37
scrapling/__init__.py CHANGED
@@ -3,45 +3,24 @@ __version__ = "0.3-beta"
3
  __copyright__ = "Copyright (c) 2024 Karim Shoair"
4
 
5
 
6
- # A lightweight approach to create lazy loader for each import for backward compatibility
7
  # This will reduces initial memory footprint significantly (only loads what's used)
8
  def __getattr__(name):
9
- if name == "Fetcher":
10
- from scrapling.fetchers import Fetcher as cls
11
-
12
- return cls
13
- elif name == "Selector":
14
- from scrapling.parser import Selector as cls
15
-
16
- return cls
17
- elif name == "Selectors":
18
- from scrapling.parser import Selectors as cls
19
-
20
- return cls
21
- elif name == "AttributesHandler":
22
- from scrapling.core.custom_types import AttributesHandler as cls
23
-
24
- return cls
25
- elif name == "TextHandler":
26
- from scrapling.core.custom_types import TextHandler as cls
27
-
28
- return cls
29
- elif name == "AsyncFetcher":
30
- from scrapling.fetchers import AsyncFetcher as cls
31
-
32
- return cls
33
- elif name == "StealthyFetcher":
34
- from scrapling.fetchers import StealthyFetcher as cls
35
-
36
- return cls
37
- elif name == "DynamicFetcher":
38
- from scrapling.fetchers import DynamicFetcher as cls
39
-
40
- return cls
41
- elif name == "CustomFetcher":
42
- from scrapling.fetchers import CustomFetcher as cls
43
-
44
- return cls
45
  else:
46
  raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
47
 
 
3
  __copyright__ = "Copyright (c) 2024 Karim Shoair"
4
 
5
 
6
+ # A lightweight approach to create a lazy loader for each import for backward compatibility
7
  # This will reduces initial memory footprint significantly (only loads what's used)
8
  def __getattr__(name):
9
+ lazy_imports = {
10
+ "Fetcher": ("scrapling.fetchers", "Fetcher"),
11
+ "Selector": ("scrapling.parser", "Selector"),
12
+ "Selectors": ("scrapling.parser", "Selectors"),
13
+ "AttributesHandler": ("scrapling.core.custom_types", "AttributesHandler"),
14
+ "TextHandler": ("scrapling.core.custom_types", "TextHandler"),
15
+ "AsyncFetcher": ("scrapling.fetchers", "AsyncFetcher"),
16
+ "StealthyFetcher": ("scrapling.fetchers", "StealthyFetcher"),
17
+ "DynamicFetcher": ("scrapling.fetchers", "DynamicFetcher"),
18
+ }
19
+
20
+ if name in lazy_imports:
21
+ module_path, class_name = lazy_imports[name]
22
+ module = __import__(module_path, fromlist=[class_name])
23
+ return getattr(module, class_name)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
24
  else:
25
  raise AttributeError(f"module 'scrapling' has no attribute '{name}'")
26