Trae Bot commited on
Commit ·
f26a077
1
Parent(s): 2e1ee73
Fix relative import errors in engines package for HuggingFace deployment
Browse files- engines/base.py +1 -1
- engines/mediacrawler.py +4 -4
- engines/spider_xhs.py +3 -3
engines/base.py
CHANGED
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
| 3 |
from dataclasses import dataclass
|
| 4 |
from typing import Any, Dict, Protocol
|
| 5 |
|
| 6 |
-
from
|
| 7 |
|
| 8 |
|
| 9 |
@dataclass(frozen=True)
|
|
|
|
| 3 |
from dataclasses import dataclass
|
| 4 |
from typing import Any, Dict, Protocol
|
| 5 |
|
| 6 |
+
from service.tasks import TaskRecord
|
| 7 |
|
| 8 |
|
| 9 |
@dataclass(frozen=True)
|
engines/mediacrawler.py
CHANGED
|
@@ -22,7 +22,7 @@ except Exception as e:
|
|
| 22 |
sync_playwright = None
|
| 23 |
_PLAYWRIGHT_IMPORT_ERROR = str(e)
|
| 24 |
|
| 25 |
-
from
|
| 26 |
from .base import EngineRunOutput
|
| 27 |
|
| 28 |
try:
|
|
@@ -819,7 +819,7 @@ class MediaCrawlerEngine:
|
|
| 819 |
return unique
|
| 820 |
|
| 821 |
def _pick_proxy(self, attempt: int) -> str | None:
|
| 822 |
-
from
|
| 823 |
dynamic_proxy = proxy_pool.get_random_proxy()
|
| 824 |
if dynamic_proxy:
|
| 825 |
return dynamic_proxy
|
|
@@ -846,7 +846,7 @@ class MediaCrawlerEngine:
|
|
| 846 |
self._humanize(page)
|
| 847 |
res = job(page)
|
| 848 |
if proxy:
|
| 849 |
-
from
|
| 850 |
|
| 851 |
proxy_pool.report_success(proxy)
|
| 852 |
return res
|
|
@@ -860,7 +860,7 @@ class MediaCrawlerEngine:
|
|
| 860 |
if proxy:
|
| 861 |
if reason in ("timeout", "proxy_failed", "rate"):
|
| 862 |
try:
|
| 863 |
-
from
|
| 864 |
|
| 865 |
proxy_pool.report_failure(proxy, reason)
|
| 866 |
except Exception:
|
|
|
|
| 22 |
sync_playwright = None
|
| 23 |
_PLAYWRIGHT_IMPORT_ERROR = str(e)
|
| 24 |
|
| 25 |
+
from service.tasks import TaskRecord
|
| 26 |
from .base import EngineRunOutput
|
| 27 |
|
| 28 |
try:
|
|
|
|
| 819 |
return unique
|
| 820 |
|
| 821 |
def _pick_proxy(self, attempt: int) -> str | None:
|
| 822 |
+
from service.proxy_pool import proxy_pool
|
| 823 |
dynamic_proxy = proxy_pool.get_random_proxy()
|
| 824 |
if dynamic_proxy:
|
| 825 |
return dynamic_proxy
|
|
|
|
| 846 |
self._humanize(page)
|
| 847 |
res = job(page)
|
| 848 |
if proxy:
|
| 849 |
+
from service.proxy_pool import proxy_pool
|
| 850 |
|
| 851 |
proxy_pool.report_success(proxy)
|
| 852 |
return res
|
|
|
|
| 860 |
if proxy:
|
| 861 |
if reason in ("timeout", "proxy_failed", "rate"):
|
| 862 |
try:
|
| 863 |
+
from service.proxy_pool import proxy_pool
|
| 864 |
|
| 865 |
proxy_pool.report_failure(proxy, reason)
|
| 866 |
except Exception:
|
engines/spider_xhs.py
CHANGED
|
@@ -10,7 +10,7 @@ from apis.xhs_pc_apis import XHS_Apis
|
|
| 10 |
from xhs_utils.data_util import handle_note_info, handle_user_info
|
| 11 |
from xhs_utils.response_guard import get_dict, get_list
|
| 12 |
|
| 13 |
-
from
|
| 14 |
from .base import EngineRunOutput
|
| 15 |
|
| 16 |
|
|
@@ -117,7 +117,7 @@ class SpiderXHSEngine:
|
|
| 117 |
explicit_proxy = payload.get("proxy") or payload.get("proxies")
|
| 118 |
proxy_value = explicit_proxy or self.proxy
|
| 119 |
if not proxy_value:
|
| 120 |
-
from
|
| 121 |
|
| 122 |
proxy_value = proxy_pool.get_random_proxy()
|
| 123 |
if proxy_value:
|
|
@@ -156,7 +156,7 @@ class SpiderXHSEngine:
|
|
| 156 |
|
| 157 |
try:
|
| 158 |
if proxy_used:
|
| 159 |
-
from
|
| 160 |
|
| 161 |
if ok:
|
| 162 |
proxy_pool.report_success(proxy_used)
|
|
|
|
| 10 |
from xhs_utils.data_util import handle_note_info, handle_user_info
|
| 11 |
from xhs_utils.response_guard import get_dict, get_list
|
| 12 |
|
| 13 |
+
from service.tasks import TaskRecord
|
| 14 |
from .base import EngineRunOutput
|
| 15 |
|
| 16 |
|
|
|
|
| 117 |
explicit_proxy = payload.get("proxy") or payload.get("proxies")
|
| 118 |
proxy_value = explicit_proxy or self.proxy
|
| 119 |
if not proxy_value:
|
| 120 |
+
from service.proxy_pool import proxy_pool
|
| 121 |
|
| 122 |
proxy_value = proxy_pool.get_random_proxy()
|
| 123 |
if proxy_value:
|
|
|
|
| 156 |
|
| 157 |
try:
|
| 158 |
if proxy_used:
|
| 159 |
+
from service.proxy_pool import proxy_pool
|
| 160 |
|
| 161 |
if ok:
|
| 162 |
proxy_pool.report_success(proxy_used)
|