arthur0leywin commited on
Commit
4a3cc0f
ยท
verified ยท
1 Parent(s): 2208644

Upload 3 files

Browse files
Files changed (3) hide show
  1. Dockerfile +16 -0
  2. app.py +296 -0
  3. requirements.txt +6 -0
Dockerfile ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.11-slim
2
+
3
+ RUN apt-get update && apt-get install -y \
4
+ wget curl gnupg ca-certificates \
5
+ libnss3 libxss1 libasound2 libatk-bridge2.0-0 \
6
+ libgtk-3-0 libgbm1 libxshmfence1 \
7
+ --no-install-recommends && rm -rf /var/lib/apt/lists/*
8
+
9
+ WORKDIR /app
10
+ COPY requirements.txt .
11
+ RUN pip install --no-cache-dir -r requirements.txt
12
+ RUN playwright install chromium --with-deps
13
+
14
+ COPY . .
15
+ EXPOSE 7860
16
+ CMD ["python", "app.py"]
app.py ADDED
@@ -0,0 +1,296 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # ============================================================
2
+ # ๐Ÿ”ฑ Emperor Fetch Space โ€” app.py
3
+ # ู†ูุณ ูƒูˆุฏ ุงู„ุณุญุจ ู…ู† main.py ุญุฑููŠุงู‹ + Gradio wrapper ููˆู‚ูŠู‡
4
+ # ============================================================
5
+
6
+ import os, asyncio, base64, json, subprocess, io
7
+ from playwright.async_api import async_playwright
8
+ import cloudscraper, requests
9
+ from bs4 import BeautifulSoup
10
+ from PIL import Image
11
+ import gradio as gr
12
+
13
+ # โ”€โ”€ ุชู†ุฒูŠู„ Chromium ุชู„ู‚ุงุฆูŠุงู‹ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
14
+ chromium_path = os.path.expanduser(
15
+ "~/.cache/ms-playwright/chromium_headless_shell-1223"
16
+ "/chrome-headless-shell-linux64/chrome-headless-shell"
17
+ )
18
+ if not os.path.exists(chromium_path):
19
+ print("โฌ‡๏ธ ุฌุงุฑูŠ ุชู†ุฒูŠู„ Chromium...", flush=True)
20
+ subprocess.run(["playwright", "install", "chromium", "--with-deps"], check=True)
21
+ print("โœ… Chromium ุฌุงู‡ุฒ", flush=True)
22
+
23
+ # โ”€โ”€ ุงู„ุฅุนุฏุงุฏุงุช โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
24
+ API_SECRET = os.getenv("FETCH_SECRET", "emperor_secret_key")
25
+ browser_semaphore = asyncio.Semaphore(1)
26
+
27
+
28
+ # ============================================================
29
+ # โœ… ู†ูุณ fetch_manhwa ู…ู† main.py ุญุฑููŠุงู‹
30
+ # ============================================================
31
+ async def fetch_manhwa(url):
32
+ print(f"โณ ุทู„ุจ ุณุญุจ ุฌุฏูŠุฏ ู„ู€ {url} - ููŠ ุงู†ุชุธุงุฑ ุฏูˆุฑ ุงู„ู…ุชุตูุญ...", flush=True)
33
+
34
+ async with browser_semaphore:
35
+ print(f"๐Ÿš€ ุงู„ู…ุชุตูุญ ู…ุชุงุญ ุงู„ุขู†! ุจุฏุก ุงู„ุณุญุจ ู„ู€ {url}...", flush=True)
36
+
37
+ async with async_playwright() as p:
38
+ browser = await p.chromium.launch(
39
+ headless=True,
40
+ args=[
41
+ "--no-sandbox",
42
+ "--disable-setuid-sandbox",
43
+ "--disable-dev-shm-usage",
44
+ "--disable-gpu",
45
+ "--no-first-run",
46
+ "--no-zygote",
47
+ "--single-process",
48
+ "--js-flags=--max-old-space-size=256",
49
+ ]
50
+ )
51
+
52
+ context = await browser.new_context(
53
+ user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
54
+ viewport={"width": 1280, "height": 1000},
55
+ extra_http_headers={"Cache-Control": "no-cache", "Pragma": "no-cache"}
56
+ )
57
+ page = await context.new_page()
58
+
59
+ captured = []
60
+ captured_urls = set()
61
+ skip_kw = ["logo", "icon", "avatar", "banner", "ads", "pixel", "1x1", ".gif", "loader", "thumb", "widgets"]
62
+
63
+ async def on_response(response):
64
+ try:
65
+ if response.request.resource_type != "image":
66
+ return
67
+ rurl = response.url
68
+ if any(kw in rurl.lower() for kw in skip_kw):
69
+ return
70
+ if rurl in captured_urls:
71
+ return
72
+ body = await response.body()
73
+ if len(body) > 4000:
74
+ captured.append(body)
75
+ captured_urls.add(rurl)
76
+ print(f" ๐Ÿ“ธ [{len(captured)}] {rurl[-50:]}")
77
+ del body
78
+ except:
79
+ pass
80
+
81
+ page.on("response", on_response)
82
+
83
+ try:
84
+ print(f"๐Ÿš€ ุฌุงุฑูŠ ูุชุญ ุงู„ุตูุญุฉ: {url}")
85
+ await page.goto(url, wait_until="domcontentloaded", timeout=60000)
86
+ await page.wait_for_timeout(3000)
87
+
88
+ print("๐Ÿ”„ ุฌุงุฑูŠ ุชุชุจุน ุงู„ุตูˆุฑ ูˆุงุณุชู‡ุฏุงูู‡ุง ุนู†ุตุฑุงู‹ ุนู†ุตุฑุงู‹...")
89
+
90
+ current_idx = 0
91
+ no_new_imgs_streak = 0
92
+ max_images_to_check = 200
93
+
94
+ while current_idx < max_images_to_check:
95
+ imgs = page.locator("img")
96
+ count = await imgs.count()
97
+
98
+ if current_idx >= count:
99
+ await page.evaluate("window.scrollBy(0, 1200)")
100
+ await page.wait_for_timeout(600)
101
+
102
+ new_count = await imgs.count()
103
+ if new_count == count:
104
+ no_new_imgs_streak += 1
105
+ if no_new_imgs_streak > 10:
106
+ break
107
+ else:
108
+ no_new_imgs_streak = 0
109
+ continue
110
+
111
+ try:
112
+ img = imgs.nth(current_idx)
113
+ current_idx += 1
114
+
115
+ src = await img.get_attribute("src") or ""
116
+ data_src = await img.get_attribute("data-src") or ""
117
+ combined = (src + data_src).lower()
118
+ if any(kw in combined for kw in skip_kw):
119
+ continue
120
+
121
+ await img.scroll_into_view_if_needed(timeout=1500)
122
+ await page.wait_for_timeout(150)
123
+
124
+ except Exception:
125
+ continue
126
+
127
+ print(f"๐Ÿ“Š ุฅุฌู…ุงู„ูŠ ุงู„ุตูˆุฑ ุงู„ู…ู…ุณูˆูƒุฉ ุญูŠุงู‹ ู…ู† ุงู„ุดุจูƒุฉ: {len(captured)}")
128
+
129
+ # โ”€โ”€ Fallback ุงู„ุดุงู…ู„ โ”€โ”€
130
+ if len(captured) < 5:
131
+ print("โš ๏ธ ุงู„ุชู‚ุงุท ุงู„ุดุจูƒุฉ ุถุนูŠูุŒ ุฌุงุฑูŠ ู‚ุฑุงุกุฉ ุงู„ู€ DOM...")
132
+ dom_urls = await page.evaluate("""
133
+ () => Array.from(document.querySelectorAll('img'))
134
+ .map(img => img.dataset.src || img.dataset.lazy || img.dataset.original || img.dataset.url || img.getAttribute('data-cfsrc') || img.src)
135
+ .filter(src => src && src.startsWith('http') && !src.includes('logo') && !src.includes('avatar'))
136
+ """)
137
+
138
+ unique_urls = [u for u in set(dom_urls) if u not in captured_urls]
139
+ print(f"๐Ÿ“„ ุชู… ุงู„ุนุซูˆุฑ ุนู„ู‰ {len(unique_urls)} ุฑุงุจุท ุจุงู„ู€ DOM.")
140
+
141
+ async def fetch_single_img(img_url):
142
+ try:
143
+ res = await context.request.get(img_url, headers={"Referer": url}, timeout=12000)
144
+ if res.ok:
145
+ b = await res.body()
146
+ if len(b) > 4000:
147
+ captured.append(b)
148
+ captured_urls.add(img_url)
149
+ except:
150
+ pass
151
+
152
+ if unique_urls:
153
+ await asyncio.gather(*[fetch_single_img(u) for u in unique_urls])
154
+
155
+ await browser.close()
156
+ print(f"โœ… ุงู†ุชู‡ู‰ ุงู„ุณุญุจ ุจู†ุฌุงุญ! ุฅุฌู…ุงู„ูŠ ุงู„ุตูˆุฑ: {len(captured)}")
157
+ return captured
158
+
159
+ except Exception as e:
160
+ await browser.close()
161
+ print(f"โŒ ุฎุทุฃ ุฃุซู†ุงุก ุงู„ุณุญุจ: {e}")
162
+ return []
163
+
164
+
165
+ # ============================================================
166
+ # โœ… ู†ูุณ fetch_fast ู…ู† main.py ุญุฑููŠุงู‹
167
+ # ============================================================
168
+ async def fetch_fast(url):
169
+ try:
170
+ print(f"๐Ÿš€ fetch_fast ุจุฏุฃ: {url}", flush=True)
171
+ loop = asyncio.get_event_loop()
172
+ headers = {
173
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
174
+ "Referer": "https://www.google.com/",
175
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
176
+ "Accept-Language": "en-US,en;q=0.9",
177
+ "Accept-Encoding": "gzip, deflate, br",
178
+ "Connection": "keep-alive",
179
+ "Upgrade-Insecure-Requests": "1",
180
+ "Sec-Fetch-Dest": "document",
181
+ "Sec-Fetch-Mode": "navigate",
182
+ "Sec-Fetch-Site": "none",
183
+ "Cache-Control": "max-age=0",
184
+ }
185
+
186
+ scraper = cloudscraper.create_scraper()
187
+ res = await loop.run_in_executor(
188
+ None,
189
+ lambda: scraper.get(url, timeout=15)
190
+ )
191
+
192
+ print(f"๐Ÿ” Status: {res.status_code}", flush=True)
193
+
194
+ soup = BeautifulSoup(res.text, "html.parser")
195
+
196
+ img_urls = []
197
+ for img in soup.find_all("img"):
198
+ src = img.get("data-src") or img.get("src") or ""
199
+ if src.startswith("http") and all(x not in src.lower() for x in ["logo", "wp-content/themes", "dflazy", "avatar"]):
200
+ img_urls.append(src)
201
+
202
+ print(f"๐Ÿ” ู„ุงู‚ู‰ {len(img_urls)} ุฑุงุจุท", flush=True)
203
+
204
+ if len(img_urls) < 3:
205
+ return None
206
+
207
+ async def download_img(img_url):
208
+ try:
209
+ r = await loop.run_in_executor(
210
+ None,
211
+ lambda: requests.get(img_url, headers={"Referer": url, "User-Agent": headers["User-Agent"]}, timeout=15)
212
+ )
213
+ if r.ok and len(r.content) > 5000:
214
+ return r.content
215
+ except:
216
+ pass
217
+ return None
218
+
219
+ tasks = [download_img(u) for u in img_urls]
220
+ results = await asyncio.gather(*tasks)
221
+ images_bytes = [r for r in results if r]
222
+
223
+ print(f"โœ… fetch_fast ุญู…ู‘ู„ {len(images_bytes)} ุตูˆุฑุฉ", flush=True)
224
+ return images_bytes if len(images_bytes) >= 3 else None
225
+
226
+ except Exception as e:
227
+ print(f"โš ๏ธ fetch_fast ูุดู„: {type(e).__name__}: {e}", flush=True)
228
+ return None
229
+
230
+
231
+ # ============================================================
232
+ # ๐ŸŽฎ Gradio Wrapper โ€” ุงู„ู€ endpoint ุงู„ู„ูŠ ุงู„ุจูˆุช ุจูŠู†ุงุฏูŠู‡
233
+ # ============================================================
234
+ def fetch_chapter(url: str, secret: str):
235
+ """
236
+ Input: url, secret
237
+ Output: images_json (base64 list as JSON string), count, status
238
+ """
239
+ if secret != API_SECRET:
240
+ return "[]", 0, "error:unauthorized"
241
+
242
+ url = url.strip()
243
+ if not url.startswith("http"):
244
+ return "[]", 0, "error:invalid_url"
245
+
246
+ print(f"๐Ÿ“ฅ ุทู„ุจ ุณุญุจ: {url}", flush=True)
247
+
248
+ loop = asyncio.new_event_loop()
249
+ asyncio.set_event_loop(loop)
250
+
251
+ try:
252
+ # ู†ูุณ ุงู„ู…ู†ุทู‚ ุงู„ุฃุตู„ูŠ: fast ุฃูˆู„ุŒ ู„ูˆ ูุดู„ deep
253
+ images = loop.run_until_complete(fetch_fast(url))
254
+ method = "fast"
255
+
256
+ if not images:
257
+ print("๐Ÿ”„ fetch_fast ูุดู„ โ†’ fetch_manhwa", flush=True)
258
+ images = loop.run_until_complete(fetch_manhwa(url))
259
+ method = "deep"
260
+
261
+ if not images:
262
+ return "[]", 0, "error:fetch_failed"
263
+
264
+ encoded = [base64.b64encode(img).decode() for img in images]
265
+ print(f"โœ… [{method}] {len(encoded)} ุตูˆุฑุฉ ุฌุงู‡ุฒุฉ", flush=True)
266
+ return json.dumps(encoded), len(encoded), f"ok:{method}"
267
+
268
+ except Exception as e:
269
+ print(f"โŒ Exception: {e}", flush=True)
270
+ return "[]", 0, f"error:{str(e)[:100]}"
271
+ finally:
272
+ loop.close()
273
+
274
+
275
+ # โ”€โ”€ Gradio Interface โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€
276
+ with gr.Blocks(title="Emperor Fetch API") as demo:
277
+ gr.Markdown("## ๐Ÿ”ฑ Emperor Fetch Space\nInternal API โ€” not for public use.")
278
+
279
+ with gr.Row():
280
+ url_in = gr.Textbox(label="URL", placeholder="https://...")
281
+ secret_in = gr.Textbox(label="Secret", type="password")
282
+
283
+ btn = gr.Button("Fetch", variant="primary")
284
+
285
+ images_out = gr.Textbox(label="images_json")
286
+ count_out = gr.Number(label="count")
287
+ status_out = gr.Textbox(label="status")
288
+
289
+ btn.click(
290
+ fn=fetch_chapter,
291
+ inputs=[url_in, secret_in],
292
+ outputs=[images_out, count_out, status_out],
293
+ api_name="fetch_chapter" # โ† ุงู„ุงุณู… ุงู„ู„ูŠ ุจูŠู†ุงุฏูŠู‡ ุงู„ุจูˆุช
294
+ )
295
+
296
+ demo.launch(server_name="0.0.0.0", server_port=7860)
requirements.txt ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ gradio>=4.0.0
2
+ playwright
3
+ requests
4
+ cloudscraper
5
+ beautifulsoup4
6
+ Pillow