q6 commited on
Commit
bd75875
·
1 Parent(s): 37f5e8d

server client

Browse files
.gitignore CHANGED
@@ -1,6 +1,7 @@
1
  .env
2
- *.txt
3
  db
4
  images
5
  merge_dev.bat
6
- __pycache__
 
 
 
1
  .env
 
2
  db
3
  images
4
  merge_dev.bat
5
+ __pycache__
6
+ *.png
7
+ *.txt
API/app.py CHANGED
@@ -2,7 +2,6 @@ from fastapi import FastAPI, Query
2
  import aiohttp
3
  import asyncio
4
  import time
5
- import os
6
  from pydantic import BaseModel
7
  from typing import List, Dict
8
 
@@ -11,16 +10,11 @@ img_base = 'https://i.pximg.net/img-original/img/'
11
 
12
  class pixifModel(BaseModel):
13
  post_ids: List[int]
 
14
 
15
  class PixifDownloadModel(BaseModel):
16
  posts: Dict[str, str]
17
 
18
- env_path = os.path.dirname(os.path.realpath(__file__)) + "/../.env"
19
-
20
- PHPSESSID = os.getenv("PHPSESSID", "122422917_9wy3h18GPbbwJRCYrPDyAYI9eNu0lYA1")
21
-
22
- cookies = {"PHPSESSID": PHPSESSID}
23
-
24
  headers = {
25
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
26
  'referer': 'https://www.pixiv.net/',
@@ -33,6 +27,9 @@ async def fetch_page(session, url):
33
  data = await response.json()
34
  return data
35
 
 
 
 
36
  def base26(n):
37
  if n == 0:
38
  return "A"
@@ -135,8 +132,10 @@ async def process_post(post_id, session, semaphore):
135
  @app.get("/allimages")
136
  async def all_images(
137
  only_first = Query("0", description="Only fetch the first image of each post."),
138
- post_ids: List[int] = Query(..., alias='post_ids')
 
139
  ):
 
140
  async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
141
  tasks = []
142
  for pid in post_ids:
@@ -157,7 +156,11 @@ async def all_images(
157
  return {"image_urls": all_image_urls}
158
 
159
  @app.get("/allimage")
160
- async def all_image(post_id: int = Query(..., description="The post ID to fetch all images from.")):
 
 
 
 
161
  async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
162
  try:
163
  data = await fetch_page(session, f"https://www.pixiv.net/ajax/illust/{post_id}/pages")
@@ -172,6 +175,7 @@ async def pixif(
172
  ):
173
  post_ids = items.post_ids
174
  semaphore = asyncio.Semaphore(8)
 
175
 
176
  async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
177
  tasks = [process_post(post_id, session, semaphore) for post_id in post_ids]
@@ -187,4 +191,4 @@ async def read_root():
187
 
188
  if __name__ == "__main__":
189
  import uvicorn
190
- uvicorn.run(app, host="127.0.0.1", port=7860)
 
2
  import aiohttp
3
  import asyncio
4
  import time
 
5
  from pydantic import BaseModel
6
  from typing import List, Dict
7
 
 
10
 
11
  class pixifModel(BaseModel):
12
  post_ids: List[int]
13
+ phpsessid: str
14
 
15
  class PixifDownloadModel(BaseModel):
16
  posts: Dict[str, str]
17
 
 
 
 
 
 
 
18
  headers = {
19
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
20
  'referer': 'https://www.pixiv.net/',
 
27
  data = await response.json()
28
  return data
29
 
30
+ def build_cookies(phpsessid: str) -> Dict[str, str]:
31
+ return {"PHPSESSID": phpsessid}
32
+
33
  def base26(n):
34
  if n == 0:
35
  return "A"
 
132
  @app.get("/allimages")
133
  async def all_images(
134
  only_first = Query("0", description="Only fetch the first image of each post."),
135
+ post_ids: List[int] = Query(..., alias='post_ids'),
136
+ phpsessid: str = Query(..., description="Pixiv PHPSESSID value.")
137
  ):
138
+ cookies = build_cookies(phpsessid)
139
  async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
140
  tasks = []
141
  for pid in post_ids:
 
156
  return {"image_urls": all_image_urls}
157
 
158
  @app.get("/allimage")
159
+ async def all_image(
160
+ post_id: int = Query(..., description="The post ID to fetch all images from."),
161
+ phpsessid: str = Query(..., description="Pixiv PHPSESSID value.")
162
+ ):
163
+ cookies = build_cookies(phpsessid)
164
  async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
165
  try:
166
  data = await fetch_page(session, f"https://www.pixiv.net/ajax/illust/{post_id}/pages")
 
175
  ):
176
  post_ids = items.post_ids
177
  semaphore = asyncio.Semaphore(8)
178
+ cookies = build_cookies(items.phpsessid)
179
 
180
  async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
181
  tasks = [process_post(post_id, session, semaphore) for post_id in post_ids]
 
191
 
192
  if __name__ == "__main__":
193
  import uvicorn
194
+ uvicorn.run(app, host="127.0.0.1", port=7860)
Client/hunt.py CHANGED
@@ -8,6 +8,32 @@ local = 0
8
  endpoint = "http://127.0.0.1:7860" if local else "https://q6-p.hf.space"
9
  img_base = 'https://i.pximg.net/img-original/img/'
10
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  os.chdir(os.path.dirname(os.path.abspath(__file__)))
12
  os.makedirs("images/Stash", exist_ok=True)
13
 
@@ -55,7 +81,7 @@ for index in indexs:
55
  print(f"Group: {group_name}\nFiltered: {len(filtered)}/{len(post_ids)}")
56
 
57
  if filtered:
58
- data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered})
59
  print(data)
60
  data = data.json()
61
  for post_id, url in data.items():
@@ -85,4 +111,4 @@ for index in indexs:
85
  if not os.listdir(f'images/{group_name}'):
86
  os.rmdir(f"images/{group_name}")
87
 
88
- db.close()
 
8
  endpoint = "http://127.0.0.1:7860" if local else "https://q6-p.hf.space"
9
  img_base = 'https://i.pximg.net/img-original/img/'
10
 
11
+ def read_dotenv_value(path, key):
12
+ try:
13
+ with open(path, "r") as env_file:
14
+ for line in env_file:
15
+ line = line.strip()
16
+ if not line or line.startswith("#") or "=" not in line:
17
+ continue
18
+ k, v = line.split("=", 1)
19
+ if k == key:
20
+ return v
21
+ except FileNotFoundError:
22
+ return None
23
+ return None
24
+
25
+ def get_phpsessid():
26
+ phpsessid = os.getenv("PHPSESSID")
27
+ if phpsessid:
28
+ return phpsessid
29
+ env_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".env"))
30
+ phpsessid = read_dotenv_value(env_path, "PHPSESSID")
31
+ if phpsessid:
32
+ return phpsessid
33
+ raise RuntimeError("PHPSESSID is not set in the environment or .env")
34
+
35
+ phpsessid = get_phpsessid()
36
+
37
  os.chdir(os.path.dirname(os.path.abspath(__file__)))
38
  os.makedirs("images/Stash", exist_ok=True)
39
 
 
81
  print(f"Group: {group_name}\nFiltered: {len(filtered)}/{len(post_ids)}")
82
 
83
  if filtered:
84
+ data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered, "phpsessid": phpsessid})
85
  print(data)
86
  data = data.json()
87
  for post_id, url in data.items():
 
111
  if not os.listdir(f'images/{group_name}'):
112
  os.rmdir(f"images/{group_name}")
113
 
114
+ db.close()
Client/run_pixif_single.py CHANGED
@@ -12,11 +12,36 @@ if PROJECT_ROOT not in sys.path:
12
  from API import app as pixif_app # noqa: E402
13
 
14
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15
  async def run_local() -> dict:
16
  post_id = 138954885
17
  semaphore = asyncio.Semaphore(8)
 
18
 
19
- async with aiohttp.ClientSession(cookies=pixif_app.cookies, headers=pixif_app.headers) as session:
20
  _, image_url = await pixif_app.process_post(post_id, session, semaphore)
21
 
22
  if not image_url:
 
12
  from API import app as pixif_app # noqa: E402
13
 
14
 
15
+ def read_dotenv_value(path, key):
16
+ try:
17
+ with open(path, "r") as env_file:
18
+ for line in env_file:
19
+ line = line.strip()
20
+ if not line or line.startswith("#") or "=" not in line:
21
+ continue
22
+ k, v = line.split("=", 1)
23
+ if k == key:
24
+ return v
25
+ except FileNotFoundError:
26
+ return None
27
+ return None
28
+
29
+ def get_phpsessid():
30
+ phpsessid = os.getenv("PHPSESSID")
31
+ if phpsessid:
32
+ return phpsessid
33
+ env_path = os.path.join(PROJECT_ROOT, ".env")
34
+ phpsessid = read_dotenv_value(env_path, "PHPSESSID")
35
+ if phpsessid:
36
+ return phpsessid
37
+ raise RuntimeError("PHPSESSID is not set in the environment or .env")
38
+
39
  async def run_local() -> dict:
40
  post_id = 138954885
41
  semaphore = asyncio.Semaphore(8)
42
+ cookies = pixif_app.build_cookies(get_phpsessid())
43
 
44
+ async with aiohttp.ClientSession(cookies=cookies, headers=pixif_app.headers) as session:
45
  _, image_url = await pixif_app.process_post(post_id, session, semaphore)
46
 
47
  if not image_url:
Client/txt logs/.gitkeep ADDED
File without changes