server client
Browse files- .gitignore +3 -2
- API/app.py +14 -10
- Client/hunt.py +28 -2
- Client/run_pixif_single.py +26 -1
- Client/txt logs/.gitkeep +0 -0
.gitignore
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
.env
|
| 2 |
-
*.txt
|
| 3 |
db
|
| 4 |
images
|
| 5 |
merge_dev.bat
|
| 6 |
-
__pycache__
|
|
|
|
|
|
|
|
|
| 1 |
.env
|
|
|
|
| 2 |
db
|
| 3 |
images
|
| 4 |
merge_dev.bat
|
| 5 |
+
__pycache__
|
| 6 |
+
*.png
|
| 7 |
+
*.txt
|
API/app.py
CHANGED
|
@@ -2,7 +2,6 @@ from fastapi import FastAPI, Query
|
|
| 2 |
import aiohttp
|
| 3 |
import asyncio
|
| 4 |
import time
|
| 5 |
-
import os
|
| 6 |
from pydantic import BaseModel
|
| 7 |
from typing import List, Dict
|
| 8 |
|
|
@@ -11,16 +10,11 @@ img_base = 'https://i.pximg.net/img-original/img/'
|
|
| 11 |
|
| 12 |
class pixifModel(BaseModel):
|
| 13 |
post_ids: List[int]
|
|
|
|
| 14 |
|
| 15 |
class PixifDownloadModel(BaseModel):
|
| 16 |
posts: Dict[str, str]
|
| 17 |
|
| 18 |
-
env_path = os.path.dirname(os.path.realpath(__file__)) + "/../.env"
|
| 19 |
-
|
| 20 |
-
PHPSESSID = os.getenv("PHPSESSID", "122422917_9wy3h18GPbbwJRCYrPDyAYI9eNu0lYA1")
|
| 21 |
-
|
| 22 |
-
cookies = {"PHPSESSID": PHPSESSID}
|
| 23 |
-
|
| 24 |
headers = {
|
| 25 |
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
|
| 26 |
'referer': 'https://www.pixiv.net/',
|
|
@@ -33,6 +27,9 @@ async def fetch_page(session, url):
|
|
| 33 |
data = await response.json()
|
| 34 |
return data
|
| 35 |
|
|
|
|
|
|
|
|
|
|
| 36 |
def base26(n):
|
| 37 |
if n == 0:
|
| 38 |
return "A"
|
|
@@ -135,8 +132,10 @@ async def process_post(post_id, session, semaphore):
|
|
| 135 |
@app.get("/allimages")
|
| 136 |
async def all_images(
|
| 137 |
only_first = Query("0", description="Only fetch the first image of each post."),
|
| 138 |
-
post_ids: List[int] = Query(..., alias='post_ids')
|
|
|
|
| 139 |
):
|
|
|
|
| 140 |
async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
|
| 141 |
tasks = []
|
| 142 |
for pid in post_ids:
|
|
@@ -157,7 +156,11 @@ async def all_images(
|
|
| 157 |
return {"image_urls": all_image_urls}
|
| 158 |
|
| 159 |
@app.get("/allimage")
|
| 160 |
-
async def all_image(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
|
| 162 |
try:
|
| 163 |
data = await fetch_page(session, f"https://www.pixiv.net/ajax/illust/{post_id}/pages")
|
|
@@ -172,6 +175,7 @@ async def pixif(
|
|
| 172 |
):
|
| 173 |
post_ids = items.post_ids
|
| 174 |
semaphore = asyncio.Semaphore(8)
|
|
|
|
| 175 |
|
| 176 |
async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
|
| 177 |
tasks = [process_post(post_id, session, semaphore) for post_id in post_ids]
|
|
@@ -187,4 +191,4 @@ async def read_root():
|
|
| 187 |
|
| 188 |
if __name__ == "__main__":
|
| 189 |
import uvicorn
|
| 190 |
-
uvicorn.run(app, host="127.0.0.1", port=7860)
|
|
|
|
| 2 |
import aiohttp
|
| 3 |
import asyncio
|
| 4 |
import time
|
|
|
|
| 5 |
from pydantic import BaseModel
|
| 6 |
from typing import List, Dict
|
| 7 |
|
|
|
|
| 10 |
|
| 11 |
class pixifModel(BaseModel):
|
| 12 |
post_ids: List[int]
|
| 13 |
+
phpsessid: str
|
| 14 |
|
| 15 |
class PixifDownloadModel(BaseModel):
|
| 16 |
posts: Dict[str, str]
|
| 17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
headers = {
|
| 19 |
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
|
| 20 |
'referer': 'https://www.pixiv.net/',
|
|
|
|
| 27 |
data = await response.json()
|
| 28 |
return data
|
| 29 |
|
| 30 |
+
def build_cookies(phpsessid: str) -> Dict[str, str]:
|
| 31 |
+
return {"PHPSESSID": phpsessid}
|
| 32 |
+
|
| 33 |
def base26(n):
|
| 34 |
if n == 0:
|
| 35 |
return "A"
|
|
|
|
| 132 |
@app.get("/allimages")
|
| 133 |
async def all_images(
|
| 134 |
only_first = Query("0", description="Only fetch the first image of each post."),
|
| 135 |
+
post_ids: List[int] = Query(..., alias='post_ids'),
|
| 136 |
+
phpsessid: str = Query(..., description="Pixiv PHPSESSID value.")
|
| 137 |
):
|
| 138 |
+
cookies = build_cookies(phpsessid)
|
| 139 |
async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
|
| 140 |
tasks = []
|
| 141 |
for pid in post_ids:
|
|
|
|
| 156 |
return {"image_urls": all_image_urls}
|
| 157 |
|
| 158 |
@app.get("/allimage")
|
| 159 |
+
async def all_image(
|
| 160 |
+
post_id: int = Query(..., description="The post ID to fetch all images from."),
|
| 161 |
+
phpsessid: str = Query(..., description="Pixiv PHPSESSID value.")
|
| 162 |
+
):
|
| 163 |
+
cookies = build_cookies(phpsessid)
|
| 164 |
async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
|
| 165 |
try:
|
| 166 |
data = await fetch_page(session, f"https://www.pixiv.net/ajax/illust/{post_id}/pages")
|
|
|
|
| 175 |
):
|
| 176 |
post_ids = items.post_ids
|
| 177 |
semaphore = asyncio.Semaphore(8)
|
| 178 |
+
cookies = build_cookies(items.phpsessid)
|
| 179 |
|
| 180 |
async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
|
| 181 |
tasks = [process_post(post_id, session, semaphore) for post_id in post_ids]
|
|
|
|
| 191 |
|
| 192 |
if __name__ == "__main__":
|
| 193 |
import uvicorn
|
| 194 |
+
uvicorn.run(app, host="127.0.0.1", port=7860)
|
Client/hunt.py
CHANGED
|
@@ -8,6 +8,32 @@ local = 0
|
|
| 8 |
endpoint = "http://127.0.0.1:7860" if local else "https://q6-p.hf.space"
|
| 9 |
img_base = 'https://i.pximg.net/img-original/img/'
|
| 10 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
os.chdir(os.path.dirname(os.path.abspath(__file__)))
|
| 12 |
os.makedirs("images/Stash", exist_ok=True)
|
| 13 |
|
|
@@ -55,7 +81,7 @@ for index in indexs:
|
|
| 55 |
print(f"Group: {group_name}\nFiltered: {len(filtered)}/{len(post_ids)}")
|
| 56 |
|
| 57 |
if filtered:
|
| 58 |
-
data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered})
|
| 59 |
print(data)
|
| 60 |
data = data.json()
|
| 61 |
for post_id, url in data.items():
|
|
@@ -85,4 +111,4 @@ for index in indexs:
|
|
| 85 |
if not os.listdir(f'images/{group_name}'):
|
| 86 |
os.rmdir(f"images/{group_name}")
|
| 87 |
|
| 88 |
-
db.close()
|
|
|
|
| 8 |
endpoint = "http://127.0.0.1:7860" if local else "https://q6-p.hf.space"
|
| 9 |
img_base = 'https://i.pximg.net/img-original/img/'
|
| 10 |
|
| 11 |
+
def read_dotenv_value(path, key):
|
| 12 |
+
try:
|
| 13 |
+
with open(path, "r") as env_file:
|
| 14 |
+
for line in env_file:
|
| 15 |
+
line = line.strip()
|
| 16 |
+
if not line or line.startswith("#") or "=" not in line:
|
| 17 |
+
continue
|
| 18 |
+
k, v = line.split("=", 1)
|
| 19 |
+
if k == key:
|
| 20 |
+
return v
|
| 21 |
+
except FileNotFoundError:
|
| 22 |
+
return None
|
| 23 |
+
return None
|
| 24 |
+
|
| 25 |
+
def get_phpsessid():
|
| 26 |
+
phpsessid = os.getenv("PHPSESSID")
|
| 27 |
+
if phpsessid:
|
| 28 |
+
return phpsessid
|
| 29 |
+
env_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".env"))
|
| 30 |
+
phpsessid = read_dotenv_value(env_path, "PHPSESSID")
|
| 31 |
+
if phpsessid:
|
| 32 |
+
return phpsessid
|
| 33 |
+
raise RuntimeError("PHPSESSID is not set in the environment or .env")
|
| 34 |
+
|
| 35 |
+
phpsessid = get_phpsessid()
|
| 36 |
+
|
| 37 |
os.chdir(os.path.dirname(os.path.abspath(__file__)))
|
| 38 |
os.makedirs("images/Stash", exist_ok=True)
|
| 39 |
|
|
|
|
| 81 |
print(f"Group: {group_name}\nFiltered: {len(filtered)}/{len(post_ids)}")
|
| 82 |
|
| 83 |
if filtered:
|
| 84 |
+
data = requests.post(f'{endpoint}/pixif', json={"post_ids": filtered, "phpsessid": phpsessid})
|
| 85 |
print(data)
|
| 86 |
data = data.json()
|
| 87 |
for post_id, url in data.items():
|
|
|
|
| 111 |
if not os.listdir(f'images/{group_name}'):
|
| 112 |
os.rmdir(f"images/{group_name}")
|
| 113 |
|
| 114 |
+
db.close()
|
Client/run_pixif_single.py
CHANGED
|
@@ -12,11 +12,36 @@ if PROJECT_ROOT not in sys.path:
|
|
| 12 |
from API import app as pixif_app # noqa: E402
|
| 13 |
|
| 14 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
async def run_local() -> dict:
|
| 16 |
post_id = 138954885
|
| 17 |
semaphore = asyncio.Semaphore(8)
|
|
|
|
| 18 |
|
| 19 |
-
async with aiohttp.ClientSession(cookies=
|
| 20 |
_, image_url = await pixif_app.process_post(post_id, session, semaphore)
|
| 21 |
|
| 22 |
if not image_url:
|
|
|
|
| 12 |
from API import app as pixif_app # noqa: E402
|
| 13 |
|
| 14 |
|
| 15 |
+
def read_dotenv_value(path, key):
|
| 16 |
+
try:
|
| 17 |
+
with open(path, "r") as env_file:
|
| 18 |
+
for line in env_file:
|
| 19 |
+
line = line.strip()
|
| 20 |
+
if not line or line.startswith("#") or "=" not in line:
|
| 21 |
+
continue
|
| 22 |
+
k, v = line.split("=", 1)
|
| 23 |
+
if k == key:
|
| 24 |
+
return v
|
| 25 |
+
except FileNotFoundError:
|
| 26 |
+
return None
|
| 27 |
+
return None
|
| 28 |
+
|
| 29 |
+
def get_phpsessid():
|
| 30 |
+
phpsessid = os.getenv("PHPSESSID")
|
| 31 |
+
if phpsessid:
|
| 32 |
+
return phpsessid
|
| 33 |
+
env_path = os.path.join(PROJECT_ROOT, ".env")
|
| 34 |
+
phpsessid = read_dotenv_value(env_path, "PHPSESSID")
|
| 35 |
+
if phpsessid:
|
| 36 |
+
return phpsessid
|
| 37 |
+
raise RuntimeError("PHPSESSID is not set in the environment or .env")
|
| 38 |
+
|
| 39 |
async def run_local() -> dict:
|
| 40 |
post_id = 138954885
|
| 41 |
semaphore = asyncio.Semaphore(8)
|
| 42 |
+
cookies = pixif_app.build_cookies(get_phpsessid())
|
| 43 |
|
| 44 |
+
async with aiohttp.ClientSession(cookies=cookies, headers=pixif_app.headers) as session:
|
| 45 |
_, image_url = await pixif_app.process_post(post_id, session, semaphore)
|
| 46 |
|
| 47 |
if not image_url:
|
Client/txt logs/.gitkeep
ADDED
|
File without changes
|