Moved seasrch to client

.gitignore

@@ -2,4 +2,5 @@
 *.txt
 db
 images
-merge_dev.bat
+merge_dev.bat
+__pycache__

API/app.py

@@ -34,43 +34,6 @@ async def fetch_page(session, url):
         data = await response.json()
         return data
 
-async def search(raw, pages, ai_only=True, real_only=True, cookies=None, headers=None):
-    keywords = raw.split('tags/')[-1].split('/')[0]
-    url = f"https://www.pixiv.net/ajax/search/artworks/{keywords}?word={keywords}"
-    if "?" in raw:
-        params = raw.split('?')[1]
-        url += f"&{params}"
-    if "s_mode" not in url:
-        url += "&s_mode=s_tag_full"
-
-    post_ids = []
-    tasks = []
-
-    async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
-        for page in range(1, pages + 1):
-            page_url = f"{url.strip()}&p={page}"
-            task = fetch_page(session, page_url)
-            tasks.append(task)
-
-        responses = await asyncio.gather(*tasks)
-        posts = []
-        for data in responses:
-            if ai_only:
-                for post in data['body']['illustManga']['data']:
-                    if post['aiType'] == 2:
-                        posts.append(post)
-            if real_only:
-                for post in data['body']['illustManga']['data']:
-                    if post['aiType'] != 2:
-                        posts.append(post)
-            else:
-                posts = data['body']['illustManga']['data']
-            if not posts:
-                break
-            post_ids.extend([post['id'] for post in posts])
-
-    return post_ids, requests.utils.unquote(keywords, encoding='utf-8')
-
 def base26(n):
     if n == 0:
         return "A"
@@ -85,58 +48,6 @@ def base26(n):
 def base26_time():
     return base26(int(time.time()))
 
-@app.get("/search")
-async def search_endpoint(
-    raw: str = Query(..., description="The raw URL to search."),
-    pages: int = Query(1, description="Number of pages to fetch."),
-    ai_only: bool = Query(True, description="Filter for AI-generated content."),
-    real_only: bool = Query(True, description="Filter for real content."),
-):
-    try:
-        post_ids, keywords = await search(raw, pages, ai_only, real_only, cookies=cookies, headers=headers)
-        return {"post_ids": post_ids, "filename": base26_time() + "_" + keywords}
-    except Exception as e:
-        return {"error": str(e)}
-
-@app.get("/user")
-async def user(
-    user_id: int = Query(..., description="The user ID to fetch.")
-):
-    async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
-        data = await fetch_page(session, f'https://www.pixiv.net/ajax/user/{user_id}/profile/all')
-        posts = data["body"]["illusts"].keys()
-        try:
-            username = data['body']['pickup'][0]['userName']
-        except (KeyError, IndexError):
-            user_data = await fetch_page(session, f"https://www.pixiv.net/ajax/user/{user_id}")
-            username = user_data['body']['name']
-    
-    return {"post_ids": list(posts), "filename": base26_time() + "_" + username.replace("|", "")}
-
-@app.get("/users")
-async def users(
-    user_ids: List[int] = Query(..., description="List of user IDs to fetch.", alias="user_ids")
-):
-    async def fetch_user_data(session, uid):
-        try:
-            data = await fetch_page(session, f'https://www.pixiv.net/ajax/user/{uid}/profile/all')
-            posts = list(data["body"]["illusts"].keys())
-            try:
-                username = data['body']['pickup'][0]['userName']
-            except (KeyError, IndexError):
-                user_data = await fetch_page(session, f"https://www.pixiv.net/ajax/user/{uid}")
-                username = user_data['body']['name']
-            filename = base26_time() + "_" + username.replace("|", "")
-            return {"post_ids": posts, "filename": filename}
-        except Exception as e:
-            return {"user_id": uid, "error": str(e)}
-
-    async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
-        tasks = [fetch_user_data(session, uid) for uid in user_ids]
-        results = await asyncio.gather(*tasks)
-
-    return results
-
 def determine_exif_type(metadata):
     if metadata is None:
         return None
@@ -275,4 +186,4 @@ async def read_root():
 
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run(app, host="127.0.0.1", port=7860)
+    uvicorn.run(app, host="127.0.0.1", port=7860)

Client/Extract Pixiv/ai_search.py

@@ -1,30 +1,20 @@
-import requests
 import os
-
-local = 1
-if local:
-    endpoint = "http://127.0.0.1:7860"
-else:
-    endpoint = "https://q6-p.hf.space"
+import asyncio
+import sys
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+from pixiv_api import search, cookies, headers
 
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
 input_url = input("Enter the URL: ")
 
-pages = 3
-
-params = {
-    'raw': input_url,
-    'pages': pages,
-    'ai_only': True,
-    'real_only': False,
-}
+pages = 5
 
-response = requests.get(f'{endpoint}/search', params=params)
+async def main():
+    post_ids, filename = await search(input_url, pages, ai_only=True, real_only=False, cookies=cookies, headers=headers)
+    post_ids = list(dict.fromkeys(post_ids))
+    with open(f"../{filename}.txt", "w") as f:
+        f.write("\n".join(map(str, post_ids)))
 
-data = response.json()
-print(data)
-post_ids = data['post_ids']
-post_ids = list(dict.fromkeys(post_ids))
-with open(f"../{data['filename']}.txt", "w") as f:
-    f.write("\n".join(post_ids))
+if __name__ == "__main__":
+    asyncio.run(main())

Client/Extract Pixiv/pixiv_api.py

@@ -0,0 +1,192 @@
+import aiohttp
+import asyncio
+import requests.utils
+import time
+import os
+from dotenv import load_dotenv
+
+img_base = 'https://i.pximg.net/img-original/img/'
+
+load_dotenv()
+
+PHPSESSID = os.getenv("PHPSESSID")
+
+cookies = {"PHPSESSID": PHPSESSID}
+
+headers = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
+    'referer': 'https://www.pixiv.net/',
+}
+
+async def fetch_page(session, url):
+    async with session.get(url) as response:
+        data = await response.json()
+        return data
+
+async def search(raw, pages, ai_only=True, real_only=True, cookies=None, headers=None):
+    keywords = raw.split('tags/')[-1].split('/')[0]
+    url = f"https://www.pixiv.net/ajax/search/artworks/{keywords}?word={keywords}"
+    if "?" in raw:
+        params = raw.split('?')[1]
+        url += f"&{params}"
+    if "s_mode" not in url:
+        url += "&s_mode=s_tag_full"
+
+    post_ids = []
+    tasks = []
+
+    async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
+        for page in range(1, pages + 1):
+            page_url = f"{url.strip()}&p={page}"
+            task = fetch_page(session, page_url)
+            tasks.append(task)
+
+        responses = await asyncio.gather(*tasks)
+        posts = []
+        for data in responses:
+            if ai_only:
+                for post in data['body']['illustManga']['data']:
+                    if post['aiType'] == 2:
+                        posts.append(post)
+            if real_only:
+                for post in data['body']['illustManga']['data']:
+                    if post['aiType'] != 2:
+                        posts.append(post)
+            else:
+                posts = data['body']['illustManga']['data']
+            if not posts:
+                break
+            post_ids.extend([post['id'] for post in posts])
+
+    return post_ids, requests.utils.unquote(keywords, encoding='utf-8')
+
+def base26(n):
+    if n == 0:
+        return "A"
+
+    b26 = ""
+    while n > 0:
+        n, remainder = divmod(n, 26)
+        b26 = chr(97 + remainder) + b26
+
+    return b26
+
+def base26_time():
+    return base26(int(time.time()))
+
+async def get_user(user_id, session):
+    data = await fetch_page(session, f'https://www.pixiv.net/ajax/user/{user_id}/profile/all')
+    posts = data["body"]["illusts"].keys()
+    try:
+        username = data['body']['pickup'][0]['userName']
+    except (KeyError, IndexError):
+        user_data = await fetch_page(session, f"https://www.pixiv.net/ajax/user/{user_id}")
+        username = user_data['body']['name']
+    
+    return {"post_ids": list(posts), "filename": base26_time() + "_" + username.replace("|", "")}
+
+async def get_users(user_ids):
+    async def fetch_user_data(session, uid):
+        try:
+            return await get_user(uid, session)
+        except Exception as e:
+            return {"user_id": uid, "error": str(e)}
+
+    async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
+        tasks = [fetch_user_data(session, uid) for uid in user_ids]
+        results = await asyncio.gather(*tasks)
+
+    return results
+
+def determine_exif_type(metadata):
+    if metadata is None:
+        return None
+    elif metadata == b'TitleAI generated image':
+        return "novelai"
+    elif metadata.startswith(b"parameter"):
+        return "sd"
+    elif b'{"' in metadata:
+        return "comfy"
+    elif b"Dig" in metadata:
+        return "mj"
+    elif metadata.startswith(b"SoftwareCelsys"):
+        return "celsys"
+    else:
+        return "photoshop"
+
+async def get_exif(url, session):
+    start_range = 0
+    end_range = 512
+
+    headers = {
+        "Referer": "https://www.pixiv.net/",
+        "Range": f"bytes={start_range}-{end_range}"
+    }
+
+    async with session.get(url, headers=headers) as response:
+        data = await response.read()
+        return parse_png_metadata(data)
+
+def parse_png_metadata(data):
+    index = 8
+
+    while index < len(data):
+        if index + 8 > len(data):
+            break
+        chunk_len = int.from_bytes(data[index:index+4], 'big')
+        chunk_type = data[index+4:index+8].decode('ascii')
+        index += 8
+
+        if chunk_type in ['tEXt', 'iTXt']:
+            content = data[index:index+chunk_len]
+            if chunk_type == 'tEXt':
+                return content.replace(b'\0', b'')
+            elif chunk_type == 'iTXt':
+                return content.strip()
+
+        index += chunk_len + 4
+    return None
+
+async def process_post(post_id, session, semaphore):
+    async with semaphore:
+        try:
+            data = await fetch_page(session, f"https://www.pixiv.net/ajax/illust/{post_id}/pages")
+            image_urls = [page['urls']['original'] for page in data['body'] if 'png' in page['urls']['original']]
+            
+            initial_offsets = [1, 5, 5, 10, 10, 10]
+            
+            chunks = []
+            start = 0
+            for offset in initial_offsets:
+                end = start + offset
+                if end > len(image_urls):
+                    end = len(image_urls)
+                chunks.append((start, end))
+                start = end
+            
+            while start < len(image_urls):
+                end = min(start + 10, len(image_urls))
+                chunks.append((start, end))
+                start = end
+
+            for s, e in chunks:
+                chunk_tasks = [get_exif(image_urls[i], session) for i in range(s, e)]
+                results = await asyncio.gather(*chunk_tasks)
+
+                for image_url, metadata in zip(image_urls[s:e], results):
+                    exif_type = determine_exif_type(metadata)
+                    if exif_type not in ['photoshop', 'celsys', None]:
+                        return post_id, image_url
+                    
+            return post_id, None
+        except:
+            return post_id, None
+
+async def get_pixif_data(post_ids):
+    semaphore = asyncio.Semaphore(100)
+    async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
+        tasks = [process_post(post_id, session, semaphore) for post_id in post_ids]
+        results = await asyncio.gather(*tasks)
+
+    image_exifs = {post_id: image_url.replace(img_base, '', 1) for post_id, image_url in results if image_url}
+    return image_exifs

Client/Extract Pixiv/real_search.py

@@ -1,11 +1,8 @@
-import requests
 import os
-
-local = 0
-if local:
-    endpoint = "http://127.0.0.1:7860"
-else:
-    endpoint = "https://q6-p.hf.space"
+import asyncio
+import sys
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+from pixiv_api import search, cookies, headers
 
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
@@ -13,15 +10,10 @@ input_url = input("Enter the URL: ")
 
 pages = 300 // 60
 
-params = {
-    'raw': input_url,
-    'pages': pages,
-    'ai_only': False,
-    'real_only': True,
-}
-
-response = requests.get(f'{endpoint}/search', params=params)
+async def main():
+    post_ids, filename = await search(input_url, pages, ai_only=False, real_only=True, cookies=cookies, headers=headers)
+    with open(f"../{filename}.txt", "w") as f:
+        f.write("\n".join(map(str, post_ids)))
 
-data = response.json()
-with open(f"../{data['filename']}.txt", "w") as f:
-    f.write("\n".join(data['post_ids']))
+if __name__ == "__main__":
+    asyncio.run(main())

Client/Extract Pixiv/user.py

@@ -1,13 +1,9 @@
-import requests
 import re
 import os
-
-local = 0
-
-if local:
-    endpoint = "http://127.0.0.1:7860"
-else:
-    endpoint = "https://q6-p.hf.space"
+import asyncio
+import sys
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+from pixiv_api import get_users
 
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
@@ -20,17 +16,18 @@ if len(user_ids) == 0:
     import sys
     
     sys.exit()
-    
-response = requests.get(f'{endpoint}/users', params={'user_ids': user_ids})
-response.raise_for_status()
-data = response.json()
 
-for user_data in data:
-    if 'error' in user_data:
-        print(f"User ID {user_data.get('user_id')} Error: {user_data.get('error')}")
-        continue
-    filename = user_data['filename']
-    post_ids = user_data['post_ids']
+async def main():
+    data = await get_users(user_ids)
+    for user_data in data:
+        if 'error' in user_data:
+            print(f"User ID {user_data.get('user_id')} Error: {user_data.get('error')}")
+            continue
+        filename = user_data['filename']
+        post_ids = user_data['post_ids']
+
+        with open(os.path.join("..", filename + '.txt'), "w", encoding='utf-8') as f:
+            f.write("\n".join(map(str, post_ids)))
 
-    with open(os.path.join("..", filename + '.txt'), "w", encoding='utf-8') as f:
-        f.write("\n".join(post_ids))
+if __name__ == "__main__":
+    asyncio.run(main())

Client/hunt.py

@@ -76,10 +76,11 @@ for index in indexs:
     for i, post_id in enumerate(post_ids):
         stash_path = f"images/Stash/{post_id}.png"
         dest_path = f"images/{group_name}/{i}_{post_id}.png"
+
         if os.path.exists(stash_path) and not os.path.exists(dest_path):
             os.link(stash_path, dest_path)
 
     if not os.listdir(f'images/{group_name}'):
         os.rmdir(f"images/{group_name}")
 
-db.close()
+db.close()