x

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+.env
+*.txt
+db

API/README.md

@@ -0,0 +1,10 @@
+---
+title: P
+emoji: 🚀
+colorFrom: purple
+colorTo: yellow
+sdk: docker
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

API/app.py

@@ -0,0 +1,130 @@
+from fastapi import FastAPI
+import aiohttp
+import asyncio
+import time
+import uvicorn
+from dotenv import load_dotenv
+import os
+from pydantic import BaseModel
+from typing import List
+
+class pixifModel(BaseModel):
+    post_ids: List[int]
+
+env_path = os.path.dirname(os.path.realpath(__file__)) + "/../.env"
+# config = dotenv_values(env_path)
+# PHPSESSID = config.get("PHPSESSID")
+if os.path.exists(env_path):
+    load_dotenv(env_path)
+PHPSESSID = os.getenv("PHPSESSID")
+cookies = {"PHPSESSID": PHPSESSID}
+
+headers = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
+    'referer': 'https://www.pixiv.net/',
+}
+
+app = FastAPI()
+
+async def fetch_page(session, url):
+    async with session.get(url) as response:
+        data = await response.json()
+        return data
+
+def base26(n):
+    if n == 0:
+        return "A"
+
+    b26 = ""
+    while n > 0:
+        n, remainder = divmod(n, 26)
+        b26 = chr(97 + remainder) + b26
+
+    return b26
+
+def base26_time():
+    return base26(int(time.time()))
+
+def determine_exif_type(metadata):
+    if metadata is None:
+        return None
+    elif metadata == b'TitleAI generated image':
+        return "novelai"
+    elif metadata.startswith(b"parameter"):
+        return "sd"
+    elif b'{"' in metadata:
+        return "comfy"
+    elif b"Dig" in metadata:
+        return "mj"
+    elif metadata.startswith(b"SoftwareCelsys"):
+        return "celsys"
+    else:
+        return "photoshop"
+
+async def get_exif(url, session):
+    start_range = 0
+    end_range = 1024
+
+    headers = {
+        "Referer": "https://www.pixiv.net/",
+        "Range": f"bytes={start_range}-{end_range}"
+    }
+
+    async with session.get(url, headers=headers) as response:
+        data = await response.read()
+        return parse_png_metadata(data)
+
+def parse_png_metadata(data):
+    index = 8
+
+    while index < len(data):
+        if index + 8 > len(data):
+            break
+        chunk_len = int.from_bytes(data[index:index+4], 'big')
+        chunk_type = data[index+4:index+8].decode('ascii')
+        index += 8
+
+        if chunk_type in ['tEXt', 'iTXt']:
+            content = data[index:index+chunk_len]
+            if chunk_type == 'tEXt':
+                return content.replace(b'\0', b'')
+            elif chunk_type == 'iTXt':
+                return content.strip()
+
+        index += chunk_len + 4
+    return None
+
+async def process_post(post_id, session, semaphore):
+    async with semaphore:
+        try:
+            data = await fetch_page(session, f"https://www.pixiv.net/ajax/illust/{post_id}/pages")
+            image_urls = [page['urls']['original'] for page in data['body'] if 'png' in page['urls']['original']][:12]
+            for image_url in image_urls:
+                metadata = await get_exif(image_url, session)
+                exif_type = determine_exif_type(metadata)
+                if exif_type not in ['photoshop', 'celsys', None]:
+                    return post_id, image_url
+            return post_id, None
+        except Exception as e:
+            return post_id, None
+
+@app.post("/pixif")
+async def pixif(
+    items: pixifModel
+):
+    post_ids = items.post_ids
+    semaphore = asyncio.Semaphore(100)
+
+    async with aiohttp.ClientSession(cookies=cookies, headers=headers) as session:
+        tasks = [process_post(post_id, session, semaphore) for post_id in post_ids]
+        results = await asyncio.gather(*tasks)
+
+    image_exifs = {post_id: image_url for post_id, image_url in results if image_url}
+    return {"image_exifs": image_exifs}
+
+@app.get("/")
+async def read_root():
+    return {"message": "Hello, World!"}
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="127.0.0.1", port=7860)

Client/Extract Pixiv/ai_search.py

@@ -0,0 +1,21 @@
+import requests
+import os
+
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+input_url = input("Enter the URL: ")
+
+pages = 60 // 60
+
+params = {
+    'raw': input_url,
+    'pages': pages,
+    'ai_only': True,
+}
+
+response = requests.get('http://127.0.0.1:8000/search', params=params)
+
+data = response.json()
+
+with open(f"../{data['filename']}.txt", "w") as f:
+    f.write("\n".join(data['post_ids']))

Client/Extract Pixiv/user.py

@@ -0,0 +1,27 @@
+import requests
+import re
+import os
+
+def main():
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+    inp = input("Enter User IDs (separated by spaces or commas): ")
+    user_ids = re.findall(r"\d+", inp)
+
+    user_ids = [int(uid) for uid in user_ids]
+
+    response = requests.get('http://127.0.0.1:8000/users', params={'user_ids': user_ids})
+    response.raise_for_status()
+    data = response.json()
+
+    for user_data in data:
+        if 'error' in user_data:
+            print(f"User ID {user_data.get('user_id')} Error: {user_data.get('error')}")
+            continue
+        filename = user_data['filename']
+        post_ids = user_data['post_ids']
+        with open(os.path.join("..", filename + '.txt'), "w", encoding='utf-8') as f:
+            f.write("\n".join(post_ids))
+
+if __name__ == "__main__":
+    main()

Client/hunt.py

@@ -0,0 +1,35 @@
+import os
+import lmdb
+import requests
+
+
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+db = lmdb.open("db", subdir=True, map_size=10485760)
+
+valid = [f for f in os.listdir() if f.endswith(".txt")]
+print(valid)
+for idx, file in enumerate(valid):
+    print(f"{idx + 1}: {file}")
+
+inputs = input("Enter the index of the file: ")
+inputs = inputs.split()
+indexs = []
+
+for inp in inputs:
+    if inp.isdigit():
+        indexs.append(int(inp) - 1)
+    elif "-" in inp:
+        start, end = map(int, inp.split("-"))
+        indexs.extend(range(start - 1, end))
+
+for index in indexs:
+    with open(valid[index], "r") as f:
+        post_ids = [x for x in f.read().split("\n") if x]
+
+    with db.begin(write=False) as txn:
+        post_ids = [post_id for post_id in post_ids if not txn.get(post_id.encode())]
+
+    data = requests.post('http://127.0.0.1:8000/pixif', json={"post_ids": post_ids}).json()
+
+    print(data)

Dockerfile

@@ -0,0 +1,14 @@
+
+FROM python:3.10-slim
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+WORKDIR /app
+
+RUN pip install --no-cache-dir fastapi aiohttp uvicorn python-dotenv pydantic
+
+COPY --chown=user ./API /app
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,11 @@
+---
+title: P
+emoji: 🐠
+colorFrom: gray
+colorTo: purple
+sdk: docker
+pinned: false
+app_port: 7860
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference