New table names
Browse files- backend/app.py +12 -12
- migrate.py +0 -63
backend/app.py
CHANGED
|
@@ -98,12 +98,12 @@ async def init_db():
|
|
| 98 |
await turso_execute(
|
| 99 |
[
|
| 100 |
{
|
| 101 |
-
"sql": "CREATE TABLE IF NOT EXISTS
|
| 102 |
},
|
| 103 |
{
|
| 104 |
-
"sql": "CREATE TABLE IF NOT EXISTS
|
| 105 |
},
|
| 106 |
-
{"sql": "DELETE FROM
|
| 107 |
]
|
| 108 |
)
|
| 109 |
|
|
@@ -342,7 +342,7 @@ async def save_scan_results(results):
|
|
| 342 |
short_url = url.replace(IMG_BASE, "", 1) if url else ""
|
| 343 |
stmts.append(
|
| 344 |
{
|
| 345 |
-
"sql": "INSERT OR REPLACE INTO
|
| 346 |
"args": [
|
| 347 |
{"type": "text", "value": str(post_id)},
|
| 348 |
{"type": "text", "value": short_url},
|
|
@@ -365,7 +365,7 @@ async def get_scanned_post_ids(post_ids):
|
|
| 365 |
for chunk in chunks:
|
| 366 |
placeholders = ",".join("?" for _ in chunk)
|
| 367 |
stmt = {
|
| 368 |
-
"sql": f"SELECT post_id, url, exif_type FROM
|
| 369 |
"args": [{"type": "text", "value": str(pid)} for pid in chunk],
|
| 370 |
}
|
| 371 |
resp = await turso_execute([stmt])
|
|
@@ -391,7 +391,7 @@ async def bg_search_task(search_id, url, pages, mode, phpsessid):
|
|
| 391 |
try:
|
| 392 |
post_ids, _ = await pixiv_search(url, pages, mode, phpsessid)
|
| 393 |
stmt = {
|
| 394 |
-
"sql": "INSERT OR REPLACE INTO
|
| 395 |
"args": [
|
| 396 |
{"type": "text", "value": search_id},
|
| 397 |
{"type": "text", "value": json.dumps(post_ids)},
|
|
@@ -421,7 +421,7 @@ async def bg_user_task(search_id, user_ids, phpsessid):
|
|
| 421 |
all_post_ids.extend(r["post_ids"])
|
| 422 |
all_post_ids = list(dict.fromkeys(all_post_ids))
|
| 423 |
stmt = {
|
| 424 |
-
"sql": "INSERT OR REPLACE INTO
|
| 425 |
"args": [
|
| 426 |
{"type": "text", "value": search_id},
|
| 427 |
{"type": "text", "value": json.dumps(all_post_ids)},
|
|
@@ -470,7 +470,7 @@ async def bg_search_and_scan_task(search_id, url, pages, mode, phpsessid):
|
|
| 470 |
try:
|
| 471 |
post_ids, _ = await pixiv_search(url, pages, mode, phpsessid)
|
| 472 |
stmt = {
|
| 473 |
-
"sql": "INSERT OR REPLACE INTO
|
| 474 |
"args": [
|
| 475 |
{"type": "text", "value": search_id},
|
| 476 |
{"type": "text", "value": json.dumps(post_ids)},
|
|
@@ -556,7 +556,7 @@ async def scan_search(req: ScanRequest, bg: BackgroundTasks):
|
|
| 556 |
resp = await turso_execute(
|
| 557 |
[
|
| 558 |
{
|
| 559 |
-
"sql": "SELECT post_ids FROM
|
| 560 |
"args": [{"type": "text", "value": req.search_id}],
|
| 561 |
}
|
| 562 |
]
|
|
@@ -581,7 +581,7 @@ async def list_searches():
|
|
| 581 |
resp = await turso_execute(
|
| 582 |
[
|
| 583 |
{
|
| 584 |
-
"sql": "SELECT id, created_at FROM
|
| 585 |
}
|
| 586 |
]
|
| 587 |
)
|
|
@@ -605,7 +605,7 @@ async def get_search(search_id: str):
|
|
| 605 |
resp = await turso_execute(
|
| 606 |
[
|
| 607 |
{
|
| 608 |
-
"sql": "SELECT id, post_ids, created_at FROM
|
| 609 |
"args": [{"type": "text", "value": search_id}],
|
| 610 |
}
|
| 611 |
]
|
|
@@ -632,7 +632,7 @@ async def get_results(search_id: str):
|
|
| 632 |
resp = await turso_execute(
|
| 633 |
[
|
| 634 |
{
|
| 635 |
-
"sql": "SELECT post_ids FROM
|
| 636 |
"args": [{"type": "text", "value": search_id}],
|
| 637 |
}
|
| 638 |
]
|
|
|
|
| 98 |
await turso_execute(
|
| 99 |
[
|
| 100 |
{
|
| 101 |
+
"sql": "CREATE TABLE IF NOT EXISTS pi_searches (id TEXT PRIMARY KEY, post_ids TEXT, created_at INTEGER)"
|
| 102 |
},
|
| 103 |
{
|
| 104 |
+
"sql": "CREATE TABLE IF NOT EXISTS pi_scans (post_id TEXT PRIMARY KEY, url TEXT, exif_type INTEGER)"
|
| 105 |
},
|
| 106 |
+
{"sql": "DELETE FROM pi_searches"},
|
| 107 |
]
|
| 108 |
)
|
| 109 |
|
|
|
|
| 342 |
short_url = url.replace(IMG_BASE, "", 1) if url else ""
|
| 343 |
stmts.append(
|
| 344 |
{
|
| 345 |
+
"sql": "INSERT OR REPLACE INTO pi_scans (post_id, url, exif_type) VALUES (?, ?, ?)",
|
| 346 |
"args": [
|
| 347 |
{"type": "text", "value": str(post_id)},
|
| 348 |
{"type": "text", "value": short_url},
|
|
|
|
| 365 |
for chunk in chunks:
|
| 366 |
placeholders = ",".join("?" for _ in chunk)
|
| 367 |
stmt = {
|
| 368 |
+
"sql": f"SELECT post_id, url, exif_type FROM pi_scans WHERE post_id IN ({placeholders})",
|
| 369 |
"args": [{"type": "text", "value": str(pid)} for pid in chunk],
|
| 370 |
}
|
| 371 |
resp = await turso_execute([stmt])
|
|
|
|
| 391 |
try:
|
| 392 |
post_ids, _ = await pixiv_search(url, pages, mode, phpsessid)
|
| 393 |
stmt = {
|
| 394 |
+
"sql": "INSERT OR REPLACE INTO pi_searches (id, post_ids, created_at) VALUES (?, ?, ?)",
|
| 395 |
"args": [
|
| 396 |
{"type": "text", "value": search_id},
|
| 397 |
{"type": "text", "value": json.dumps(post_ids)},
|
|
|
|
| 421 |
all_post_ids.extend(r["post_ids"])
|
| 422 |
all_post_ids = list(dict.fromkeys(all_post_ids))
|
| 423 |
stmt = {
|
| 424 |
+
"sql": "INSERT OR REPLACE INTO pi_searches (id, post_ids, created_at) VALUES (?, ?, ?)",
|
| 425 |
"args": [
|
| 426 |
{"type": "text", "value": search_id},
|
| 427 |
{"type": "text", "value": json.dumps(all_post_ids)},
|
|
|
|
| 470 |
try:
|
| 471 |
post_ids, _ = await pixiv_search(url, pages, mode, phpsessid)
|
| 472 |
stmt = {
|
| 473 |
+
"sql": "INSERT OR REPLACE INTO pi_searches (id, post_ids, created_at) VALUES (?, ?, ?)",
|
| 474 |
"args": [
|
| 475 |
{"type": "text", "value": search_id},
|
| 476 |
{"type": "text", "value": json.dumps(post_ids)},
|
|
|
|
| 556 |
resp = await turso_execute(
|
| 557 |
[
|
| 558 |
{
|
| 559 |
+
"sql": "SELECT post_ids FROM pi_searches WHERE id = ?",
|
| 560 |
"args": [{"type": "text", "value": req.search_id}],
|
| 561 |
}
|
| 562 |
]
|
|
|
|
| 581 |
resp = await turso_execute(
|
| 582 |
[
|
| 583 |
{
|
| 584 |
+
"sql": "SELECT id, created_at FROM pi_searches ORDER BY created_at DESC LIMIT 100"
|
| 585 |
}
|
| 586 |
]
|
| 587 |
)
|
|
|
|
| 605 |
resp = await turso_execute(
|
| 606 |
[
|
| 607 |
{
|
| 608 |
+
"sql": "SELECT id, post_ids, created_at FROM pi_searches WHERE id = ?",
|
| 609 |
"args": [{"type": "text", "value": search_id}],
|
| 610 |
}
|
| 611 |
]
|
|
|
|
| 632 |
resp = await turso_execute(
|
| 633 |
[
|
| 634 |
{
|
| 635 |
+
"sql": "SELECT post_ids FROM pi_searches WHERE id = ?",
|
| 636 |
"args": [{"type": "text", "value": search_id}],
|
| 637 |
}
|
| 638 |
]
|
migrate.py
DELETED
|
@@ -1,63 +0,0 @@
|
|
| 1 |
-
import sqlite3
|
| 2 |
-
import httpx
|
| 3 |
-
import os
|
| 4 |
-
import sys
|
| 5 |
-
|
| 6 |
-
TURSO_DB_URL = os.getenv("TURSO_DB_URL", "https://main-hailxd.aws-ap-northeast-1.turso.io")
|
| 7 |
-
TURSO_AUTH_TOKEN = os.getenv("TURSO_AUTH_TOKEN_WRITE", "")
|
| 8 |
-
IMG_BASE = "https://i.pximg.net/img-original/img/"
|
| 9 |
-
BATCH_SIZE = 200
|
| 10 |
-
|
| 11 |
-
def turso_url():
|
| 12 |
-
base = TURSO_DB_URL.rstrip("/")
|
| 13 |
-
if not base.startswith("http"):
|
| 14 |
-
base = "https://" + base
|
| 15 |
-
return base + "/v2/pipeline"
|
| 16 |
-
|
| 17 |
-
def turso_post(stmts):
|
| 18 |
-
headers = {"Authorization": f"Bearer {TURSO_AUTH_TOKEN}", "Content-Type": "application/json"}
|
| 19 |
-
body = {"requests": [{"type": "execute", "stmt": s} for s in stmts] + [{"type": "close"}]}
|
| 20 |
-
r = httpx.post(turso_url(), json=body, headers=headers, timeout=120)
|
| 21 |
-
r.raise_for_status()
|
| 22 |
-
return r.json()
|
| 23 |
-
|
| 24 |
-
def main():
|
| 25 |
-
db_path = sys.argv[1] if len(sys.argv) > 1 else r"C:\Users\jyeal\Documents\programming-related\Pixif\Client\db.sqlite"
|
| 26 |
-
conn = sqlite3.connect(db_path)
|
| 27 |
-
|
| 28 |
-
turso_post([
|
| 29 |
-
{"sql": "CREATE TABLE IF NOT EXISTS searches (id TEXT PRIMARY KEY, query TEXT, post_ids TEXT, created_at INTEGER)"},
|
| 30 |
-
{"sql": "CREATE TABLE IF NOT EXISTS scans (post_id TEXT PRIMARY KEY, url TEXT, exif_type INTEGER)"},
|
| 31 |
-
])
|
| 32 |
-
|
| 33 |
-
rows = conn.execute("SELECT post_id, url, exif_type FROM pixif_cache").fetchall()
|
| 34 |
-
print(f"Found {len(rows)} rows in local db")
|
| 35 |
-
|
| 36 |
-
stmts = []
|
| 37 |
-
for post_id, url, exif_type in rows:
|
| 38 |
-
short_url = url.replace(IMG_BASE, "", 1) if url else ""
|
| 39 |
-
args = [
|
| 40 |
-
{"type": "text", "value": str(post_id)},
|
| 41 |
-
{"type": "text", "value": short_url},
|
| 42 |
-
]
|
| 43 |
-
if exif_type is not None:
|
| 44 |
-
args.append({"type": "integer", "value": str(exif_type)})
|
| 45 |
-
else:
|
| 46 |
-
args.append({"type": "null"})
|
| 47 |
-
stmts.append({
|
| 48 |
-
"sql": "INSERT OR IGNORE INTO scans (post_id, url, exif_type) VALUES (?, ?, ?)",
|
| 49 |
-
"args": args,
|
| 50 |
-
})
|
| 51 |
-
|
| 52 |
-
uploaded = 0
|
| 53 |
-
for i in range(0, len(stmts), BATCH_SIZE):
|
| 54 |
-
batch = stmts[i:i+BATCH_SIZE]
|
| 55 |
-
turso_post(batch)
|
| 56 |
-
uploaded += len(batch)
|
| 57 |
-
print(f" {uploaded}/{len(stmts)}")
|
| 58 |
-
|
| 59 |
-
print(f"Done. Uploaded {len(stmts)} rows to Turso.")
|
| 60 |
-
conn.close()
|
| 61 |
-
|
| 62 |
-
if __name__ == "__main__":
|
| 63 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|