q6 commited on
Commit
7d1b2dc
·
1 Parent(s): b8b8795

New table names

Browse files
Files changed (2) hide show
  1. backend/app.py +12 -12
  2. migrate.py +0 -63
backend/app.py CHANGED
@@ -98,12 +98,12 @@ async def init_db():
98
  await turso_execute(
99
  [
100
  {
101
- "sql": "CREATE TABLE IF NOT EXISTS searches (id TEXT PRIMARY KEY, post_ids TEXT, created_at INTEGER)"
102
  },
103
  {
104
- "sql": "CREATE TABLE IF NOT EXISTS scans (post_id TEXT PRIMARY KEY, url TEXT, exif_type INTEGER)"
105
  },
106
- {"sql": "DELETE FROM searches"},
107
  ]
108
  )
109
 
@@ -342,7 +342,7 @@ async def save_scan_results(results):
342
  short_url = url.replace(IMG_BASE, "", 1) if url else ""
343
  stmts.append(
344
  {
345
- "sql": "INSERT OR REPLACE INTO scans (post_id, url, exif_type) VALUES (?, ?, ?)",
346
  "args": [
347
  {"type": "text", "value": str(post_id)},
348
  {"type": "text", "value": short_url},
@@ -365,7 +365,7 @@ async def get_scanned_post_ids(post_ids):
365
  for chunk in chunks:
366
  placeholders = ",".join("?" for _ in chunk)
367
  stmt = {
368
- "sql": f"SELECT post_id, url, exif_type FROM scans WHERE post_id IN ({placeholders})",
369
  "args": [{"type": "text", "value": str(pid)} for pid in chunk],
370
  }
371
  resp = await turso_execute([stmt])
@@ -391,7 +391,7 @@ async def bg_search_task(search_id, url, pages, mode, phpsessid):
391
  try:
392
  post_ids, _ = await pixiv_search(url, pages, mode, phpsessid)
393
  stmt = {
394
- "sql": "INSERT OR REPLACE INTO searches (id, post_ids, created_at) VALUES (?, ?, ?)",
395
  "args": [
396
  {"type": "text", "value": search_id},
397
  {"type": "text", "value": json.dumps(post_ids)},
@@ -421,7 +421,7 @@ async def bg_user_task(search_id, user_ids, phpsessid):
421
  all_post_ids.extend(r["post_ids"])
422
  all_post_ids = list(dict.fromkeys(all_post_ids))
423
  stmt = {
424
- "sql": "INSERT OR REPLACE INTO searches (id, post_ids, created_at) VALUES (?, ?, ?)",
425
  "args": [
426
  {"type": "text", "value": search_id},
427
  {"type": "text", "value": json.dumps(all_post_ids)},
@@ -470,7 +470,7 @@ async def bg_search_and_scan_task(search_id, url, pages, mode, phpsessid):
470
  try:
471
  post_ids, _ = await pixiv_search(url, pages, mode, phpsessid)
472
  stmt = {
473
- "sql": "INSERT OR REPLACE INTO searches (id, post_ids, created_at) VALUES (?, ?, ?)",
474
  "args": [
475
  {"type": "text", "value": search_id},
476
  {"type": "text", "value": json.dumps(post_ids)},
@@ -556,7 +556,7 @@ async def scan_search(req: ScanRequest, bg: BackgroundTasks):
556
  resp = await turso_execute(
557
  [
558
  {
559
- "sql": "SELECT post_ids FROM searches WHERE id = ?",
560
  "args": [{"type": "text", "value": req.search_id}],
561
  }
562
  ]
@@ -581,7 +581,7 @@ async def list_searches():
581
  resp = await turso_execute(
582
  [
583
  {
584
- "sql": "SELECT id, created_at FROM searches ORDER BY created_at DESC LIMIT 100"
585
  }
586
  ]
587
  )
@@ -605,7 +605,7 @@ async def get_search(search_id: str):
605
  resp = await turso_execute(
606
  [
607
  {
608
- "sql": "SELECT id, post_ids, created_at FROM searches WHERE id = ?",
609
  "args": [{"type": "text", "value": search_id}],
610
  }
611
  ]
@@ -632,7 +632,7 @@ async def get_results(search_id: str):
632
  resp = await turso_execute(
633
  [
634
  {
635
- "sql": "SELECT post_ids FROM searches WHERE id = ?",
636
  "args": [{"type": "text", "value": search_id}],
637
  }
638
  ]
 
98
  await turso_execute(
99
  [
100
  {
101
+ "sql": "CREATE TABLE IF NOT EXISTS pi_searches (id TEXT PRIMARY KEY, post_ids TEXT, created_at INTEGER)"
102
  },
103
  {
104
+ "sql": "CREATE TABLE IF NOT EXISTS pi_scans (post_id TEXT PRIMARY KEY, url TEXT, exif_type INTEGER)"
105
  },
106
+ {"sql": "DELETE FROM pi_searches"},
107
  ]
108
  )
109
 
 
342
  short_url = url.replace(IMG_BASE, "", 1) if url else ""
343
  stmts.append(
344
  {
345
+ "sql": "INSERT OR REPLACE INTO pi_scans (post_id, url, exif_type) VALUES (?, ?, ?)",
346
  "args": [
347
  {"type": "text", "value": str(post_id)},
348
  {"type": "text", "value": short_url},
 
365
  for chunk in chunks:
366
  placeholders = ",".join("?" for _ in chunk)
367
  stmt = {
368
+ "sql": f"SELECT post_id, url, exif_type FROM pi_scans WHERE post_id IN ({placeholders})",
369
  "args": [{"type": "text", "value": str(pid)} for pid in chunk],
370
  }
371
  resp = await turso_execute([stmt])
 
391
  try:
392
  post_ids, _ = await pixiv_search(url, pages, mode, phpsessid)
393
  stmt = {
394
+ "sql": "INSERT OR REPLACE INTO pi_searches (id, post_ids, created_at) VALUES (?, ?, ?)",
395
  "args": [
396
  {"type": "text", "value": search_id},
397
  {"type": "text", "value": json.dumps(post_ids)},
 
421
  all_post_ids.extend(r["post_ids"])
422
  all_post_ids = list(dict.fromkeys(all_post_ids))
423
  stmt = {
424
+ "sql": "INSERT OR REPLACE INTO pi_searches (id, post_ids, created_at) VALUES (?, ?, ?)",
425
  "args": [
426
  {"type": "text", "value": search_id},
427
  {"type": "text", "value": json.dumps(all_post_ids)},
 
470
  try:
471
  post_ids, _ = await pixiv_search(url, pages, mode, phpsessid)
472
  stmt = {
473
+ "sql": "INSERT OR REPLACE INTO pi_searches (id, post_ids, created_at) VALUES (?, ?, ?)",
474
  "args": [
475
  {"type": "text", "value": search_id},
476
  {"type": "text", "value": json.dumps(post_ids)},
 
556
  resp = await turso_execute(
557
  [
558
  {
559
+ "sql": "SELECT post_ids FROM pi_searches WHERE id = ?",
560
  "args": [{"type": "text", "value": req.search_id}],
561
  }
562
  ]
 
581
  resp = await turso_execute(
582
  [
583
  {
584
+ "sql": "SELECT id, created_at FROM pi_searches ORDER BY created_at DESC LIMIT 100"
585
  }
586
  ]
587
  )
 
605
  resp = await turso_execute(
606
  [
607
  {
608
+ "sql": "SELECT id, post_ids, created_at FROM pi_searches WHERE id = ?",
609
  "args": [{"type": "text", "value": search_id}],
610
  }
611
  ]
 
632
  resp = await turso_execute(
633
  [
634
  {
635
+ "sql": "SELECT post_ids FROM pi_searches WHERE id = ?",
636
  "args": [{"type": "text", "value": search_id}],
637
  }
638
  ]
migrate.py DELETED
@@ -1,63 +0,0 @@
1
- import sqlite3
2
- import httpx
3
- import os
4
- import sys
5
-
6
- TURSO_DB_URL = os.getenv("TURSO_DB_URL", "https://main-hailxd.aws-ap-northeast-1.turso.io")
7
- TURSO_AUTH_TOKEN = os.getenv("TURSO_AUTH_TOKEN_WRITE", "")
8
- IMG_BASE = "https://i.pximg.net/img-original/img/"
9
- BATCH_SIZE = 200
10
-
11
- def turso_url():
12
- base = TURSO_DB_URL.rstrip("/")
13
- if not base.startswith("http"):
14
- base = "https://" + base
15
- return base + "/v2/pipeline"
16
-
17
- def turso_post(stmts):
18
- headers = {"Authorization": f"Bearer {TURSO_AUTH_TOKEN}", "Content-Type": "application/json"}
19
- body = {"requests": [{"type": "execute", "stmt": s} for s in stmts] + [{"type": "close"}]}
20
- r = httpx.post(turso_url(), json=body, headers=headers, timeout=120)
21
- r.raise_for_status()
22
- return r.json()
23
-
24
- def main():
25
- db_path = sys.argv[1] if len(sys.argv) > 1 else r"C:\Users\jyeal\Documents\programming-related\Pixif\Client\db.sqlite"
26
- conn = sqlite3.connect(db_path)
27
-
28
- turso_post([
29
- {"sql": "CREATE TABLE IF NOT EXISTS searches (id TEXT PRIMARY KEY, query TEXT, post_ids TEXT, created_at INTEGER)"},
30
- {"sql": "CREATE TABLE IF NOT EXISTS scans (post_id TEXT PRIMARY KEY, url TEXT, exif_type INTEGER)"},
31
- ])
32
-
33
- rows = conn.execute("SELECT post_id, url, exif_type FROM pixif_cache").fetchall()
34
- print(f"Found {len(rows)} rows in local db")
35
-
36
- stmts = []
37
- for post_id, url, exif_type in rows:
38
- short_url = url.replace(IMG_BASE, "", 1) if url else ""
39
- args = [
40
- {"type": "text", "value": str(post_id)},
41
- {"type": "text", "value": short_url},
42
- ]
43
- if exif_type is not None:
44
- args.append({"type": "integer", "value": str(exif_type)})
45
- else:
46
- args.append({"type": "null"})
47
- stmts.append({
48
- "sql": "INSERT OR IGNORE INTO scans (post_id, url, exif_type) VALUES (?, ?, ?)",
49
- "args": args,
50
- })
51
-
52
- uploaded = 0
53
- for i in range(0, len(stmts), BATCH_SIZE):
54
- batch = stmts[i:i+BATCH_SIZE]
55
- turso_post(batch)
56
- uploaded += len(batch)
57
- print(f" {uploaded}/{len(stmts)}")
58
-
59
- print(f"Done. Uploaded {len(stmts)} rows to Turso.")
60
- conn.close()
61
-
62
- if __name__ == "__main__":
63
- main()