Update main.py

main.py

@@ -4,7 +4,7 @@
 # Optional envs: GAME_SALT, ADMIN_KEY, IA_USER_AGENT, MIN_IA_POOL, IA_QUERY,
 #                BOOTSTRAP_IA, LOG_LEVEL, ALLOW_DEV_BOOTSTRAP, ALLOW_DEV_DIAGNOSTICS
 
-import os, io, uuid, json, hmac, hashlib, random, traceback, requests
+import os, io, uuid, json, hmac, hashlib, random, traceback, requests, re, hashlib as _hash
 from datetime import datetime, timedelta, timezone
 from typing import Dict, Any, Tuple, List, Optional
 
@@ -55,7 +55,7 @@ try:
     bucket = storage.bucket()
     db_root = db.reference("/")
     log.info("Firebase Realtime DB + Storage initialized.")
-except Exception as e:
+except Exception:
     log.exception("FATAL: Firebase init failed")
     raise
 
@@ -66,7 +66,7 @@ try:
         raise ValueError("The 'Gemini' environment variable is not set.")
     client = genai.Client(api_key=GEMINI_API_KEY)
     log.info("Gemini client initialized.")
-except Exception as e:
+except Exception:
     log.exception("FATAL: Gemini init failed")
     raise
 
@@ -93,11 +93,8 @@ ALLOW_DEV_BOOTSTRAP = os.environ.get("ALLOW_DEV_BOOTSTRAP", "0") == "1"
 ALLOW_DEV_DIAGNOSTICS = os.environ.get("ALLOW_DEV_DIAGNOSTICS", "0") == "1"
 
 FALLBACK_IA_QUERIES = [
-    # 1) broad (fastest win)
     '(mediatype:image AND (format:JPEG OR format:PNG))',
-    # 2) portraits bias (nice for the game)
     '(mediatype:image AND (format:JPEG OR format:PNG) AND (subject:portrait OR title:portrait))',
-    # 3) slightly narrower but still broad
     '(mediatype:image AND format:JPEG)',
 ]
 
@@ -125,6 +122,18 @@ def ia_pool_ref():
 def hmac_hex(s: str) -> str:
     return hmac.new(GAME_SALT.encode(), s.encode(), hashlib.sha256).hexdigest()
 
+# Firebase RTDB key sanitizer (no . $ # [ ] / or control chars)
+_FB_BAD = re.compile(r'[.$#\[\]/\x00-\x1F\x7F]')
+
+def fb_key(raw: str) -> str:
+    safe = _FB_BAD.sub('_', raw or '')
+    if len(safe) > 700:
+        safe = safe[:700]
+    if safe != raw:
+        suffix = _hash.sha1((raw or '').encode('utf-8')).hexdigest()[:8]
+        safe = f"{safe}__{suffix}"
+    return safe or _hash.sha1(b'empty').hexdigest()[:8]
+
 def upload_bytes_to_storage(data: bytes, path: str, content_type: str) -> str:
     log.debug(f"Uploading to Storage: path={path}, content_type={content_type}, bytes={len(data)}")
     blob = bucket.blob(path)
@@ -214,11 +223,12 @@ def ia_best_image_from_metadata(meta: dict) -> Optional[dict]:
     return best
 
 def ingest_ia_doc(doc: dict) -> Optional[dict]:
-    """Fetch /metadata and store best image entry into ia_pool."""
+    """Fetch /metadata and store best image entry into ia_pool (sanitized key)."""
     identifier = doc.get("identifier")
     if not identifier:
         return None
-    log.info(f"Ingesting IA identifier={identifier}")
+    pool_key = fb_key(identifier)
+    log.info(f"Ingesting IA identifier={identifier} -> pool_key={pool_key}")
     meta = ia_metadata(identifier)
     best = ia_best_image_from_metadata(meta)
     if not best:
@@ -234,7 +244,8 @@ def ingest_ia_doc(doc: dict) -> Optional[dict]:
 
     download_url = f"https://archive.org/download/{identifier}/{best['name']}"
     record = {
-        "identifier": identifier,
+        "identifier": identifier,       # original IA id preserved
+        "_pool_key": pool_key,          # sanitized RTDB key
         "title": title,
         "date": str(date),
         "creator": creator,
@@ -248,8 +259,8 @@ def ingest_ia_doc(doc: dict) -> Optional[dict]:
         "size": best.get("size"),
         "source": "internet_archive"
     }
-    ia_pool_ref().child(identifier).set(record)
-    log.info(f"Ingested {identifier} -> ia_pool (title='{title}')")
+    ia_pool_ref().child(pool_key).set(record)
+    log.info(f"Ingested {identifier} -> ia_pool/{pool_key} (title='{title}')")
     return record
 
 def choose_ia_item_for_case(case_id: str) -> Optional[dict]:
@@ -257,11 +268,11 @@ def choose_ia_item_for_case(case_id: str) -> Optional[dict]:
     if not pool:
         log.warning("choose_ia_item_for_case: pool is empty")
         return None
-    identifiers = sorted(pool.keys())
+    keys = sorted(pool.keys())
     case_seed = seed_for_date(case_id)
-    ident = identifiers[case_seed % len(identifiers)]
-    log.info(f"Chosen IA item for case {case_id}: {ident}")
-    return pool[ident]
+    pool_key = keys[case_seed % len(keys)]
+    log.info(f"Chosen IA pool_key for case {case_id}: {pool_key}")
+    return pool[pool_key]
 
 def download_image_to_pil(url: str) -> Image.Image:
     data = http_get_bytes(url)
@@ -295,30 +306,31 @@ def _resize_if_needed(img: Image.Image, max_dim: int = 4096) -> Image.Image:
     return img.resize((new_w, new_h), Image.LANCZOS)
 
 def cache_single_ia_identifier(
-    identifier: str,
+    pool_key: str,
     overwrite: bool = False,
     max_dim: int = 4096,
     jpeg_quality: int = 90,
     skip_if_restricted: bool = True,
 ) -> dict:
-    rec_ref = ia_pool_ref().child(identifier)
+    rec_ref = ia_pool_ref().child(pool_key)
     rec = rec_ref.get() or {}
     if not rec:
-        return {"identifier": identifier, "stored": False, "reason": "not_in_pool"}
+        return {"pool_key": pool_key, "stored": False, "reason": "not_in_pool"}
 
+    identifier = rec.get("identifier") or pool_key
     rights = (rec.get("rights") or "").lower()
     if skip_if_restricted and ("in copyright" in rights or "all rights reserved" in rights):
         log.info(f"Skipping {identifier}: restricted rights")
-        return {"identifier": identifier, "stored": False, "reason": "restricted_rights"}
+        return {"pool_key": pool_key, "stored": False, "reason": "restricted_rights"}
 
     if rec.get("storage_url") and not overwrite:
         log.info(f"Skipping {identifier}: already cached")
-        return {"identifier": identifier, "stored": False, "reason": "already_cached", "storage_url": rec["storage_url"]}
+        return {"pool_key": pool_key, "stored": False, "reason": "already_cached", "storage_url": rec["storage_url"]}
 
     source_url = rec.get("storage_url") or rec.get("download_url")
     if not source_url:
         log.warning(f"{identifier}: missing source_url")
-        return {"identifier": identifier, "stored": False, "reason": "missing_source_url"}
+        return {"pool_key": pool_key, "stored": False, "reason": "missing_source_url"}
 
     try:
         log.info(f"Caching {identifier} from {source_url}")
@@ -330,10 +342,10 @@ def cache_single_ia_identifier(
                 img = download_image_to_pil(rec["download_url"])
             except Exception as e2:
                 log.exception(f"{identifier}: download failed")
-                return {"identifier": identifier, "stored": False, "reason": f"download_failed: {e2}"}
+                return {"pool_key": pool_key, "stored": False, "reason": f"download_failed: {e2}"}
         else:
             log.exception(f"{identifier}: download failed")
-            return {"identifier": identifier, "stored": False, "reason": f"download_failed: {e}"}
+            return {"pool_key": pool_key, "stored": False, "reason": f"download_failed: {e}"}
 
     img = _resize_if_needed(img, max_dim=max_dim)
     w, h = img.size
@@ -342,7 +354,7 @@ def cache_single_ia_identifier(
     img_bytes = io.BytesIO()
     img.save(img_bytes, format="JPEG", quality=jpeg_quality, optimize=True)
     img_bytes.seek(0)
-    img_path = f"ia_cache/{identifier}/original.jpg"
+    img_path = f"ia_cache/{pool_key}/original.jpg"
     storage_url = upload_bytes_to_storage(img_bytes.getvalue(), img_path, "image/jpeg")
 
     # Upload macro crop
@@ -350,7 +362,7 @@ def cache_single_ia_identifier(
     crop_bytes = io.BytesIO()
     crop.save(crop_bytes, format="JPEG", quality=jpeg_quality, optimize=True)
     crop_bytes.seek(0)
-    crop_path = f"ia_cache/{identifier}/signature_crop.jpg"
+    crop_path = f"ia_cache/{pool_key}/signature_crop.jpg"
     signature_crop_url = upload_bytes_to_storage(crop_bytes.getvalue(), crop_path, "image/jpeg")
 
     rec_update = {
@@ -366,7 +378,7 @@ def cache_single_ia_identifier(
     log.info(f"Cached {identifier} -> {storage_url}")
 
     return {
-        "identifier": identifier,
+        "pool_key": pool_key,
         "stored": True,
         "storage_url": storage_url,
         "signature_crop_url": signature_crop_url,
@@ -390,14 +402,14 @@ def batch_cache_ia_pool(
         return {"ok": True, "processed": 0, "stored": 0, "skipped": 0, "results": []}
 
     candidates = []
-    for ident, rec in pool.items():
+    for pkey, rec in pool.items():
         if overwrite or not rec.get("storage_url"):
             w = int(rec.get("width") or 0)
             h = int(rec.get("height") or 0)
             if (w and h) and (w < min_width or h < min_height):
-                log.debug(f"Skip {ident}: too small {w}x{h}")
+                log.debug(f"Skip {pkey}: too small {w}x{h}")
                 continue
-            candidates.append(ident)
+            candidates.append(pkey)
 
     if randomize:
         random.shuffle(candidates)
@@ -405,9 +417,9 @@ def batch_cache_ia_pool(
     log.info(f"Caching candidates: {len(candidates)} (limit={limit})")
 
     results, stored, skipped = [], 0, 0
-    for ident in candidates:
+    for pkey in candidates:
         res = cache_single_ia_identifier(
-            ident,
+            pkey,
             overwrite=overwrite,
             max_dim=max_dim,
             jpeg_quality=jpeg_quality,
@@ -429,13 +441,11 @@ def ensure_minimum_ia_pool(min_items: int = MIN_IA_POOL, rows: int = 100, max_pa
     cached = 0
     log.info(f"ensure_minimum_ia_pool: have={have}, target={min_items}")
 
-    # Decide which queries to try: env IA_QUERY first, then fallbacks
     candidate_queries = []
     if DEFAULT_IA_QUERY:
         candidate_queries.append(DEFAULT_IA_QUERY)
     candidate_queries.extend([q for q in FALLBACK_IA_QUERIES if q not in candidate_queries])
 
-    # Ingest until we reach the target or run out of queries/pages
     for q in candidate_queries:
         if have + added >= min_items:
             break
@@ -454,7 +464,7 @@ def ensure_minimum_ia_pool(min_items: int = MIN_IA_POOL, rows: int = 100, max_pa
                 ident = d.get("identifier")
                 if not ident:
                     continue
-                if ia_pool_ref().child(ident).get():
+                if ia_pool_ref().child(fb_key(ident)).get():
                     continue
                 try:
                     rec = ingest_ia_doc(d)
@@ -467,7 +477,6 @@ def ensure_minimum_ia_pool(min_items: int = MIN_IA_POOL, rows: int = 100, max_pa
                     break
             page += 1
 
-    # Cache up to min_items (unchanged)
     pool = ia_pool_ref().get() or {}
     have_now = len(pool)
     need_cache = max(0, min_items - have_now)
@@ -481,7 +490,6 @@ def ensure_minimum_ia_pool(min_items: int = MIN_IA_POOL, rows: int = 100, max_pa
     log.info(f"ensure_minimum_ia_pool: stats={stats}")
     return stats
 
-
 # -----------------------------------------------------------------------------
 # 4) CASE GENERATION (uses IA for authentic image, Gemini for forgeries/meta)
 # -----------------------------------------------------------------------------
@@ -498,19 +506,16 @@ def ensure_case_generated(case_id: str) -> Dict[str, Any]:
     except Exception:
         log.exception("Bootstrap failed inside ensure_case_generated")
 
-    # Pick authentic from ia_pool deterministically
     ia_item = choose_ia_item_for_case(case_id)
     if not ia_item:
         raise RuntimeError("No IA items available. Ingest needed.")
 
-    # Deterministic mode
     case_seed = seed_for_date(case_id)
     mode = "knowledge" if (case_seed % 2 == 0) else "observation"
     log.info(f"Case {case_id}: mode={mode}")
 
     style_period = "sourced from Internet Archive; museum catalog reproduction"
 
-    # Load authentic image (prefer cached)
     source_url = ia_item.get("storage_url") or ia_item["download_url"]
     log.info(f"Case {case_id}: authentic source={source_url}")
     auth_img = download_image_to_pil(source_url)
@@ -518,19 +523,17 @@ def ensure_case_generated(case_id: str) -> Dict[str, Any]:
     images_urls: List[str] = []
     signature_crops: List[str] = []
 
-    # Save authentic as image #1
     url1 = save_image_return_url(auth_img, f"hidden_stroke/{case_id}/images/img_1.jpg")
     images_urls.append(url1)
     log.debug(f"Case {case_id}: saved authentic -> {url1}")
 
-    # Macro crop for signature area
     crop1 = crop_signature_macro(auth_img, 512)
     crop1_url = save_image_return_url(crop1, f"hidden_stroke/{case_id}/signature_crops/crop_1.jpg", quality=88)
     signature_crops.append(crop1_url)
     log.debug(f"Case {case_id}: saved authentic crop -> {crop1_url}")
 
     if mode == "knowledge":
-        for idx in [2, 3]:
+        for _ in [2, 3]:
             images_urls.append(images_urls[0])
             signature_crops.append(signature_crops[0])
     else:
@@ -563,7 +566,6 @@ No annotations. Differences must be visible only at macro zoom.
             signature_crops.append(c_url)
             log.debug(f"Case {case_id}: forgery saved -> {url}; crop -> {c_url}")
 
-    # === Gemini: Case brief + metadata + ledger + solution ===
     title = ia_item.get("title") or "Untitled"
     creator = ia_item.get("creator") or ""
     date = ia_item.get("date") or ""
@@ -780,7 +782,7 @@ def admin_ingest_ia():
             ident = d.get("identifier")
             if not ident:
                 continue
-            if ia_pool_ref().child(ident).get():
+            if ia_pool_ref().child(fb_key(ident)).get():
                 continue
             try:
                 rec = ingest_ia_doc(d)
@@ -858,7 +860,6 @@ def admin_bootstrap_now():
     finally:
         DEFAULT_IA_QUERY = original_q  # restore
 
-
 # --- DEV-ONLY: diagnostics (network + firebase sanity) ---
 @app.route("/admin/diagnostics", methods=["GET"])
 def diagnostics():
@@ -883,7 +884,6 @@ def diagnostics():
     except Exception as e:
         diag["ia"]["error"] = str(e)
 
-    # Try a tiny upload
     try:
         tiny = upload_bytes_to_storage(b"ping", f"diag/ping_{uuid.uuid4().hex}.txt", "text/plain")
         diag["firebase"]["upload_test"] = tiny
@@ -899,7 +899,6 @@ def start_case():
     case_id = utc_today_str()
     public = ensure_case_generated(case_id)
 
-    # Create/reuse an active session for this user+case
     existing = sessions_ref().order_by_child("user_id").equal_to(user_id).get()
     sess = None
     if existing: