Phase 3: near-duplicate (dHash) rejection on contributed photos

game/datastore.py

@@ -96,17 +96,57 @@ def load_contributors() -> list[dict]:
         return []
 
 
-def append_sighting(image, user_label: str, machine: dict, contributor: str) -> bool:
+# Near-duplicate detection via perceptual hash (dHash). Catches the same photo
+# re-saved/resized/re-compressed, not just byte-identical files — which is the
+# realistic abuse/pollution case (re-uploading a popular web image, gaming the
+# contributor board, accidental double-submits). PIL + numpy only, no new dep.
+DUP_HAMMING_THRESHOLD = 5   # <=5 of 64 bits differ => treat as the same image
+
+
+def compute_phash(image) -> str:
+    """64-bit dHash as a 16-char hex string (row->row horizontal gradient)."""
+    import numpy as np
+    from PIL import Image
+    small = image.convert("L").resize((9, 8), Image.BILINEAR)
+    a = np.asarray(small, dtype=np.int16)
+    bits = (a[:, 1:] > a[:, :-1]).flatten()           # 8x8 = 64 bits
+    val = 0
+    for b in bits:
+        val = (val << 1) | int(b)
+    return f"{val:016x}"
+
+
+def _hamming(a: str, b: str) -> int:
+    return bin(int(a, 16) ^ int(b, 16)).count("1")
+
+
+def append_sighting(image, user_label: str, machine: dict, contributor: str) -> str:
     """
-    Phase 2: commit one contributed photo + metadata row to the dataset.
-    `machine` carries the model's call (prediction/confidence/abstained/safety/domain).
-    Returns True on persisted write. Stub raises if no token so callers gate on
-    persistence_enabled() first.
+    Commit one contributed photo + metadata row to the dataset, with near-dup
+    rejection. `machine` carries the model's call.
+    Returns: "stored" | "duplicate" | "disabled".
     """
     if not _TOKEN:
-        return False
+        return "disabled"
     from io import BytesIO
-    from huggingface_hub import CommitOperationAdd, HfApi
+    from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download
+
+    # read existing metadata (+ its phashes) once
+    existing, rows = "", []
+    try:
+        with open(hf_hub_download(DATASET_REPO, "metadata.jsonl", repo_type="dataset",
+                                  token=_TOKEN, force_download=True)) as f:
+            txt = f.read()
+        existing = txt.rstrip("\n")
+        rows = [json.loads(line) for line in txt.splitlines() if line.strip()]
+    except Exception:
+        pass
+
+    ph = compute_phash(image)
+    for r in rows:
+        h = r.get("phash")
+        if h and _hamming(h, ph) <= DUP_HAMMING_THRESHOLD:
+            return "duplicate"
 
     ts = _now()
     fname = f"images/{contributor}_{ts.replace(':', '').replace('-', '')}.jpg"
@@ -119,20 +159,11 @@ def append_sighting(image, user_label: str, machine: dict, contributor: str) ->
         "machine_abstained": bool(machine.get("abstained", True)),
         "machine_safety": machine.get("safety", "UNKNOWN"),
         "routed_domain": machine.get("domain", "unknown"),
-        "contributor": contributor, "consent": True, "license": LICENSE, "timestamp": ts,
+        "contributor": contributor, "consent": True, "license": LICENSE,
+        "timestamp": ts, "phash": ph,
     }
-    api = HfApi(token=_TOKEN)
-    # append the metadata line to the existing jsonl
-    existing = ""
-    try:
-        from huggingface_hub import hf_hub_download
-        with open(hf_hub_download(DATASET_REPO, "metadata.jsonl", repo_type="dataset",
-                                  token=_TOKEN, force_download=True)) as f:
-            existing = f.read().rstrip("\n")
-    except Exception:
-        pass
     new_meta = (existing + "\n" if existing else "") + json.dumps(row) + "\n"
-    api.create_commit(
+    HfApi(token=_TOKEN).create_commit(
         repo_id=DATASET_REPO, repo_type="dataset",
         commit_message=f"sighting: {row['machine_prediction']} by {contributor}",
         operations=[
@@ -141,4 +172,4 @@ def append_sighting(image, user_label: str, machine: dict, contributor: str) ->
                                path_or_fileobj=new_meta.encode("utf-8")),
         ],
     )
-    return True
+    return "stored"