File size: 3,480 Bytes
de1e3fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | """Quarantine the flagged outlier photos: move each file to its own folder and delete its DB rows.
Reads eval_out/photo_consistency_by_dog.csv, takes every row with removable_outlier == True (a dog
whose MIN pairwise similarity is < the threshold AND is lifted back above it by dropping one
isolated photo), and for each such photo:
* moves the media file to data/outlier_photos/ (renamed descriptively), then
* deletes its BreedPrediction + Embedding + Picture rows.
The photos are only ever attached to dogs that still have >=2 photos afterwards, so no dog is
emptied. Files are moved (not deleted), so this is reversible. Idempotent: already-moved files are
skipped.
Usage (from backend/):
python -m scripts.remove_outliers # do it
python -m scripts.remove_outliers --dry-run # just report
"""
from __future__ import annotations
import argparse
import csv
import shutil
from pathlib import Path
from sqlalchemy import delete, select
from app.db import SessionLocal
from app.models import BreedPrediction, Embedding, Picture
from app.storage import get_storage
CSV = Path("eval_out/photo_consistency_by_dog.csv")
QUARANTINE = Path("data/outlier_photos")
def run(dry_run: bool) -> None:
if not CSV.exists():
raise SystemExit(f"{CSV} not found — run scripts.eval_photo_consistency first.")
rows = [r for r in csv.DictReader(CSV.open()) if r["removable_outlier"] == "True"]
print(f"{len(rows)} outlier photo(s) flagged for removal.")
QUARANTINE.mkdir(parents=True, exist_ok=True)
storage = get_storage()
db = SessionLocal()
moved = deleted = skipped = 0
try:
for r in rows:
key = r["most_isolated_file"]
pic = db.execute(select(Picture).where(Picture.file_path == key)).scalar_one_or_none()
src = storage.abs_path(key)
if pic is None or src is None or not Path(src).exists():
print(f" skip (already gone): {key}")
skipped += 1
continue
safe_ident = str(r["identity"]).replace("/", "_")
dest = QUARANTINE / f"{r['dataset']}__{safe_ident}__{Path(key).name}"
print(f" {r['dataset']}/{r['identity']} min {r['min']}->{r['min_after_removing_isolated']} "
f"{key} -> {dest}")
if dry_run:
continue
shutil.move(str(src), str(dest))
db.execute(delete(BreedPrediction).where(BreedPrediction.picture_id == pic.id))
db.execute(delete(Embedding).where(Embedding.picture_id == pic.id))
db.execute(delete(Picture).where(Picture.id == pic.id))
moved += 1
if not dry_run:
db.commit()
# count remaining outlier files in the DB-referenced media just for a sanity line
print(f"\n{'DRY RUN — nothing changed.' if dry_run else 'Done.'} "
f"moved+deleted={moved}, skipped={skipped}")
print(f"Quarantine folder: {QUARANTINE.resolve()}")
except Exception:
db.rollback()
raise
finally:
db.close()
def main() -> None:
ap = argparse.ArgumentParser(description="Quarantine flagged outlier photos.")
ap.add_argument("--dry-run", action="store_true")
ap.add_argument("--csv", default=None, help="Override the consistency CSV path")
args = ap.parse_args()
global CSV
if args.csv:
CSV = Path(args.csv)
run(args.dry_run)
if __name__ == "__main__":
main()
|