refactor: migrate user and gallery data storage from JSON files to MongoDB

- Updated auth.py to load and save users from/to MongoDB instead of a JSON file.
- Refactored gallery.py to manage gallery entries in MongoDB, removing local file dependencies.
- Introduced mongo.py for MongoDB connection handling and index management.
- Removed obsolete JSON data files for users and gallery entries.
- Updated create_user.py to create users directly in MongoDB.

This change enhances data persistence and scalability by leveraging MongoDB.

backend/app/auth.py

@@ -1,42 +1,40 @@
 """
-Simple auth: users in JSON file, JWT for sessions.
+Simple auth: users in MongoDB, JWT for sessions.
 Use scripts/create_user.py to add users.
 """
 
-import json
 import os
-from pathlib import Path
 
 import bcrypt
 import jwt
 from fastapi import Depends, HTTPException, status
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from app.mongo import get_mongo_db
 
-# Users file next to backend app (backend/data/users.json). On Hugging Face use DATA_DIR=/tmp/amalfa-data
-_data_dir = os.environ.get("DATA_DIR")
-DATA_DIR = Path(_data_dir) if _data_dir else Path(__file__).resolve().parent.parent / "data"
-USERS_PATH = DATA_DIR / "users.json"
-DATA_DIR.mkdir(parents=True, exist_ok=True)
 JWT_ALGORITHM = "HS256"
 JWT_SECRET = os.environ.get("JWT_SECRET", "amalfa-dev-secret-change-in-production")
 TOKEN_EXPIRE_HOURS = 24 * 7  # 7 days
+USERS_COLLECTION = "users"
 
 security = HTTPBearer(auto_error=False)
 
 
 def _load_users() -> list[dict]:
-    if not USERS_PATH.exists():
-        return []
-    try:
-        with open(USERS_PATH, "r") as f:
-            return json.load(f)
-    except (json.JSONDecodeError, OSError):
+    db = get_mongo_db()
+    if db is None:
         return []
+    users = list(db[USERS_COLLECTION].find({}, {"_id": 0}))
+    return users
 
 
 def _save_users(users: list[dict]) -> None:
-    with open(USERS_PATH, "w") as f:
-        json.dump(users, f, indent=2)
+    db = get_mongo_db()
+    if db is None:
+        return
+    coll = db[USERS_COLLECTION]
+    coll.delete_many({})
+    if users:
+        coll.insert_many(users)
 
 
 def verify_password(plain: str, hashed: str) -> bool:
@@ -51,9 +49,16 @@ def hash_password(plain: str) -> str:
 
 
 def get_user_by_username(username: str) -> dict | None:
+    db = get_mongo_db()
+    normalized = (username or "").strip().lower()
+    if db is None:
+        return None
+    doc = db[USERS_COLLECTION].find_one({"username_lower": normalized}, {"_id": 0})
+    if doc:
+        return doc
     users = _load_users()
     for u in users:
-        if (u.get("username") or "").strip().lower() == username.strip().lower():
+        if (u.get("username") or "").strip().lower() == normalized:
             return u
     return None
 

backend/app/gallery.py

@@ -1,90 +1,44 @@
 """
-Gallery store: persisted list of generated creatives per user (JSON file per user).
-Uses DATA_DIR/gallery when DATA_DIR env is set (e.g. Hugging Face: /tmp/amalfa-data).
-When R2 is configured, gallery index is also stored in R2 (gallery/{username}.json) so it
-survives Hugging Face redeploys where /tmp is wiped.
+Gallery store: persisted list of generated creatives per user in MongoDB.
 """
 
-import json
-import os
-import re
 import uuid
 from datetime import datetime, timezone, timedelta
-from pathlib import Path
 from typing import Any
 
-from app.r2 import delete_object as r2_delete_object, get_object as r2_get_object, put_object as r2_put_object
-
-_data_dir = os.environ.get("DATA_DIR")
-_base = Path(_data_dir) if _data_dir else Path(__file__).resolve().parent.parent / "data"
-GALLERY_DIR = _base / "gallery"
-GALLERY_DIR.mkdir(parents=True, exist_ok=True)
-
-# Safe filename: alphanumeric and underscore only
-USERNAME_SAFE = re.compile(r"[^a-zA-Z0-9_]")
+from app.mongo import get_mongo_db
+from app.r2 import delete_object as r2_delete_object
 DEFAULT_LIMIT = 100
 MAX_LIMIT = 500
-R2_GALLERY_PREFIX = "gallery/"
-
-
-def _safe_username(username: str) -> str:
-    return USERNAME_SAFE.sub("_", (username or "").strip()) or "default"
-
-
-def _user_path(username: str) -> Path:
-    return GALLERY_DIR / f"{_safe_username(username)}.json"
-
+GALLERY_COLLECTION = "gallery_entries"
 
-def _r2_gallery_key(username: str) -> str:
-    return f"{R2_GALLERY_PREFIX}{_safe_username(username)}.json"
-
-
-def _load_from_r2(username: str) -> list[dict[str, Any]] | None:
-    """Load gallery entries from R2. Returns None if R2 not configured or key missing."""
-    raw = r2_get_object(_r2_gallery_key(username))
-    if raw is None:
-        return None
-    try:
-        data = json.loads(raw.decode("utf-8"))
-        return data if isinstance(data, list) else None
-    except (json.JSONDecodeError, UnicodeDecodeError):
+def _load_from_mongo(username: str) -> list[dict[str, Any]] | None:
+    db = get_mongo_db()
+    if db is None:
         return None
+    docs = list(
+        db[GALLERY_COLLECTION]
+        .find({"username": username}, {"_id": 0})
+        .sort("created_at", -1)
+    )
+    return docs
 
 
-def _save_to_r2(username: str, entries: list[dict[str, Any]]) -> bool:
-    """Persist gallery entries to R2. Returns True if written."""
-    body = json.dumps(entries, indent=2).encode("utf-8")
-    return r2_put_object(_r2_gallery_key(username), body)
+def _save_to_mongo(username: str, entries: list[dict[str, Any]]) -> bool:
+    db = get_mongo_db()
+    if db is None:
+        return False
+    coll = db[GALLERY_COLLECTION]
+    coll.delete_many({"username": username})
+    if entries:
+        coll.insert_many(entries)
+    return True
 
 
 def load_entries(username: str, limit: int = DEFAULT_LIMIT, offset: int = 0) -> tuple[list[dict[str, Any]], int]:
-    """Load gallery entries for user, newest first. Returns (page_slice, total_count). Uses R2 and local; prefers whichever has more entries (never write on read)."""
-    entries: list[dict[str, Any]] = []
-    path = _user_path(username)
-    from_r2 = _load_from_r2(username)
-    local_list: list[dict[str, Any]] = []
-    if path.exists():
-        try:
-            with open(path, "r") as f:
-                raw = json.load(f)
-            local_list = raw if isinstance(raw, list) else []
-        except (json.JSONDecodeError, OSError):
-            pass
-    if from_r2 is not None:
-        # Use the source with more entries so stale R2 (e.g. one item) doesn't hide a full local gallery.
-        if len(local_list) > len(from_r2):
-            entries = local_list
-            _save_to_r2(username, entries)  # repair R2
-        else:
-            entries = from_r2
-        if len(entries) == 0 and len(local_list) > 0:
-            entries = local_list
-            _save_to_r2(username, entries)
-    else:
-        entries = local_list
-        if local_list:
-            _save_to_r2(username, entries)  # initial sync: local has data but R2 key is absent
-    entries = sorted(entries, key=lambda e: e.get("created_at", ""), reverse=True)
+    """Load gallery entries for user, newest first. Returns (page_slice, total_count)."""
+    from_mongo = _load_from_mongo(username)
+    entries: list[dict[str, Any]] = from_mongo or []
     total = len(entries)
     page = entries[offset : offset + min(limit, MAX_LIMIT)]
     return (page, total)
@@ -100,18 +54,8 @@ def append_entry(
     scene_prompt: str | None = None,
     image_model: str | None = None,
 ) -> dict[str, Any]:
-    """Append a gallery entry. Writes to local file and R2 (when configured). Returns the new entry."""
+    """Append a gallery entry in MongoDB. Returns the new entry."""
     import datetime
-    path = _user_path(username)
-    entries = []
-    if path.exists():
-        try:
-            with open(path, "r") as f:
-                entries = json.load(f)
-            if not isinstance(entries, list):
-                entries = []
-        except (json.JSONDecodeError, OSError):
-            entries = []
     now = datetime.datetime.now(datetime.timezone.utc).isoformat()
     entry = {
         "id": str(uuid.uuid4()),
@@ -126,10 +70,10 @@ def append_entry(
         entry["scene_prompt"] = scene_prompt
     if image_model is not None:
         entry["image_model"] = image_model
-    entries.append(entry)
-    with open(path, "w") as f:
-        json.dump(entries, f, indent=2)
-    _save_to_r2(username, entries)
+    db = get_mongo_db()
+    if db is None:
+        raise RuntimeError("MongoDB is not configured")
+    db[GALLERY_COLLECTION].insert_one(entry)
     return entry
 
 
@@ -139,104 +83,46 @@ def update_entry(
     **updates: Any,
 ) -> bool:
     """Update fields of an existing gallery entry. Returns True if updated."""
-    path = _user_path(username)
-    if not path.exists():
-        from_r2 = _load_from_r2(username)
-        if from_r2 is None:
-            return False
-        entries = from_r2
-    else:
-        try:
-            with open(path, "r") as f:
-                entries = json.load(f)
-        except (json.JSONDecodeError, OSError):
-            return False
-    if not isinstance(entries, list):
-        return False
     allowed = {"r2_key", "concept_name", "creative_id", "product_name", "scene_prompt", "image_model"}
     for key in list(updates.keys()):
         if key not in allowed:
             del updates[key]
     if not updates:
         return True
-    found = False
-    for e in entries:
-        if e.get("id") == entry_id:
-            e.update(updates)
-            found = True
-            break
-    if not found:
-        return False
-    with open(path, "w") as f:
-        json.dump(entries, f, indent=2)
-    _save_to_r2(username, entries)
-    return True
+    db = get_mongo_db()
+    if db is not None:
+        result = db[GALLERY_COLLECTION].update_one(
+            {"username": username, "id": entry_id},
+            {"$set": updates},
+        )
+        return result.matched_count > 0
+    return False
 
 
 def delete_entry(username: str, entry_id: str) -> bool:
-    """Remove entry by id. Writes to local file and R2 (when configured). Returns True if removed."""
-    path = _user_path(username)
-    if not path.exists():
-        from_r2 = _load_from_r2(username)
-        if from_r2 is None:
-            return False
-        entries = from_r2
-    else:
-        try:
-            with open(path, "r") as f:
-                entries = json.load(f)
-        except (json.JSONDecodeError, OSError):
-            return False
-    if not isinstance(entries, list):
-        return False
-    orig_len = len(entries)
-    entries = [e for e in entries if e.get("id") != entry_id]
-    if len(entries) == orig_len:
-        return False
-    with open(path, "w") as f:
-        json.dump(entries, f, indent=2)
-    _save_to_r2(username, entries)
-    return True
+    """Remove entry by id from MongoDB."""
+    db = get_mongo_db()
+    if db is not None:
+        result = db[GALLERY_COLLECTION].delete_one({"username": username, "id": entry_id})
+        return result.deleted_count > 0
+    return False
 
 
 def get_entry(username: str, entry_id: str) -> dict[str, Any] | None:
-    """Get a single entry by id (for presigned URL / delete check). Uses R2 when local file missing."""
-    entries: list[dict[str, Any]] = []
-    path = _user_path(username)
-    if path.exists():
-        try:
-            with open(path, "r") as f:
-                raw = json.load(f)
-            entries = raw if isinstance(raw, list) else []
-        except (json.JSONDecodeError, OSError):
-            pass
-    else:
-        from_r2 = _load_from_r2(username)
-        if from_r2 is not None:
-            entries = from_r2
-    for e in entries:
-        if e.get("id") == entry_id:
-            return e
+    """Get a single entry by id."""
+    db = get_mongo_db()
+    if db is not None:
+        return db[GALLERY_COLLECTION].find_one(
+            {"username": username, "id": entry_id},
+            {"_id": 0},
+        )
     return None
 
 
 def _load_all_entries_for_user(username: str) -> list[dict[str, Any]]:
-    """Load full list of gallery entries for a user (local + R2 merge, same logic as load_entries)."""
-    path = _user_path(username)
-    from_r2 = _load_from_r2(username)
-    local_list: list[dict[str, Any]] = []
-    if path.exists():
-        try:
-            with open(path, "r") as f:
-                raw = json.load(f)
-            local_list = raw if isinstance(raw, list) else []
-        except (json.JSONDecodeError, OSError):
-            pass
-    if from_r2 is not None:
-        if len(local_list) > len(from_r2):
-            return local_list
-        return from_r2
-    return local_list
+    """Load full list of gallery entries for a user."""
+    from_mongo = _load_from_mongo(username)
+    return from_mongo or []
 
 
 def delete_entries_older_than_days(
@@ -263,9 +149,10 @@ def delete_entries_older_than_days(
             return None
 
     if username is not None:
-        usernames = [_safe_username(username)]
+        usernames = [username]
     else:
-        usernames = [p.stem for p in GALLERY_DIR.glob("*.json") if p.is_file()]
+        db = get_mongo_db()
+        usernames = db[GALLERY_COLLECTION].distinct("username") if db is not None else []
 
     for uname in usernames:
         entries = _load_all_entries_for_user(uname)
@@ -287,11 +174,8 @@ def delete_entries_older_than_days(
                     errors.append(f"R2 delete failed: {r2_key}")
             deleted += 1
         if len(to_keep) < len(entries):
-            try:
-                with open(_user_path(uname), "w") as f:
-                    json.dump(to_keep, f, indent=2)
-                _save_to_r2(uname, to_keep)
-            except OSError as err:
-                errors.append(f"Save failed for {uname}: {err}")
+            saved_to_mongo = _save_to_mongo(uname, to_keep)
+            if not saved_to_mongo:
+                errors.append(f"Save failed for {uname}: MongoDB unavailable")
 
     return {"deleted": deleted, "users_processed": users_processed, "errors": errors}

backend/app/main.py

@@ -22,6 +22,7 @@ from app.analysis import analyze_product
 from app.analysis_flows import analyze_product_archetype, analyze_product_cross_vertical
 from app.auth import authenticate_user, create_access_token, get_current_user
 from app.creatives import generate_ad_creatives
+from app.mongo import ensure_mongo_indexes, mongo_is_configured, ping_mongo
 from app.gallery import (
     append_entry as gallery_append_entry,
     delete_entries_older_than_days as gallery_delete_entries_older_than_days,
@@ -65,6 +66,8 @@ _ENV_SPEC = [
     ("CANVA_CLIENT_SECRET", False, True),
     ("CANVA_REDIRECT_URI", False, False),
     ("CANVA_SCOPES", False, False),
+    ("MONGODB_URL", False, True),
+    ("MONGODB_DB_NAME", False, False),
 ]
 
 
@@ -195,17 +198,22 @@ def _run_gallery_cleanup():
 
 @app.on_event("startup")
 def _startup():
-    """Check env vars and ensure DATA_DIR/users on Hugging Face."""
+    """Check env vars and initialize MongoDB."""
     _check_env_on_startup()
-    # On Hugging Face (DATA_DIR set), copy default users.json into DATA_DIR if missing.
-    if os.environ.get("DATA_DIR"):
-        from app.auth import USERS_PATH
-        if not USERS_PATH.exists():
-            default = Path(__file__).resolve().parent.parent / "data" / "users.json"
-            if default.exists():
-                USERS_PATH.parent.mkdir(parents=True, exist_ok=True)
-                import shutil
-                shutil.copy(default, USERS_PATH)
+    log = logging.getLogger("uvicorn.error")
+    if mongo_is_configured():
+        ok, err = ping_mongo()
+        if ok:
+            log.info("MongoDB connection: ok")
+            idx_ok, idx_err = ensure_mongo_indexes()
+            if idx_ok:
+                log.info("MongoDB indexes: ensured")
+            else:
+                log.warning("MongoDB indexes ensure failed: %s", idx_err)
+        else:
+            log.warning("MongoDB configured but connection failed: %s", err)
+    else:
+        log.warning("MongoDB is not configured; auth and gallery will not function correctly")
     # Delete gallery entries (and R2 images) older than 60 days, in background
     import threading
     threading.Thread(target=_run_gallery_cleanup, daemon=True).start()

backend/app/mongo.py

@@ -0,0 +1,76 @@
+"""
+MongoDB connection helpers.
+
+Uses:
+- MONGODB_URL
+- MONGODB_DB_NAME
+"""
+
+import os
+from typing import Any
+
+from dotenv import load_dotenv
+from pymongo import MongoClient
+from pymongo.errors import PyMongoError
+from pymongo.database import Database
+
+load_dotenv()
+
+_MONGO_CLIENT: MongoClient[Any] | None = None
+
+
+def mongo_is_configured() -> bool:
+    url = (os.environ.get("MONGODB_URL") or "").strip()
+    db_name = (os.environ.get("MONGODB_DB_NAME") or "").strip()
+    return bool(url and db_name)
+
+
+def get_mongo_db() -> Database[Any] | None:
+    """Return Mongo database when configured; otherwise None."""
+    global _MONGO_CLIENT
+    if not mongo_is_configured():
+        return None
+    if _MONGO_CLIENT is None:
+        _MONGO_CLIENT = MongoClient(
+            (os.environ.get("MONGODB_URL") or "").strip(),
+            serverSelectionTimeoutMS=3000,
+        )
+    db_name = (os.environ.get("MONGODB_DB_NAME") or "").strip()
+    return _MONGO_CLIENT[db_name]
+
+
+def ping_mongo() -> tuple[bool, str | None]:
+    """Return (ok, error_message)."""
+    try:
+        db = get_mongo_db()
+        if db is None:
+            return (False, "MongoDB not configured")
+        db.command("ping")
+        return (True, None)
+    except Exception as exc:
+        return (False, str(exc))
+
+
+def ensure_mongo_indexes() -> tuple[bool, str | None]:
+    """
+    Create indexes used by auth and gallery access patterns.
+    Safe to call repeatedly on startup.
+    """
+    try:
+        db = get_mongo_db()
+        if db is None:
+            return (False, "MongoDB not configured")
+
+        users = db["users"]
+        # Case-insensitive user lookup and uniqueness for login.
+        users.create_index("username_lower", unique=True, name="uq_users_username_lower")
+        users.create_index("id", name="ix_users_id")
+
+        gallery = db["gallery_entries"]
+        # Frequent access patterns: per-user list newest-first, lookup/delete by id.
+        gallery.create_index([("username", 1), ("created_at", -1)], name="ix_gallery_user_created_at")
+        gallery.create_index([("username", 1), ("id", 1)], unique=True, name="uq_gallery_user_id")
+        gallery.create_index("created_at", name="ix_gallery_created_at")
+        return (True, None)
+    except PyMongoError as exc:
+        return (False, str(exc))

backend/data/users.json

@@ -1,7 +0,0 @@
-[
-  {
-    "id": 1,
-    "username": "admin",
-    "password_hash": "$2b$12$VOqXwimdOqv1jQOLXeIKiecH3OryASPr3c9elGuEb9cTH6s8qsObu"
-  }
-]

backend/requirements.txt

@@ -11,3 +11,4 @@ bcrypt>=4.0,<5
 pyjwt==2.10.1
 boto3>=1.35,<2
 ddgs>=7.0
+pymongo>=4.10,<5

backend/scripts/create_user.py

@@ -6,7 +6,6 @@ Or from backend: python scripts/create_user.py --username admin --password yourp
 """
 
 import argparse
-import json
 import sys
 from pathlib import Path
 
@@ -14,7 +13,8 @@ from pathlib import Path
 backend_dir = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(backend_dir))
 
-from app.auth import DATA_DIR, USERS_PATH, get_user_by_username, hash_password
+from app.auth import get_user_by_username, hash_password
+from app.mongo import get_mongo_db
 
 
 def main():
@@ -37,28 +37,22 @@ def main():
         print(f"Error: User '{username}' already exists.", file=sys.stderr)
         sys.exit(1)
 
-    DATA_DIR.mkdir(parents=True, exist_ok=True)
-    users = []
-    if USERS_PATH.exists():
-        try:
-            with open(USERS_PATH, "r") as f:
-                users = json.load(f)
-        except (json.JSONDecodeError, OSError):
-            users = []
+    db = get_mongo_db()
+    if db is None:
+        print("Error: MongoDB is not configured. Set MONGODB_URL and MONGODB_DB_NAME.", file=sys.stderr)
+        sys.exit(1)
+    users = list(db["users"].find({}, {"_id": 0}))
 
     next_id = max((u.get("id") or 0) for u in users) + 1 if users else 1
     new_user = {
         "id": next_id,
         "username": username,
+        "username_lower": username.lower(),
         "password_hash": hash_password(password),
     }
-    users.append(new_user)
-
-    with open(USERS_PATH, "w") as f:
-        json.dump(users, f, indent=2)
-
+    db["users"].insert_one(new_user)
     print(f"User '{username}' created successfully (id={next_id}).")
-    print(f"Users file: {USERS_PATH}")
+    print("Storage: MongoDB")
 
 
 if __name__ == "__main__":