Deploy URAAS — African Research Archival & Analytics System

scripts/init_db.py

@@ -9,7 +9,7 @@ import sys
 # Add project root to path (parent of scripts/)
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from uraas.database import Collection, Community, SessionLocal, init_db
+from uraas.database import Collection, Community, SessionLocal, init_db, sync_schema_columns
 
 # Import citation tracker models so SQLAlchemy registers their tables with
 # Base.metadata before create_all() runs — otherwise citations,
@@ -18,6 +18,53 @@ import uraas.services.citation_tracker  # noqa: F401
 
 from uraas.utils.unilag_classifier import UNILAG_STRUCTURE
 
+# Base.metadata.create_all() (called by init_db() below) only creates tables
+# that don't exist yet — it never alters an EXISTING table to add a column a
+# newer version of the ORM model expects. On a fresh DB that's a no-op (every
+# table is new, so every column is already there); on a database that
+# already existed before some column was added to the model (e.g. an HF
+# Space's persistent /data/uraas.db surviving across deploys), the table is
+# silently left on its old schema and the very next query touching that
+# column crashes — confirmed live 2026-07-19: the Space failed to start
+# entirely ("no such column: communities.unit_type") because its persistent
+# DB predated that column and nothing ever ran the matching migration
+# against it. Running every idempotent (column_exists()-guarded, ALTER-TABLE-
+# only) migration here means any existing database — this one included —
+# self-heals to the current schema on every startup, not just fresh ones.
+# migrate_add_ror.py is deliberately excluded: unlike the others it also
+# *writes* a default ROR value to existing NULL rows, and that default is
+# the deprecated pre-2026 UNILAG ROR (03qcnxw14, since corrected to
+# 05rk03822 by migrate_unilag_ror.py) — safe to run once by hand, not safe
+# to run unconditionally on every boot.
+_SCHEMA_MIGRATIONS = [
+    "migrate_add_pid_source",
+    "migrate_add_community_ace_columns",
+    "migrate_add_author_isni",
+    "migrate_add_item_funders",
+    "migrate_2026_upgrade",
+    "migrate_add_sc_columns",
+]
+
+
+def run_schema_migrations():
+    import importlib
+
+    print("Applying schema migrations (idempotent — safe to re-run)...")
+    print("  Generic column sync:")
+    sync_schema_columns()
+    for mod_name in _SCHEMA_MIGRATIONS:
+        try:
+            mod = importlib.import_module(mod_name)
+            mod.main()
+        except Exception as e:
+            # A migration failing shouldn't take down the whole app if the
+            # underlying tables/columns it depends on genuinely aren't there
+            # yet for some other reason — log and continue rather than abort
+            # startup entirely (seeding below will surface the real error if
+            # it still matters).
+            print(f"  [WARN] {mod_name} failed (continuing): {e}")
+    print()
+
 
 def seed_communities_and_collections():
     """Seed the database with UNILAG faculty and department structure."""
@@ -72,6 +119,10 @@ def main():
     print("[OK] Tables created successfully!")
     print()
 
+    # Heal any existing database (e.g. a persistent volume surviving across
+    # deploys) whose tables predate a column the current models expect.
+    run_schema_migrations()
+
     # Seed communities and collections
     seed_communities_and_collections()
     print()

scripts/push_to_hf.py

@@ -39,7 +39,18 @@ IGNORE_FILES = {
     "docker-compose.replica.yml",
     "docker-compose.demo.yml",
 }
-IGNORE_EXTS = {".pyc", ".pyo", ".pyd"}
+IGNORE_EXTS = {
+    ".pyc", ".pyo", ".pyd",
+    # Database files/backups — belt-and-suspenders beyond the exact
+    # "uraas.db" name check below and the startswith("uraas.db") check,
+    # since a missed pattern here means real crawled data (author names,
+    # DOIs, institutional affiliations, emails) goes to a PUBLIC repo.
+    # Confirmed this actually happened (2026-07-19): uraas.db.bak, a
+    # 1819-item/14MB snapshot, was uploaded because only "uraas.db" itself
+    # was excluded, not the ".bak" variant — deleted from the live Space
+    # after the fact, but should never have gone up in the first place.
+    ".bak", ".backup", ".old", ".orig", ".db", ".sqlite", ".sqlite3",
+}
 
 
 def should_skip(rel_path: str, is_dir: bool) -> bool:
@@ -47,9 +58,14 @@ def should_skip(rel_path: str, is_dir: bool) -> bool:
     name  = parts[-1]
     if is_dir:
         return name in IGNORE_DIRS
+    # Catches any suffix variant regardless of extension — uraas.db-wal,
+    # uraas.db-shm, uraas.db-journal, timestamped backups like
+    # uraas.db.20260629, etc. — not just the exact names/extensions above.
+    if name.startswith("uraas.db"):
+        return True
     return (
         name in IGNORE_FILES
-        or os.path.splitext(name)[1] in IGNORE_EXTS
+        or os.path.splitext(name)[1].lower() in IGNORE_EXTS
     )
 
 

uraas/database.py

@@ -393,3 +393,49 @@ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 def init_db():
     Base.metadata.create_all(bind=engine, checkfirst=True)
+
+
+def sync_schema_columns():
+    """Add any column declared on an ORM model but missing from the actual
+    database table.
+
+    create_all() only creates whole tables that don't exist yet — it never
+    alters an existing table, so a database that predates some column
+    (e.g. a persistent volume surviving across deploys) is silently left on
+    its old schema forever, and the first query touching that column
+    crashes. Confirmed live 2026-07-19: an HF Space failed to start
+    entirely because its persistent DB predated Community.unit_type, and a
+    synthetic even-older test schema also turned up a second, completely
+    undocumented gap (Community.ror) with no dedicated migration script at
+    all — one-column-at-a-time migration scripts don't scale to catching
+    every possible drift. This is generic instead: it walks every declared
+    ORM column and ALTERs in whatever the live table is missing, so it
+    self-heals regardless of which columns happen to be absent or when they
+    were added to the models.
+    """
+    from sqlalchemy import inspect, text
+
+    insp = inspect(engine)
+    existing_tables = set(insp.get_table_names())
+    added = 0
+    with engine.begin() as conn:
+        for table in Base.metadata.sorted_tables:
+            if table.name not in existing_tables:
+                continue  # brand new table — create_all() already made it correctly
+            existing_cols = {c["name"] for c in insp.get_columns(table.name)}
+            for column in table.columns:
+                if column.name in existing_cols:
+                    continue
+                try:
+                    ddl_type = column.type.compile(dialect=engine.dialect)
+                except Exception as e:
+                    print(f"  [WARN] cannot compile DDL type for {table.name}.{column.name}: {e}")
+                    continue
+                stmt = f"ALTER TABLE {table.name} ADD COLUMN {column.name} {ddl_type}"
+                try:
+                    conn.execute(text(stmt))
+                    print(f"  -> {stmt}")
+                    added += 1
+                except Exception as e:
+                    print(f"  [WARN] {stmt} failed: {e}")
+    print(f"  schema sync: {added} missing column(s) added" if added else "  schema sync: already in sync")