Deploy URAAS — African Research Archival & Analytics System

.gitignore

@@ -10,6 +10,9 @@ env/
 .env.*
 !.env.example
 
+# Claude Code local settings (may contain tokens/keys from shell history)
+.claude/settings.local.json
+
 # Databases
 *.db
 *.db-journal

Dockerfile

@@ -1,40 +1,40 @@
-# ── URAAS — Hugging Face Spaces Dockerfile ────────────────────────────────────
-# Single container: SQLite on /data (persistent bucket), gunicorn on port 7860.
-# No PostgreSQL or Redis needed — SQLite stored in HF persistent storage.
-# ──────────────────────────────────────────────────────────────────────────────
-FROM python:3.11-slim
-
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    URAAS_ENV=production \
-    PORT=7860
-
-# System deps
-RUN apt-get update && apt-get install -y \
-    gcc g++ curl \
-    && rm -rf /var/lib/apt/lists/*
-
-# App user — HF Spaces runs as root but we keep the same uid as prod
-RUN useradd -m -u 1000 uraas && \
-    mkdir -p /app /app/storage/pdfs /app/data /app/logs && \
-    chown -R uraas:uraas /app
-
-WORKDIR /app
-
-# Install Python dependencies (no libpq — SQLite only)
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt && \
-    python -m spacy download en_core_web_sm
-
-# Copy application code
-COPY --chown=uraas:uraas . .
-
-# Make startup script executable
-RUN chmod +x scripts/start_hf.sh
-
-USER uraas
-
-EXPOSE 7860
-
-# Startup: init DB in /data then start gunicorn
-CMD ["bash", "scripts/start_hf.sh"]
+# ── URAAS — Hugging Face Spaces Dockerfile ────────────────────────────────────
+# Single container: SQLite on /data (persistent bucket), gunicorn on port 7860.
+# No PostgreSQL or Redis needed — SQLite stored in HF persistent storage.
+# ──────────────────────────────────────────────────────────────────────────────
+FROM python:3.11-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    URAAS_ENV=production \
+    PORT=7860
+
+# System deps
+RUN apt-get update && apt-get install -y \
+    gcc g++ curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# App user — HF Spaces runs as root but we keep the same uid as prod
+RUN useradd -m -u 1000 uraas && \
+    mkdir -p /app /app/storage/pdfs /app/data /app/logs && \
+    chown -R uraas:uraas /app
+
+WORKDIR /app
+
+# Install Python dependencies (no libpq — SQLite only)
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt && \
+    python -m spacy download en_core_web_sm
+
+# Copy application code
+COPY --chown=uraas:uraas . .
+
+# Make startup script executable
+RUN chmod +x scripts/start_hf.sh
+
+USER uraas
+
+EXPOSE 7860
+
+# Startup: init DB in /data then start gunicorn
+CMD ["bash", "scripts/start_hf.sh"]

api/index.py

@@ -1,17 +1,17 @@
-import os
-import sys
-
-# Add root directory to sys.path so 'uraas' package is findable
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-# Set environment variables for Vercel
-# Vercel filesystem is read-only, so we point SQLite to a temp dir if we want to write,
-# or just keep it in the project root if it's read-only.
-os.environ["DATABASE_URL"] = "sqlite:///uraas.db"
-
-from uraas.dashboard.app import app
-
-# For Vercel, the variable must be named 'app'
-# but since we imported 'app' from uraas.dashboard.app, it's already there.
-# We just need to make sure it's exported at the module level.
-handler = app
+import os
+import sys
+
+# Add root directory to sys.path so 'uraas' package is findable
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+# Set environment variables for Vercel
+# Vercel filesystem is read-only, so we point SQLite to a temp dir if we want to write,
+# or just keep it in the project root if it's read-only.
+os.environ["DATABASE_URL"] = "sqlite:///uraas.db"
+
+from uraas.dashboard.app import app
+
+# For Vercel, the variable must be named 'app'
+# but since we imported 'app' from uraas.dashboard.app, it's already there.
+# We just need to make sure it's exported at the module level.
+handler = app

gunicorn_config.py

@@ -1,64 +1,64 @@
-"""
-Gunicorn configuration for production deployment on Render.
-Optimized for Flask-SocketIO with WebSocket support.
-"""
-
-import multiprocessing
-import os
-
-# Server socket — default 8080 matches Dockerfile EXPOSE and health checks.
-# HF Spaces overrides this with PORT=7860 via Space config.
-port = os.getenv("PORT", "8080")
-bind = f"0.0.0.0:{port}"
-
-# Worker processes
-# Free tier: 2 workers, Starter tier: 4 workers
-workers = int(os.getenv("GUNICORN_WORKERS", "2"))
-
-# Worker class — must match Flask-SocketIO async_mode.
-# app.py uses async_mode="threading", so we use gthread (synchronous + threads).
-# Do NOT use eventlet or gevent here without also changing async_mode in SocketIO.
-worker_class = "gthread"
-
-# Threads per worker (for gthread worker_class)
-threads = 4
-
-# Worker connections
-worker_connections = 1000
-
-# Restart workers after handling this many requests (prevents memory leaks)
-max_requests = 1000
-max_requests_jitter = 50
-
-# Timeout for requests (120 seconds for long-running crawler operations)
-timeout = 120
-
-# Keep-alive connections
-keepalive = 5
-
-# Logging
-accesslog = "-"  # Log to stdout (Render captures this)
-errorlog = "-"  # Log to stderr (Render captures this)
-loglevel = "info"
-access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
-
-# Process naming
-proc_name = "uraas-dashboard"
-
-# Graceful shutdown timeout
-graceful_timeout = 30
-
-# Preload app for faster worker spawning
-preload_app = True
-
-# Server mechanics
-daemon = False
-pidfile = None
-umask = 0
-user = None
-group = None
-tmp_upload_dir = None
-
-# SSL (handled by Render's load balancer)
-keyfile = None
-certfile = None
+"""
+Gunicorn configuration for production deployment on Render.
+Optimized for Flask-SocketIO with WebSocket support.
+"""
+
+import multiprocessing
+import os
+
+# Server socket — default 8080 matches Dockerfile EXPOSE and health checks.
+# HF Spaces overrides this with PORT=7860 via Space config.
+port = os.getenv("PORT", "8080")
+bind = f"0.0.0.0:{port}"
+
+# Worker processes
+# Free tier: 2 workers, Starter tier: 4 workers
+workers = int(os.getenv("GUNICORN_WORKERS", "2"))
+
+# Worker class — must match Flask-SocketIO async_mode.
+# app.py uses async_mode="threading", so we use gthread (synchronous + threads).
+# Do NOT use eventlet or gevent here without also changing async_mode in SocketIO.
+worker_class = "gthread"
+
+# Threads per worker (for gthread worker_class)
+threads = 4
+
+# Worker connections
+worker_connections = 1000
+
+# Restart workers after handling this many requests (prevents memory leaks)
+max_requests = 1000
+max_requests_jitter = 50
+
+# Timeout for requests (120 seconds for long-running crawler operations)
+timeout = 120
+
+# Keep-alive connections
+keepalive = 5
+
+# Logging
+accesslog = "-"  # Log to stdout (Render captures this)
+errorlog = "-"  # Log to stderr (Render captures this)
+loglevel = "info"
+access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
+
+# Process naming
+proc_name = "uraas-dashboard"
+
+# Graceful shutdown timeout
+graceful_timeout = 30
+
+# Preload app for faster worker spawning
+preload_app = True
+
+# Server mechanics
+daemon = False
+pidfile = None
+umask = 0
+user = None
+group = None
+tmp_upload_dir = None
+
+# SSL (handled by Render's load balancer)
+keyfile = None
+certfile = None

netlify/functions/app.py

@@ -1,49 +1,49 @@
-import os
-import sys
-
-# Add project root to path
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../.."))
-
-from uraas.dashboard.app import app
-
-
-def handler(event, context):
-    """Netlify serverless function handler"""
-    from io import BytesIO
-
-    from werkzeug.wrappers import Request, Response
-
-    # Convert Netlify event to WSGI environ
-    environ = {
-        "REQUEST_METHOD": event["httpMethod"],
-        "SCRIPT_NAME": "",
-        "PATH_INFO": event["path"],
-        "QUERY_STRING": event.get("rawQuery", ""),
-        "CONTENT_TYPE": event["headers"].get("content-type", ""),
-        "CONTENT_LENGTH": str(len(event.get("body", ""))),
-        "SERVER_NAME": event["headers"].get("host", "localhost"),
-        "SERVER_PORT": "443",
-        "SERVER_PROTOCOL": "HTTP/1.1",
-        "wsgi.version": (1, 0),
-        "wsgi.url_scheme": "https",
-        "wsgi.input": BytesIO(event.get("body", "").encode()),
-        "wsgi.errors": sys.stderr,
-        "wsgi.multithread": False,
-        "wsgi.multiprocess": True,
-        "wsgi.run_once": False,
-    }
-
-    # Add headers
-    for key, value in event.get("headers", {}).items():
-        key = key.upper().replace("-", "_")
-        if key not in ("CONTENT_TYPE", "CONTENT_LENGTH"):
-            environ[f"HTTP_{key}"] = value
-
-    # Call Flask app
-    response = Response.from_app(app, environ)
-
-    return {
-        "statusCode": response.status_code,
-        "headers": dict(response.headers),
-        "body": response.get_data(as_text=True),
-    }
+import os
+import sys
+
+# Add project root to path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../.."))
+
+from uraas.dashboard.app import app
+
+
+def handler(event, context):
+    """Netlify serverless function handler"""
+    from io import BytesIO
+
+    from werkzeug.wrappers import Request, Response
+
+    # Convert Netlify event to WSGI environ
+    environ = {
+        "REQUEST_METHOD": event["httpMethod"],
+        "SCRIPT_NAME": "",
+        "PATH_INFO": event["path"],
+        "QUERY_STRING": event.get("rawQuery", ""),
+        "CONTENT_TYPE": event["headers"].get("content-type", ""),
+        "CONTENT_LENGTH": str(len(event.get("body", ""))),
+        "SERVER_NAME": event["headers"].get("host", "localhost"),
+        "SERVER_PORT": "443",
+        "SERVER_PROTOCOL": "HTTP/1.1",
+        "wsgi.version": (1, 0),
+        "wsgi.url_scheme": "https",
+        "wsgi.input": BytesIO(event.get("body", "").encode()),
+        "wsgi.errors": sys.stderr,
+        "wsgi.multithread": False,
+        "wsgi.multiprocess": True,
+        "wsgi.run_once": False,
+    }
+
+    # Add headers
+    for key, value in event.get("headers", {}).items():
+        key = key.upper().replace("-", "_")
+        if key not in ("CONTENT_TYPE", "CONTENT_LENGTH"):
+            environ[f"HTTP_{key}"] = value
+
+    # Call Flask app
+    response = Response.from_app(app, environ)
+
+    return {
+        "statusCode": response.status_code,
+        "headers": dict(response.headers),
+        "body": response.get_data(as_text=True),
+    }

scratch/apply_remaining_ror_fixes.py

@@ -1,56 +1,56 @@
-"""
-Apply the manually-found ROR corrections for the 7 remaining institutions.
-"""
-import json
-from pathlib import Path
-
-config_dir = Path(__file__).parent.parent / "config" / "institutions"
-
-CORRECTIONS = {
-    "agostinhoneto.json": "https://ror.org/0057ag334",   # Agostinho Neto University (5726 works)
-    "kinshasa.json":      "https://ror.org/05rrz2q74",   # University of Kinshasa (10374 works)
-    "marienngouabi.json": "https://ror.org/00tt5kf04",   # Marien Ngouabi University (4236 works)
-    "masuku.json":        "https://ror.org/03f0njg03",   # Univ. Sciences et Techniques de Masuku (1522)
-    "mohammedv.json":     "https://ror.org/00r8w8f84",   # Mohammed V University (49646 works)
-    "tunis.json":         "https://ror.org/029cgt552",   # Tunis El Manar University (37992 works)
-    "yaoundei.json":      None,  # Need to search for Université de Yaoundé I specifically
-}
-
-# For Yaoundé I, search for the proper institution (not the hospital)
-import urllib.request, urllib.parse
-q = urllib.parse.quote("Universite de Yaounde")
-url = f"https://api.openalex.org/institutions?search={q}&per-page=5&mailto=cokiki@unilag.edu.ng"
-req = urllib.request.urlopen(url, timeout=15)
-resp = json.loads(req.read())
-print("=== Yaoundé I search results ===")
-for r in resp.get("results", []):
-    print(f"  [{r['works_count']:6d}] {r['display_name']} | ROR: {r['ror']}")
-
-# The actual Université de Yaoundé I
-# Will manually set from search result
-CORRECTIONS["yaoundei.json"] = "https://ror.org/01ktt0j77"  # Université de Yaoundé I (verified below)
-
-# Re-verify with direct lookup
-import urllib.request as ur
-try:
-    check_url = "https://api.openalex.org/works?filter=institutions.ror:01ktt0j77&select=id&per-page=1&mailto=cokiki@unilag.edu.ng"
-    r2 = json.loads(ur.urlopen(check_url, timeout=10).read())
-    print(f"\nYaoundé I (01ktt0j77): count={r2['meta']['count']}")
-except Exception as e:
-    print(f"Check failed: {e}")
-
-print("\n--- APPLYING REMAINING PATCHES ---")
-for fname, new_ror in CORRECTIONS.items():
-    if new_ror is None:
-        print(f"[SKIP] {fname}")
-        continue
-    jf = config_dir / fname
-    with open(jf) as f:
-        data = json.load(f)
-    old_ror = data["ror"]
-    data["ror"] = new_ror
-    with open(jf, "w", encoding="utf-8") as f:
-        json.dump(data, f, indent=2, ensure_ascii=False)
-    print(f"[PATCHED] {fname}: {old_ror} -> {new_ror}")
-
-print("\nAll done!")
+"""
+Apply the manually-found ROR corrections for the 7 remaining institutions.
+"""
+import json
+from pathlib import Path
+
+config_dir = Path(__file__).parent.parent / "config" / "institutions"
+
+CORRECTIONS = {
+    "agostinhoneto.json": "https://ror.org/0057ag334",   # Agostinho Neto University (5726 works)
+    "kinshasa.json":      "https://ror.org/05rrz2q74",   # University of Kinshasa (10374 works)
+    "marienngouabi.json": "https://ror.org/00tt5kf04",   # Marien Ngouabi University (4236 works)
+    "masuku.json":        "https://ror.org/03f0njg03",   # Univ. Sciences et Techniques de Masuku (1522)
+    "mohammedv.json":     "https://ror.org/00r8w8f84",   # Mohammed V University (49646 works)
+    "tunis.json":         "https://ror.org/029cgt552",   # Tunis El Manar University (37992 works)
+    "yaoundei.json":      None,  # Need to search for Université de Yaoundé I specifically
+}
+
+# For Yaoundé I, search for the proper institution (not the hospital)
+import urllib.request, urllib.parse
+q = urllib.parse.quote("Universite de Yaounde")
+url = f"https://api.openalex.org/institutions?search={q}&per-page=5&mailto=cokiki@unilag.edu.ng"
+req = urllib.request.urlopen(url, timeout=15)
+resp = json.loads(req.read())
+print("=== Yaoundé I search results ===")
+for r in resp.get("results", []):
+    print(f"  [{r['works_count']:6d}] {r['display_name']} | ROR: {r['ror']}")
+
+# The actual Université de Yaoundé I
+# Will manually set from search result
+CORRECTIONS["yaoundei.json"] = "https://ror.org/01ktt0j77"  # Université de Yaoundé I (verified below)
+
+# Re-verify with direct lookup
+import urllib.request as ur
+try:
+    check_url = "https://api.openalex.org/works?filter=institutions.ror:01ktt0j77&select=id&per-page=1&mailto=cokiki@unilag.edu.ng"
+    r2 = json.loads(ur.urlopen(check_url, timeout=10).read())
+    print(f"\nYaoundé I (01ktt0j77): count={r2['meta']['count']}")
+except Exception as e:
+    print(f"Check failed: {e}")
+
+print("\n--- APPLYING REMAINING PATCHES ---")
+for fname, new_ror in CORRECTIONS.items():
+    if new_ror is None:
+        print(f"[SKIP] {fname}")
+        continue
+    jf = config_dir / fname
+    with open(jf) as f:
+        data = json.load(f)
+    old_ror = data["ror"]
+    data["ror"] = new_ror
+    with open(jf, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
+    print(f"[PATCHED] {fname}: {old_ror} -> {new_ror}")
+
+print("\nAll done!")

scratch/check_db.py

@@ -1,18 +1,18 @@
-import sqlite3
-
-conn = sqlite3.connect("uraas.db")
-cursor = conn.cursor()
-
-cursor.execute(
-    "SELECT title, institution, created_at FROM items WHERE institution = 'Addis Ababa University' ORDER BY created_at DESC;"
-)
-rows = cursor.fetchall()
-
-if not rows:
-    print("No papers found for Addis Ababa University.")
-else:
-    print(f"Found {len(rows)} papers total:")
-    for row in rows:
-        print(f"- {row[0][:50]}... | {row[1]} | {row[2]}")
-
-conn.close()
+import sqlite3
+
+conn = sqlite3.connect("uraas.db")
+cursor = conn.cursor()
+
+cursor.execute(
+    "SELECT title, institution, created_at FROM items WHERE institution = 'Addis Ababa University' ORDER BY created_at DESC;"
+)
+rows = cursor.fetchall()
+
+if not rows:
+    print("No papers found for Addis Ababa University.")
+else:
+    print(f"Found {len(rows)} papers total:")
+    for row in rows:
+        print(f"- {row[0][:50]}... | {row[1]} | {row[2]}")
+
+conn.close()

scratch/fix_rors.py

@@ -1,91 +1,91 @@
-"""
-Lookup correct RORs from OpenAlex for all broken institutions,
-then patch the JSON files automatically.
-"""
-import json
-import time
-import urllib.request
-import urllib.parse
-from pathlib import Path
-
-BASE = "https://api.openalex.org"
-MAILTO = "cokiki@unilag.edu.ng"
-
-# Institutions we know are broken (from verify_rors.py output)
-BROKEN = [
-    "agostinhoneto.json",
-    "ainshams.json",
-    "alexandria.json",
-    "cairo.json",
-    "daressalaam.json",
-    "kinshasa.json",
-    "marienngouabi.json",
-    "masuku.json",
-    "mohammedv.json",
-    "pretoria.json",
-    "rwanda.json",
-    "tunis.json",
-    "wits.json",
-    "yaoundei.json",
-    "zimbabwe.json",
-]
-
-config_dir = Path(__file__).parent.parent / "config" / "institutions"
-fixes = {}
-
-for fname in BROKEN:
-    jf = config_dir / fname
-    with open(jf) as f:
-        data = json.load(f)
-
-    name = data["name"]
-    q = urllib.parse.quote(name)
-    url = f"{BASE}/institutions?search={q}&per-page=3&mailto={MAILTO}"
-
-    try:
-        req = urllib.request.urlopen(url, timeout=15)
-        resp = json.loads(req.read())
-        results = resp.get("results", [])
-    except Exception as e:
-        print(f"[ERROR] {name}: {e}")
-        fixes[fname] = None
-        time.sleep(1)
-        continue
-
-    if not results:
-        print(f"[NOT FOUND] {name}")
-        fixes[fname] = None
-    else:
-        best = results[0]
-        ror = best["ror"]  # e.g. "https://ror.org/00cb9w016"
-        ror_short = ror.split("/")[-1]
-        count = best.get("works_count", "?")
-        display = best["display_name"]
-        print(f"[FOUND] {name!r}")
-        print(f"        OpenAlex: {display!r}")
-        print(f"        ROR: {ror}  (works: {count})")
-        if count == 0:
-            print(f"        WARNING: works_count=0 — double check!")
-        fixes[fname] = ror
-
-    time.sleep(0.4)
-
-# Now apply the patches
-print("\n--- APPLYING PATCHES ---")
-for fname, new_ror in fixes.items():
-    if new_ror is None:
-        print(f"[SKIP] {fname} — no ROR found")
-        continue
-    jf = config_dir / fname
-    with open(jf) as f:
-        data = json.load(f)
-    old_ror = data["ror"]
-    if old_ror == new_ror:
-        print(f"[SAME] {fname} — already correct")
-        continue
-    data["ror"] = new_ror
-    with open(jf, "w", encoding="utf-8") as f:
-        json.dump(data, f, indent=2, ensure_ascii=False)
-    print(f"[PATCHED] {fname}: {old_ror} -> {new_ror}")
-
-print("\nDone. Run verify_rors.py again to confirm all are fixed.")
+"""
+Lookup correct RORs from OpenAlex for all broken institutions,
+then patch the JSON files automatically.
+"""
+import json
+import time
+import urllib.request
+import urllib.parse
+from pathlib import Path
+
+BASE = "https://api.openalex.org"
+MAILTO = "cokiki@unilag.edu.ng"
+
+# Institutions we know are broken (from verify_rors.py output)
+BROKEN = [
+    "agostinhoneto.json",
+    "ainshams.json",
+    "alexandria.json",
+    "cairo.json",
+    "daressalaam.json",
+    "kinshasa.json",
+    "marienngouabi.json",
+    "masuku.json",
+    "mohammedv.json",
+    "pretoria.json",
+    "rwanda.json",
+    "tunis.json",
+    "wits.json",
+    "yaoundei.json",
+    "zimbabwe.json",
+]
+
+config_dir = Path(__file__).parent.parent / "config" / "institutions"
+fixes = {}
+
+for fname in BROKEN:
+    jf = config_dir / fname
+    with open(jf) as f:
+        data = json.load(f)
+
+    name = data["name"]
+    q = urllib.parse.quote(name)
+    url = f"{BASE}/institutions?search={q}&per-page=3&mailto={MAILTO}"
+
+    try:
+        req = urllib.request.urlopen(url, timeout=15)
+        resp = json.loads(req.read())
+        results = resp.get("results", [])
+    except Exception as e:
+        print(f"[ERROR] {name}: {e}")
+        fixes[fname] = None
+        time.sleep(1)
+        continue
+
+    if not results:
+        print(f"[NOT FOUND] {name}")
+        fixes[fname] = None
+    else:
+        best = results[0]
+        ror = best["ror"]  # e.g. "https://ror.org/00cb9w016"
+        ror_short = ror.split("/")[-1]
+        count = best.get("works_count", "?")
+        display = best["display_name"]
+        print(f"[FOUND] {name!r}")
+        print(f"        OpenAlex: {display!r}")
+        print(f"        ROR: {ror}  (works: {count})")
+        if count == 0:
+            print(f"        WARNING: works_count=0 — double check!")
+        fixes[fname] = ror
+
+    time.sleep(0.4)
+
+# Now apply the patches
+print("\n--- APPLYING PATCHES ---")
+for fname, new_ror in fixes.items():
+    if new_ror is None:
+        print(f"[SKIP] {fname} — no ROR found")
+        continue
+    jf = config_dir / fname
+    with open(jf) as f:
+        data = json.load(f)
+    old_ror = data["ror"]
+    if old_ror == new_ror:
+        print(f"[SAME] {fname} — already correct")
+        continue
+    data["ror"] = new_ror
+    with open(jf, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
+    print(f"[PATCHED] {fname}: {old_ror} -> {new_ror}")
+
+print("\nDone. Run verify_rors.py again to confirm all are fixed.")

scratch/fix_rors_manual.py

@@ -1,56 +1,56 @@
-"""
-Manual ROR lookup for institutions not found by name search.
-Uses OpenAlex institution search with alternate spellings/names.
-"""
-import json
-import time
-import urllib.request
-import urllib.parse
-from pathlib import Path
-
-BASE = "https://api.openalex.org"
-MAILTO = "cokiki@unilag.edu.ng"
-
-# Alternate search terms for institutions not found by direct name
-ALTERNATES = {
-    "agostinhoneto.json": ["Agostinho Neto", "UAN Angola", "Luanda university"],
-    "kinshasa.json": ["Kinshasa university", "UNIKIN", "Congo kinshasa"],
-    "marienngouabi.json": ["Marien Ngouabi", "Brazzaville university", "Congo Brazzaville"],
-    "masuku.json": ["Masuku", "Franceville", "Gabon university science"],
-    "mohammedv.json": ["Mohammed V", "Rabat university", "Mohammed 5"],
-    "tunis.json": ["Tunis El Manar", "Tunis university", "UTM Tunisia"],
-    "yaoundei.json": ["Yaounde", "Cameroon university", "Yaounde 1"],
-}
-
-config_dir = Path(__file__).parent.parent / "config" / "institutions"
-
-for fname, search_terms in ALTERNATES.items():
-    jf = config_dir / fname
-    with open(jf) as f:
-        data = json.load(f)
-    name = data["name"]
-    print(f"\n=== {name} ===")
-    
-    found = False
-    for term in search_terms:
-        q = urllib.parse.quote(term)
-        url = f"{BASE}/institutions?search={q}&per-page=5&mailto={MAILTO}"
-        try:
-            req = urllib.request.urlopen(url, timeout=15)
-            resp = json.loads(req.read())
-            results = resp.get("results", [])
-        except Exception as e:
-            print(f"  ERROR searching {term!r}: {e}")
-            time.sleep(1)
-            continue
-        
-        if results:
-            print(f"  Search '{term}' -> {len(results)} results:")
-            for r in results[:3]:
-                print(f"    [{r['works_count']:6d} works] {r['display_name']} | ROR: {r['ror']}")
-            found = True
-        else:
-            print(f"  Search '{term}' -> no results")
-        time.sleep(0.4)
-        if found:
-            break
+"""
+Manual ROR lookup for institutions not found by name search.
+Uses OpenAlex institution search with alternate spellings/names.
+"""
+import json
+import time
+import urllib.request
+import urllib.parse
+from pathlib import Path
+
+BASE = "https://api.openalex.org"
+MAILTO = "cokiki@unilag.edu.ng"
+
+# Alternate search terms for institutions not found by direct name
+ALTERNATES = {
+    "agostinhoneto.json": ["Agostinho Neto", "UAN Angola", "Luanda university"],
+    "kinshasa.json": ["Kinshasa university", "UNIKIN", "Congo kinshasa"],
+    "marienngouabi.json": ["Marien Ngouabi", "Brazzaville university", "Congo Brazzaville"],
+    "masuku.json": ["Masuku", "Franceville", "Gabon university science"],
+    "mohammedv.json": ["Mohammed V", "Rabat university", "Mohammed 5"],
+    "tunis.json": ["Tunis El Manar", "Tunis university", "UTM Tunisia"],
+    "yaoundei.json": ["Yaounde", "Cameroon university", "Yaounde 1"],
+}
+
+config_dir = Path(__file__).parent.parent / "config" / "institutions"
+
+for fname, search_terms in ALTERNATES.items():
+    jf = config_dir / fname
+    with open(jf) as f:
+        data = json.load(f)
+    name = data["name"]
+    print(f"\n=== {name} ===")
+    
+    found = False
+    for term in search_terms:
+        q = urllib.parse.quote(term)
+        url = f"{BASE}/institutions?search={q}&per-page=5&mailto={MAILTO}"
+        try:
+            req = urllib.request.urlopen(url, timeout=15)
+            resp = json.loads(req.read())
+            results = resp.get("results", [])
+        except Exception as e:
+            print(f"  ERROR searching {term!r}: {e}")
+            time.sleep(1)
+            continue
+        
+        if results:
+            print(f"  Search '{term}' -> {len(results)} results:")
+            for r in results[:3]:
+                print(f"    [{r['works_count']:6d} works] {r['display_name']} | ROR: {r['ror']}")
+            found = True
+        else:
+            print(f"  Search '{term}' -> no results")
+        time.sleep(0.4)
+        if found:
+            break

scratch/generate_all_configs.py

@@ -1,145 +1,145 @@
-import json
-import os
-
-# Sub-regions and countries
-subregions = {
-    "North Africa": ["Egypt", "Morocco", "Algeria", "Tunisia", "Libya", "Sudan"],
-    "West Africa": ["Nigeria", "Ghana", "Senegal", "Cote d'Ivoire", "Benin", "Burkina Faso", "Cape Verde", "Gambia", "Guinea", "Guinea-Bissau", "Liberia", "Mali", "Mauritania", "Niger", "Sierra Leone", "Togo"],
-    "East Africa": ["Kenya", "Uganda", "Tanzania", "Ethiopia", "Rwanda", "Burundi", "Djibouti", "Eritrea", "Somalia", "South Sudan", "Madagascar", "Mauritius", "Seychelles", "Comoros"],
-    "Southern Africa": ["South Africa", "Zimbabwe", "Zambia", "Namibia", "Botswana", "Lesotho", "Eswatini", "Malawi", "Mozambique"],
-    "Central Africa": ["Cameroon", "DR Congo", "Angola", "Gabon", "Republic of the Congo", "Central African Republic", "Chad", "Equatorial Guinea", "Sao Tome and Principe"],
-}
-
-curated_universities = {
-    "Egypt": [
-        {"name": "Cairo University", "ror": "https://ror.org/03c4mpy73"},
-        {"name": "Ain Shams University", "ror": "https://ror.org/034x7p097"},
-        {"name": "Alexandria University", "ror": "https://ror.org/02078r490"},
-        {"name": "Mansoura University", "ror": "https://ror.org/032p18087"},
-        {"name": "Assiut University", "ror": "https://ror.org/047fpp722"},
-    ],
-    "Morocco": [
-        {"name": "Université Mohammed V de Rabat", "ror": "https://ror.org/03vpy3v17"},
-        {"name": "Université Cadi Ayyad", "ror": "https://ror.org/0154pcf71"},
-        {"name": "Université Hassan II de Casablanca", "ror": "https://ror.org/013y27r38"},
-    ],
-    "Tunisia": [
-        {"name": "Université de Tunis El Manar", "ror": "https://ror.org/050j3a172"},
-        {"name": "Université de Sfax", "ror": "https://ror.org/02157p641"},
-        {"name": "Université de Carthage", "ror": "https://ror.org/011yvpy71"},
-    ],
-    "Cameroon": [
-        {"name": "Université de Yaoundé I", "ror": "https://ror.org/04h7g6177"},
-        {"name": "Université de Dschang", "ror": "https://ror.org/012y7p041"},
-        {"name": "Université de Douala", "ror": "https://ror.org/041y27r28"},
-    ],
-    "DR Congo": [
-        {"name": "Université de Kinshasa", "ror": "https://ror.org/05vzwad88"},
-        {"name": "Université de Lubumbashi", "ror": "https://ror.org/02rry3m21"},
-    ],
-    "Angola": [
-        {"name": "Université Agostinho Neto", "ror": "https://ror.org/00z2bpt98"}
-    ],
-    "Gabon": [
-        {"name": "Université des Sciences et Techniques de Masuku", "ror": "https://ror.org/059gqse72"},
-        {"name": "Université Omar Bongo", "ror": "https://ror.org/041yp1812"},
-    ],
-    "Republic of the Congo": [
-        {"name": "Université Marien Ngouabi", "ror": "https://ror.org/02y1sra05"}
-    ],
-    "Nigeria": [
-        {"name": "University of Lagos", "ror": "https://ror.org/05rk03822"},
-        {"name": "University of Ibadan", "ror": "https://ror.org/01es5me90"},
-        {"name": "Covenant University", "ror": "https://ror.org/02n05rk12"},
-        {"name": "Obafemi Awolowo University", "ror": "https://ror.org/013pcr241"},
-        {"name": "University of Nigeria Nsukka", "ror": "https://ror.org/02kpy5732"},
-    ],
-    "Ghana": [
-        {"name": "University of Ghana", "ror": "https://ror.org/00zpy3v12"},
-        {"name": "Kwame Nkrumah University of Science and Technology", "ror": "https://ror.org/00x4mpy73"},
-        {"name": "University of Cape Coast", "ror": "https://ror.org/01es3v123"},
-    ],
-    "South Africa": [
-        {"name": "University of Cape Town", "ror": "https://ror.org/017620319"},
-        {"name": "Stellenbosch University", "ror": "https://ror.org/05777p686"},
-        {"name": "University of the Witwatersrand", "ror": "https://ror.org/039482g93"},
-        {"name": "University of Pretoria", "ror": "https://ror.org/047fpp722"},
-        {"name": "University of KwaZulu-Natal", "ror": "https://ror.org/01267r312"},
-    ],
-    "Zimbabwe": [
-        {"name": "University of Zimbabwe", "ror": "https://ror.org/03w489125"},
-        {"name": "National University of Science and Technology", "ror": "https://ror.org/01y6mpy73"},
-    ],
-    "Kenya": [
-        {"name": "University of Nairobi", "ror": "https://ror.org/01078r490"},
-        {"name": "Kenyatta University", "ror": "https://ror.org/01py3v171"},
-        {"name": "Jomo Kenyatta University of Agriculture and Technology", "ror": "https://ror.org/03pyvpy71"},
-    ],
-    "Uganda": [
-        {"name": "Makerere University", "ror": "https://ror.org/05vzwad88"},
-        {"name": "Mbarara University of Science and Technology", "ror": "https://ror.org/0155pcf71"},
-    ],
-    "Tanzania": [
-        {"name": "University of Dar es Salaam", "ror": "https://ror.org/0199e1957"},
-        {"name": "Sokoine University of Agriculture", "ror": "https://ror.org/011y27r38"},
-    ],
-    "Ethiopia": [
-        {"name": "Addis Ababa University", "ror": "https://ror.org/01py3v171"}
-    ],
-    "Rwanda": [
-        {"name": "University of Rwanda", "ror": "https://ror.org/02yr01r27"}
-    ],
-}
-
-os.makedirs("config/institutions", exist_ok=True)
-count = 0
-
-for country, unis in curated_universities.items():
-    region = next(r for r, c in subregions.items() if country in c)
-    for u in unis:
-        # Create a safe shortname / filename
-        short_name = u["name"].replace("University of ", "").replace("Université de ", "").replace("Université ", "")
-        if len(short_name.split()) > 3:
-            short_name = "".join([word[0] for word in short_name.split() if word.istitle()])
-        if not short_name:
-            short_name = u["name"].split()[0]
-            
-        # Overrides for some known ones
-        if "Lagos" in u["name"]: short_name = "UNILAG"
-        elif "Ibadan" in u["name"]: short_name = "UI"
-        elif "Cape Town" in u["name"]: short_name = "UCT"
-        elif "Witwatersrand" in u["name"]: short_name = "Wits"
-        elif "Kwame Nkrumah" in u["name"]: short_name = "KNUST"
-        elif "Yaoundé" in u["name"]: short_name = "Yaounde I"
-        
-        file_name = "".join(x for x in short_name.lower() if x.isalnum()) + ".json"
-        
-        cfg = {
-            "ror": u["ror"],
-            "name": u["name"],
-            "short_name": short_name,
-            "country": country,
-            "sub_region": region,
-            "staff_file": f"data/{short_name.lower().replace(' ', '_')}_staff.json",
-            "affiliation_patterns": [u["name"], short_name, f"{u['name']} Department"],
-            "faculties": [
-                "Science",
-                "Humanities",
-                "Engineering",
-                "Medicine",
-                "Social Sciences",
-                "Arts",
-                "Law",
-            ],
-            "crawler_settings": {
-                "rate_limit": 2.0,
-                "concurrent_requests": 8,
-                "retry_times": 3,
-                "download_delay": 2.0,
-            },
-        }
-        with open(f"config/institutions/{file_name}", "w", encoding="utf-8") as f:
-            json.dump(cfg, f, indent=2, ensure_ascii=False)
-        count += 1
-
-print(f"Generated {count} university configurations.")
+import json
+import os
+
+# Sub-regions and countries
+subregions = {
+    "North Africa": ["Egypt", "Morocco", "Algeria", "Tunisia", "Libya", "Sudan"],
+    "West Africa": ["Nigeria", "Ghana", "Senegal", "Cote d'Ivoire", "Benin", "Burkina Faso", "Cape Verde", "Gambia", "Guinea", "Guinea-Bissau", "Liberia", "Mali", "Mauritania", "Niger", "Sierra Leone", "Togo"],
+    "East Africa": ["Kenya", "Uganda", "Tanzania", "Ethiopia", "Rwanda", "Burundi", "Djibouti", "Eritrea", "Somalia", "South Sudan", "Madagascar", "Mauritius", "Seychelles", "Comoros"],
+    "Southern Africa": ["South Africa", "Zimbabwe", "Zambia", "Namibia", "Botswana", "Lesotho", "Eswatini", "Malawi", "Mozambique"],
+    "Central Africa": ["Cameroon", "DR Congo", "Angola", "Gabon", "Republic of the Congo", "Central African Republic", "Chad", "Equatorial Guinea", "Sao Tome and Principe"],
+}
+
+curated_universities = {
+    "Egypt": [
+        {"name": "Cairo University", "ror": "https://ror.org/03c4mpy73"},
+        {"name": "Ain Shams University", "ror": "https://ror.org/034x7p097"},
+        {"name": "Alexandria University", "ror": "https://ror.org/02078r490"},
+        {"name": "Mansoura University", "ror": "https://ror.org/032p18087"},
+        {"name": "Assiut University", "ror": "https://ror.org/047fpp722"},
+    ],
+    "Morocco": [
+        {"name": "Université Mohammed V de Rabat", "ror": "https://ror.org/03vpy3v17"},
+        {"name": "Université Cadi Ayyad", "ror": "https://ror.org/0154pcf71"},
+        {"name": "Université Hassan II de Casablanca", "ror": "https://ror.org/013y27r38"},
+    ],
+    "Tunisia": [
+        {"name": "Université de Tunis El Manar", "ror": "https://ror.org/050j3a172"},
+        {"name": "Université de Sfax", "ror": "https://ror.org/02157p641"},
+        {"name": "Université de Carthage", "ror": "https://ror.org/011yvpy71"},
+    ],
+    "Cameroon": [
+        {"name": "Université de Yaoundé I", "ror": "https://ror.org/04h7g6177"},
+        {"name": "Université de Dschang", "ror": "https://ror.org/012y7p041"},
+        {"name": "Université de Douala", "ror": "https://ror.org/041y27r28"},
+    ],
+    "DR Congo": [
+        {"name": "Université de Kinshasa", "ror": "https://ror.org/05vzwad88"},
+        {"name": "Université de Lubumbashi", "ror": "https://ror.org/02rry3m21"},
+    ],
+    "Angola": [
+        {"name": "Université Agostinho Neto", "ror": "https://ror.org/00z2bpt98"}
+    ],
+    "Gabon": [
+        {"name": "Université des Sciences et Techniques de Masuku", "ror": "https://ror.org/059gqse72"},
+        {"name": "Université Omar Bongo", "ror": "https://ror.org/041yp1812"},
+    ],
+    "Republic of the Congo": [
+        {"name": "Université Marien Ngouabi", "ror": "https://ror.org/02y1sra05"}
+    ],
+    "Nigeria": [
+        {"name": "University of Lagos", "ror": "https://ror.org/05rk03822"},
+        {"name": "University of Ibadan", "ror": "https://ror.org/01es5me90"},
+        {"name": "Covenant University", "ror": "https://ror.org/02n05rk12"},
+        {"name": "Obafemi Awolowo University", "ror": "https://ror.org/013pcr241"},
+        {"name": "University of Nigeria Nsukka", "ror": "https://ror.org/02kpy5732"},
+    ],
+    "Ghana": [
+        {"name": "University of Ghana", "ror": "https://ror.org/00zpy3v12"},
+        {"name": "Kwame Nkrumah University of Science and Technology", "ror": "https://ror.org/00x4mpy73"},
+        {"name": "University of Cape Coast", "ror": "https://ror.org/01es3v123"},
+    ],
+    "South Africa": [
+        {"name": "University of Cape Town", "ror": "https://ror.org/017620319"},
+        {"name": "Stellenbosch University", "ror": "https://ror.org/05777p686"},
+        {"name": "University of the Witwatersrand", "ror": "https://ror.org/039482g93"},
+        {"name": "University of Pretoria", "ror": "https://ror.org/047fpp722"},
+        {"name": "University of KwaZulu-Natal", "ror": "https://ror.org/01267r312"},
+    ],
+    "Zimbabwe": [
+        {"name": "University of Zimbabwe", "ror": "https://ror.org/03w489125"},
+        {"name": "National University of Science and Technology", "ror": "https://ror.org/01y6mpy73"},
+    ],
+    "Kenya": [
+        {"name": "University of Nairobi", "ror": "https://ror.org/01078r490"},
+        {"name": "Kenyatta University", "ror": "https://ror.org/01py3v171"},
+        {"name": "Jomo Kenyatta University of Agriculture and Technology", "ror": "https://ror.org/03pyvpy71"},
+    ],
+    "Uganda": [
+        {"name": "Makerere University", "ror": "https://ror.org/05vzwad88"},
+        {"name": "Mbarara University of Science and Technology", "ror": "https://ror.org/0155pcf71"},
+    ],
+    "Tanzania": [
+        {"name": "University of Dar es Salaam", "ror": "https://ror.org/0199e1957"},
+        {"name": "Sokoine University of Agriculture", "ror": "https://ror.org/011y27r38"},
+    ],
+    "Ethiopia": [
+        {"name": "Addis Ababa University", "ror": "https://ror.org/01py3v171"}
+    ],
+    "Rwanda": [
+        {"name": "University of Rwanda", "ror": "https://ror.org/02yr01r27"}
+    ],
+}
+
+os.makedirs("config/institutions", exist_ok=True)
+count = 0
+
+for country, unis in curated_universities.items():
+    region = next(r for r, c in subregions.items() if country in c)
+    for u in unis:
+        # Create a safe shortname / filename
+        short_name = u["name"].replace("University of ", "").replace("Université de ", "").replace("Université ", "")
+        if len(short_name.split()) > 3:
+            short_name = "".join([word[0] for word in short_name.split() if word.istitle()])
+        if not short_name:
+            short_name = u["name"].split()[0]
+            
+        # Overrides for some known ones
+        if "Lagos" in u["name"]: short_name = "UNILAG"
+        elif "Ibadan" in u["name"]: short_name = "UI"
+        elif "Cape Town" in u["name"]: short_name = "UCT"
+        elif "Witwatersrand" in u["name"]: short_name = "Wits"
+        elif "Kwame Nkrumah" in u["name"]: short_name = "KNUST"
+        elif "Yaoundé" in u["name"]: short_name = "Yaounde I"
+        
+        file_name = "".join(x for x in short_name.lower() if x.isalnum()) + ".json"
+        
+        cfg = {
+            "ror": u["ror"],
+            "name": u["name"],
+            "short_name": short_name,
+            "country": country,
+            "sub_region": region,
+            "staff_file": f"data/{short_name.lower().replace(' ', '_')}_staff.json",
+            "affiliation_patterns": [u["name"], short_name, f"{u['name']} Department"],
+            "faculties": [
+                "Science",
+                "Humanities",
+                "Engineering",
+                "Medicine",
+                "Social Sciences",
+                "Arts",
+                "Law",
+            ],
+            "crawler_settings": {
+                "rate_limit": 2.0,
+                "concurrent_requests": 8,
+                "retry_times": 3,
+                "download_delay": 2.0,
+            },
+        }
+        with open(f"config/institutions/{file_name}", "w", encoding="utf-8") as f:
+            json.dump(cfg, f, indent=2, ensure_ascii=False)
+        count += 1
+
+print(f"Generated {count} university configurations.")

scratch/generate_creds.py

@@ -1,18 +1,18 @@
-import secrets
-from werkzeug.security import generate_password_hash
-
-secret_key = secrets.token_hex(32)
-admin_pw = "uraas_admin_2026"
-viewer_pw = "uraas_viewer_2026"
-
-print("=== NEW URAAS CREDENTIALS ===")
-print("DASHBOARD_SECRET_KEY=" + secret_key)
-print("ADMIN_USERNAME=admin")
-print("ADMIN_PASSWORD_HASH=" + generate_password_hash(admin_pw))
-print("VIEWER_USERNAME=viewer")
-print("VIEWER_PASSWORD_HASH=" + generate_password_hash(viewer_pw))
-print()
-print(f"Admin plain password: {admin_pw}")
-print(f"Viewer plain password: {viewer_pw}")
-print()
-print("Store these safely in the server environment!")
+import secrets
+from werkzeug.security import generate_password_hash
+
+secret_key = secrets.token_hex(32)
+admin_pw = "uraas_admin_2026"
+viewer_pw = "uraas_viewer_2026"
+
+print("=== NEW URAAS CREDENTIALS ===")
+print("DASHBOARD_SECRET_KEY=" + secret_key)
+print("ADMIN_USERNAME=admin")
+print("ADMIN_PASSWORD_HASH=" + generate_password_hash(admin_pw))
+print("VIEWER_USERNAME=viewer")
+print("VIEWER_PASSWORD_HASH=" + generate_password_hash(viewer_pw))
+print()
+print(f"Admin plain password: {admin_pw}")
+print(f"Viewer plain password: {viewer_pw}")
+print()
+print("Store these safely in the server environment!")

scratch/inspect_citations.py

@@ -1,30 +1,30 @@
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from sqlalchemy import text
-
-from uraas.database import Base, SessionLocal, engine
-
-
-def inspect():
-    session = SessionLocal()
-    try:
-        # Check if tables exist first
-        tables = ["citations", "citation_metrics", "author_metrics"]
-        for table in tables:
-            try:
-                res = session.execute(text(f"SELECT COUNT(*) FROM {table}")).scalar()
-                print(f"Table '{table}' has {res} rows")
-            except Exception as e:
-                print(f"Table '{table}' error: {e}")
-
-    except Exception as e:
-        print(f"Error: {e}")
-    finally:
-        session.close()
-
-
-if __name__ == "__main__":
-    inspect()
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from sqlalchemy import text
+
+from uraas.database import Base, SessionLocal, engine
+
+
+def inspect():
+    session = SessionLocal()
+    try:
+        # Check if tables exist first
+        tables = ["citations", "citation_metrics", "author_metrics"]
+        for table in tables:
+            try:
+                res = session.execute(text(f"SELECT COUNT(*) FROM {table}")).scalar()
+                print(f"Table '{table}' has {res} rows")
+            except Exception as e:
+                print(f"Table '{table}' error: {e}")
+
+    except Exception as e:
+        print(f"Error: {e}")
+    finally:
+        session.close()
+
+
+if __name__ == "__main__":
+    inspect()

scratch/inspect_db.py

@@ -1,73 +1,73 @@
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from sqlalchemy import func
-
-from uraas.database import Item, SessionLocal
-
-
-def inspect():
-    session = SessionLocal()
-    try:
-        total_items = session.query(Item).count()
-        print(f"Total items: {total_items}")
-
-        # Group by institution/ror
-        inst_counts = (
-            session.query(Item.institution, Item.ror, func.count(Item.id))
-            .group_by(Item.institution, Item.ror)
-            .all()
-        )
-        print("\nItems per institution:")
-        for inst, ror, count in inst_counts:
-            print(f"  - {inst} ({ror}): {count} papers")
-
-        # African language papers
-        african_lang_count = (
-            session.query(func.count(Item.id))
-            .filter(Item.is_african_language == True)
-            .scalar()
-        )
-        print(f"\nAfrican language papers: {african_lang_count}")
-
-        # TK vitality papers
-        tk_count = (
-            session.query(func.count(Item.id))
-            .filter(
-                (Item.tk_label.isnot(None))
-                | (Item.content_type == "indigenous_knowledge")
-            )
-            .scalar()
-        )
-        print(f"Indigenous knowledge / TK papers: {tk_count}")
-
-        # Patents
-        patent_count = (
-            session.query(func.count(Item.id))
-            .filter(Item.patent_id.isnot(None))
-            .scalar()
-        )
-        print(f"Patents: {patent_count}")
-
-        # DocID coverage
-        docid_count = (
-            session.query(func.count(Item.id)).filter(Item.docid.isnot(None)).scalar()
-        )
-        print(f"DocID assigned papers: {docid_count}")
-
-        # Access policy / PDFs
-        from uraas.database import File
-
-        pdf_count = session.query(File).count()
-        print(f"Downloaded PDFs: {pdf_count}")
-
-    except Exception as e:
-        print(f"Error: {e}")
-    finally:
-        session.close()
-
-
-if __name__ == "__main__":
-    inspect()
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from sqlalchemy import func
+
+from uraas.database import Item, SessionLocal
+
+
+def inspect():
+    session = SessionLocal()
+    try:
+        total_items = session.query(Item).count()
+        print(f"Total items: {total_items}")
+
+        # Group by institution/ror
+        inst_counts = (
+            session.query(Item.institution, Item.ror, func.count(Item.id))
+            .group_by(Item.institution, Item.ror)
+            .all()
+        )
+        print("\nItems per institution:")
+        for inst, ror, count in inst_counts:
+            print(f"  - {inst} ({ror}): {count} papers")
+
+        # African language papers
+        african_lang_count = (
+            session.query(func.count(Item.id))
+            .filter(Item.is_african_language == True)
+            .scalar()
+        )
+        print(f"\nAfrican language papers: {african_lang_count}")
+
+        # TK vitality papers
+        tk_count = (
+            session.query(func.count(Item.id))
+            .filter(
+                (Item.tk_label.isnot(None))
+                | (Item.content_type == "indigenous_knowledge")
+            )
+            .scalar()
+        )
+        print(f"Indigenous knowledge / TK papers: {tk_count}")
+
+        # Patents
+        patent_count = (
+            session.query(func.count(Item.id))
+            .filter(Item.patent_id.isnot(None))
+            .scalar()
+        )
+        print(f"Patents: {patent_count}")
+
+        # DocID coverage
+        docid_count = (
+            session.query(func.count(Item.id)).filter(Item.docid.isnot(None)).scalar()
+        )
+        print(f"DocID assigned papers: {docid_count}")
+
+        # Access policy / PDFs
+        from uraas.database import File
+
+        pdf_count = session.query(File).count()
+        print(f"Downloaded PDFs: {pdf_count}")
+
+    except Exception as e:
+        print(f"Error: {e}")
+    finally:
+        session.close()
+
+
+if __name__ == "__main__":
+    inspect()

scratch/replica_smoke_test.sh

@@ -1,80 +1,80 @@
-#!/usr/bin/env bash
-# UNILAG mounting dry-run — Phase G smoke tests (mounting guide §5).
-# Runs against the production replica stack.
-#
-# NOTE on cookies: the app sets SESSION_COOKIE_SECURE=True in production, so the
-# session cookie is only sent over HTTPS. All authenticated flows therefore go
-# through the nginx TLS reverse proxy (https://localhost:8443, -k = self-signed),
-# which is the real production request path anyway. Unauthenticated checks use
-# the direct gunicorn port (18080) to prove the app itself is hardened.
-set -u
-
-APP=http://localhost:18080          # gunicorn app (direct, bypasses nginx)
-TLS=https://localhost:8443          # nginx TLS reverse proxy (prod path)
-J=/tmp/uraas_cookies.txt
-PASS=0; FAIL=0
-ok(){ echo "  PASS: $1"; PASS=$((PASS+1)); }
-no(){ echo "  FAIL: $1"; FAIL=$((FAIL+1)); }
-CURL="curl -sk"                     # -k: trust the self-signed dry-run cert
-
-echo "=== 1. /health returns 200 (app direct + nginx TLS) ==="
-code=$($CURL -o /dev/null -w '%{http_code}' $APP/health)
-[ "$code" = "200" ] && ok "app /health = 200" || no "app /health = $code"
-code=$($CURL -o /dev/null -w '%{http_code}' $TLS/health)
-[ "$code" = "200" ] && ok "nginx TLS /health = 200" || no "nginx TLS /health = $code"
-
-echo "=== 2. HTTP -> HTTPS redirect (nginx :8081) ==="
-loc=$($CURL -o /dev/null -w '%{http_code} %{redirect_url}' http://localhost:8081/)
-echo "    $loc"
-echo "$loc" | grep -q "301" && ok "HTTP returns 301 redirect to https" || no "no 301 redirect: $loc"
-
-echo "=== 3. Anonymous API control route -> 401 ==="
-code=$($CURL -o /dev/null -w '%{http_code}' $APP/api/crawler/status)
-[ "$code" = "401" ] && ok "anon /api/crawler/status = 401" || no "anon crawler status = $code"
-code=$($CURL -o /dev/null -w '%{http_code}' $APP/api/analytics/overview)
-[ "$code" = "401" ] && ok "anon /api/analytics/overview = 401" || no "anon analytics = $code"
-code=$($CURL -o /dev/null -w '%{http_code}' $APP/api/staff/directory)
-echo "    (staff directory PII, anon): $code"
-[ "$code" = "401" ] && ok "anon staff directory (PII) = 401" || no "anon staff directory = $code"
-
-echo "=== 4. Anonymous HTML route -> redirect to /login ==="
-loc=$($CURL -o /dev/null -w '%{http_code} %{redirect_url}' "$APP/")
-echo "    $loc"
-echo "$loc" | grep -qE "302.*/login" && ok "anon / redirects to /login" || no "anon / = $loc"
-
-echo "=== 5. Security headers (CSP, XFO, nosniff, HSTS over TLS) ==="
-h=$($CURL -D - -o /dev/null $APP/login)
-echo "$h" | grep -qi "Content-Security-Policy" && ok "CSP header present" || no "CSP missing"
-echo "$h" | grep -qi "X-Frame-Options: DENY" && ok "X-Frame-Options DENY" || no "XFO missing"
-echo "$h" | grep -qi "X-Content-Type-Options: nosniff" && ok "nosniff present" || no "nosniff missing"
-echo "$h" | grep -qi "Strict-Transport-Security" && ok "HSTS present (prod)" || no "HSTS missing"
-ht=$($CURL -D - -o /dev/null $TLS/login)
-echo "$ht" | grep -qi "Strict-Transport-Security" && ok "HSTS present via nginx TLS" || no "HSTS via nginx missing"
-
-echo "=== 6. Bad login -> 401, no session granted ==="
-code=$($CURL -o /dev/null -w '%{http_code}' -d "username=admin&password=wrong" $TLS/login)
-[ "$code" = "401" ] && ok "bad login = 401" || no "bad login = $code"
-
-echo "=== 7. Admin login works + reaches admin-only route (over TLS) ==="
-rm -f $J
-code=$($CURL -o /dev/null -w '%{http_code}' -c $J -d "username=admin&password=UnilagAdmin#2026" $TLS/login)
-echo "    admin login status (302 expected): $code"
-[ "$code" = "302" ] && ok "admin login = 302" || no "admin login = $code"
-code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/crawler/status)
-[ "$code" = "200" ] && ok "admin reaches crawler status (200)" || no "admin crawler status = $code"
-
-echo "=== 8. Viewer login works but is NOT admin (crawler -> 403) ==="
-rm -f $J
-$CURL -o /dev/null -c $J -d "username=viewer&password=UnilagView#2026" $TLS/login
-code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/crawler/status)
-[ "$code" = "403" ] && ok "viewer crawler status = 403 (admin-only)" || no "viewer crawler status = $code"
-code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/analytics/overview)
-[ "$code" = "200" ] && ok "viewer reads analytics (200)" || no "viewer analytics = $code"
-code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/staff/directory)
-[ "$code" = "403" ] && ok "viewer staff directory (PII) = 403" || no "viewer staff directory = $code"
-
-echo
-echo "================= SMOKE TEST SUMMARY ================="
-echo "  PASSED: $PASS    FAILED: $FAIL"
-echo "====================================================="
-[ "$FAIL" = "0" ] && exit 0 || exit 1
+#!/usr/bin/env bash
+# UNILAG mounting dry-run — Phase G smoke tests (mounting guide §5).
+# Runs against the production replica stack.
+#
+# NOTE on cookies: the app sets SESSION_COOKIE_SECURE=True in production, so the
+# session cookie is only sent over HTTPS. All authenticated flows therefore go
+# through the nginx TLS reverse proxy (https://localhost:8443, -k = self-signed),
+# which is the real production request path anyway. Unauthenticated checks use
+# the direct gunicorn port (18080) to prove the app itself is hardened.
+set -u
+
+APP=http://localhost:18080          # gunicorn app (direct, bypasses nginx)
+TLS=https://localhost:8443          # nginx TLS reverse proxy (prod path)
+J=/tmp/uraas_cookies.txt
+PASS=0; FAIL=0
+ok(){ echo "  PASS: $1"; PASS=$((PASS+1)); }
+no(){ echo "  FAIL: $1"; FAIL=$((FAIL+1)); }
+CURL="curl -sk"                     # -k: trust the self-signed dry-run cert
+
+echo "=== 1. /health returns 200 (app direct + nginx TLS) ==="
+code=$($CURL -o /dev/null -w '%{http_code}' $APP/health)
+[ "$code" = "200" ] && ok "app /health = 200" || no "app /health = $code"
+code=$($CURL -o /dev/null -w '%{http_code}' $TLS/health)
+[ "$code" = "200" ] && ok "nginx TLS /health = 200" || no "nginx TLS /health = $code"
+
+echo "=== 2. HTTP -> HTTPS redirect (nginx :8081) ==="
+loc=$($CURL -o /dev/null -w '%{http_code} %{redirect_url}' http://localhost:8081/)
+echo "    $loc"
+echo "$loc" | grep -q "301" && ok "HTTP returns 301 redirect to https" || no "no 301 redirect: $loc"
+
+echo "=== 3. Anonymous API control route -> 401 ==="
+code=$($CURL -o /dev/null -w '%{http_code}' $APP/api/crawler/status)
+[ "$code" = "401" ] && ok "anon /api/crawler/status = 401" || no "anon crawler status = $code"
+code=$($CURL -o /dev/null -w '%{http_code}' $APP/api/analytics/overview)
+[ "$code" = "401" ] && ok "anon /api/analytics/overview = 401" || no "anon analytics = $code"
+code=$($CURL -o /dev/null -w '%{http_code}' $APP/api/staff/directory)
+echo "    (staff directory PII, anon): $code"
+[ "$code" = "401" ] && ok "anon staff directory (PII) = 401" || no "anon staff directory = $code"
+
+echo "=== 4. Anonymous HTML route -> redirect to /login ==="
+loc=$($CURL -o /dev/null -w '%{http_code} %{redirect_url}' "$APP/")
+echo "    $loc"
+echo "$loc" | grep -qE "302.*/login" && ok "anon / redirects to /login" || no "anon / = $loc"
+
+echo "=== 5. Security headers (CSP, XFO, nosniff, HSTS over TLS) ==="
+h=$($CURL -D - -o /dev/null $APP/login)
+echo "$h" | grep -qi "Content-Security-Policy" && ok "CSP header present" || no "CSP missing"
+echo "$h" | grep -qi "X-Frame-Options: DENY" && ok "X-Frame-Options DENY" || no "XFO missing"
+echo "$h" | grep -qi "X-Content-Type-Options: nosniff" && ok "nosniff present" || no "nosniff missing"
+echo "$h" | grep -qi "Strict-Transport-Security" && ok "HSTS present (prod)" || no "HSTS missing"
+ht=$($CURL -D - -o /dev/null $TLS/login)
+echo "$ht" | grep -qi "Strict-Transport-Security" && ok "HSTS present via nginx TLS" || no "HSTS via nginx missing"
+
+echo "=== 6. Bad login -> 401, no session granted ==="
+code=$($CURL -o /dev/null -w '%{http_code}' -d "username=admin&password=wrong" $TLS/login)
+[ "$code" = "401" ] && ok "bad login = 401" || no "bad login = $code"
+
+echo "=== 7. Admin login works + reaches admin-only route (over TLS) ==="
+rm -f $J
+code=$($CURL -o /dev/null -w '%{http_code}' -c $J -d "username=admin&password=UnilagAdmin#2026" $TLS/login)
+echo "    admin login status (302 expected): $code"
+[ "$code" = "302" ] && ok "admin login = 302" || no "admin login = $code"
+code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/crawler/status)
+[ "$code" = "200" ] && ok "admin reaches crawler status (200)" || no "admin crawler status = $code"
+
+echo "=== 8. Viewer login works but is NOT admin (crawler -> 403) ==="
+rm -f $J
+$CURL -o /dev/null -c $J -d "username=viewer&password=UnilagView#2026" $TLS/login
+code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/crawler/status)
+[ "$code" = "403" ] && ok "viewer crawler status = 403 (admin-only)" || no "viewer crawler status = $code"
+code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/analytics/overview)
+[ "$code" = "200" ] && ok "viewer reads analytics (200)" || no "viewer analytics = $code"
+code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/staff/directory)
+[ "$code" = "403" ] && ok "viewer staff directory (PII) = 403" || no "viewer staff directory = $code"
+
+echo
+echo "================= SMOKE TEST SUMMARY ================="
+echo "  PASSED: $PASS    FAILED: $FAIL"
+echo "====================================================="
+[ "$FAIL" = "0" ] && exit 0 || exit 1

scratch/verify_rors.py

@@ -1,39 +1,39 @@
-"""
-Script to verify all institution RORs against OpenAlex API.
-Identifies wrong RORs by checking if count > 0.
-"""
-import json
-import time
-import urllib.request
-from pathlib import Path
-
-BASE = "https://api.openalex.org"
-MAILTO = "cokiki@unilag.edu.ng"
-
-config_dir = Path(__file__).parent.parent / "config" / "institutions"
-results = {}
-
-for jf in sorted(config_dir.glob("*.json")):
-    with open(jf) as f:
-        data = json.load(f)
-    name = data["name"]
-    ror_full = data["ror"]
-    ror_short = ror_full.split("/")[-1]
-
-    url = f"{BASE}/works?filter=institutions.ror:{ror_short}&select=id&per-page=1&mailto={MAILTO}"
-    try:
-        req = urllib.request.urlopen(url, timeout=10)
-        resp = json.loads(req.read())
-        count = resp["meta"]["count"]
-    except Exception as e:
-        count = f"ERROR: {e}"
-
-    status = "OK" if isinstance(count, int) and count > 0 else "ZERO/ERROR"
-    print(f"[{status:5}] {name:40s}  ROR: {ror_short}  count={count}")
-    results[jf.name] = {"name": name, "ror": ror_short, "count": count, "status": status}
-    time.sleep(0.5)
-
-print("\n--- PROBLEM INSTITUTIONS ---")
-for fname, r in results.items():
-    if r["status"] != "OK":
-        print(f"  {fname}: {r['name']} -- ROR {r['ror']} returns {r['count']}")
+"""
+Script to verify all institution RORs against OpenAlex API.
+Identifies wrong RORs by checking if count > 0.
+"""
+import json
+import time
+import urllib.request
+from pathlib import Path
+
+BASE = "https://api.openalex.org"
+MAILTO = "cokiki@unilag.edu.ng"
+
+config_dir = Path(__file__).parent.parent / "config" / "institutions"
+results = {}
+
+for jf in sorted(config_dir.glob("*.json")):
+    with open(jf) as f:
+        data = json.load(f)
+    name = data["name"]
+    ror_full = data["ror"]
+    ror_short = ror_full.split("/")[-1]
+
+    url = f"{BASE}/works?filter=institutions.ror:{ror_short}&select=id&per-page=1&mailto={MAILTO}"
+    try:
+        req = urllib.request.urlopen(url, timeout=10)
+        resp = json.loads(req.read())
+        count = resp["meta"]["count"]
+    except Exception as e:
+        count = f"ERROR: {e}"
+
+    status = "OK" if isinstance(count, int) and count > 0 else "ZERO/ERROR"
+    print(f"[{status:5}] {name:40s}  ROR: {ror_short}  count={count}")
+    results[jf.name] = {"name": name, "ror": ror_short, "count": count, "status": status}
+    time.sleep(0.5)
+
+print("\n--- PROBLEM INSTITUTIONS ---")
+for fname, r in results.items():
+    if r["status"] != "OK":
+        print(f"  {fname}: {r['name']} -- ROR {r['ror']} returns {r['count']}")

scripts/backfill_alignment.py

@@ -1,99 +1,99 @@
-"""
-Backfill framework alignment scores for existing items and rebuild the
-AlignmentAggregate table (per institution + global).
-
-Usage:
-    python scripts/backfill_alignment.py            # DRY RUN — counts only
-    python scripts/backfill_alignment.py --apply
-    python scripts/backfill_alignment.py --apply --force   # re-score current-version items
-
-Safe to re-run: items already at ALIGNMENT_VERSION are skipped unless --force.
-No network needed beyond the one-time embedding-model download.
-"""
-
-import argparse
-import json
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from uraas.config.alignment_frameworks import ALIGNMENT_VERSION
-from uraas.database import Item, SessionLocal
-from uraas.services.alignment_engine import (
-    recompute_aggregates,
-    score_item_alignment,
-    scoring_mode,
-)
-from uraas.utils.analytics_cache import analytics_cache
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)")
-    parser.add_argument("--force", action="store_true", help="Re-score items already at current version")
-    parser.add_argument("--batch", type=int, default=500)
-    args = parser.parse_args()
-
-    session = SessionLocal()
-    try:
-        q = session.query(Item)
-        if not args.force:
-            q = q.filter(
-                (Item.alignment_version.is_(None))
-                | (Item.alignment_version < ALIGNMENT_VERSION)
-            )
-        todo = q.count()
-        total = session.query(Item).count()
-        print("=" * 64)
-        print(f"Scoring mode: {scoring_mode()}  |  version: {ALIGNMENT_VERSION}")
-        print(f"Items to score: {todo} / {total}")
-        print("=" * 64)
-        if not args.apply:
-            print("[DRY RUN] No writes. Re-run with --apply.")
-            return 0
-
-        scored = aligned = 0
-        framework_hits = {}
-        while True:
-            batch = q.limit(args.batch).all()
-            if not batch:
-                break
-            for it in batch:
-                j, v = score_item_alignment(
-                    it.title or "", it.abstract or "", it.dc_subject or ""
-                )
-                it.alignment_scores = j
-                it.alignment_version = v
-                scored += 1
-                if j:
-                    aligned += 1
-                    for fk in json.loads(j):
-                        framework_hits[fk] = framework_hits.get(fk, 0) + 1
-            session.commit()
-            print(f"  scored {scored}/{todo}")
-
-        print("\nPer-framework items with alignment:")
-        for fk, n in sorted(framework_hits.items(), key=lambda kv: -kv[1]):
-            print(f"  {fk:24s} {n}")
-
-        # Aggregates: global + each distinct institution
-        rows = recompute_aggregates(session, None)
-        institutions = [
-            i for (i,) in session.query(Item.institution).distinct() if i
-        ]
-        for inst in institutions:
-            rows += recompute_aggregates(session, inst)
-        print(f"\nAggregate rows written: {rows} ({1 + len(institutions)} scopes)")
-
-        analytics_cache.invalidate_all()
-        print("\n" + "=" * 64)
-        print(f"DONE. scored={scored}  with_alignment={aligned}")
-        print("=" * 64)
-        return 0
-    finally:
-        session.close()
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+"""
+Backfill framework alignment scores for existing items and rebuild the
+AlignmentAggregate table (per institution + global).
+
+Usage:
+    python scripts/backfill_alignment.py            # DRY RUN — counts only
+    python scripts/backfill_alignment.py --apply
+    python scripts/backfill_alignment.py --apply --force   # re-score current-version items
+
+Safe to re-run: items already at ALIGNMENT_VERSION are skipped unless --force.
+No network needed beyond the one-time embedding-model download.
+"""
+
+import argparse
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from uraas.config.alignment_frameworks import ALIGNMENT_VERSION
+from uraas.database import Item, SessionLocal
+from uraas.services.alignment_engine import (
+    recompute_aggregates,
+    score_item_alignment,
+    scoring_mode,
+)
+from uraas.utils.analytics_cache import analytics_cache
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)")
+    parser.add_argument("--force", action="store_true", help="Re-score items already at current version")
+    parser.add_argument("--batch", type=int, default=500)
+    args = parser.parse_args()
+
+    session = SessionLocal()
+    try:
+        q = session.query(Item)
+        if not args.force:
+            q = q.filter(
+                (Item.alignment_version.is_(None))
+                | (Item.alignment_version < ALIGNMENT_VERSION)
+            )
+        todo = q.count()
+        total = session.query(Item).count()
+        print("=" * 64)
+        print(f"Scoring mode: {scoring_mode()}  |  version: {ALIGNMENT_VERSION}")
+        print(f"Items to score: {todo} / {total}")
+        print("=" * 64)
+        if not args.apply:
+            print("[DRY RUN] No writes. Re-run with --apply.")
+            return 0
+
+        scored = aligned = 0
+        framework_hits = {}
+        while True:
+            batch = q.limit(args.batch).all()
+            if not batch:
+                break
+            for it in batch:
+                j, v = score_item_alignment(
+                    it.title or "", it.abstract or "", it.dc_subject or ""
+                )
+                it.alignment_scores = j
+                it.alignment_version = v
+                scored += 1
+                if j:
+                    aligned += 1
+                    for fk in json.loads(j):
+                        framework_hits[fk] = framework_hits.get(fk, 0) + 1
+            session.commit()
+            print(f"  scored {scored}/{todo}")
+
+        print("\nPer-framework items with alignment:")
+        for fk, n in sorted(framework_hits.items(), key=lambda kv: -kv[1]):
+            print(f"  {fk:24s} {n}")
+
+        # Aggregates: global + each distinct institution
+        rows = recompute_aggregates(session, None)
+        institutions = [
+            i for (i,) in session.query(Item.institution).distinct() if i
+        ]
+        for inst in institutions:
+            rows += recompute_aggregates(session, inst)
+        print(f"\nAggregate rows written: {rows} ({1 + len(institutions)} scopes)")
+
+        analytics_cache.invalidate_all()
+        print("\n" + "=" * 64)
+        print(f"DONE. scored={scored}  with_alignment={aligned}")
+        print("=" * 64)
+        return 0
+    finally:
+        session.close()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/backfill_citation_velocity.py

@@ -1,81 +1,81 @@
-"""
-Backfill Pan-African citation share (and citation velocity for items missed
-by the collaboration backfill).
-
-The share is one OpenAlex request per item (filter=cites:W... grouped by
-citing-country), so run it for the most-cited works first:
-
-    python scripts/backfill_citation_velocity.py                  # DRY RUN
-    python scripts/backfill_citation_velocity.py --apply --limit 200
-
-Skips items whose share is already computed unless --force.
-"""
-
-import argparse
-import json
-import os
-import sys
-import time
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from uraas.database import Item, SessionLocal
-from uraas.services.citation_tracker import CitationTracker
-from uraas.utils.analytics_cache import analytics_cache
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)")
-    parser.add_argument("--limit", type=int, default=200, help="Max items (most-cited first)")
-    parser.add_argument("--force", action="store_true", help="Recompute existing shares")
-    args = parser.parse_args()
-
-    session = SessionLocal()
-    try:
-        q = (
-            session.query(Item)
-            .filter(Item.openalex_id.isnot(None), Item.cited_by_count > 0)
-            .order_by(Item.cited_by_count.desc())
-        )
-        if not args.force:
-            q = q.filter(Item.african_citation_share.is_(None))
-        items = q.limit(args.limit).all()
-
-        print("=" * 64)
-        print(f"Items to process (most-cited first): {len(items)}")
-        print("=" * 64)
-        if not args.apply:
-            print("[DRY RUN] No API calls or writes. Re-run with --apply.")
-            return 0
-
-        updated = velocity_fixed = 0
-        for i, it in enumerate(items, 1):
-            share = CitationTracker.fetch_african_citation_share(it.openalex_id)
-            if share is not None:
-                it.african_citation_share = share
-                updated += 1
-            # Opportunistic velocity fix for items missing counts_by_year
-            if not it.counts_by_year:
-                vel = CitationTracker.fetch_work_velocity(it.openalex_id)
-                if vel and vel["counts_by_year"]:
-                    it.counts_by_year = json.dumps(vel["counts_by_year"])
-                    it.cited_by_count = vel["cited_by_count"]
-                    velocity_fixed += 1
-            if i % 25 == 0:
-                session.commit()
-                print(f"  {i}/{len(items)} processed (share set: {updated})")
-            time.sleep(1.0)
-        session.commit()
-        analytics_cache.invalidate_all()
-
-        print("\n" + "=" * 64)
-        print(f"DONE. shares set={updated}  velocity backfilled={velocity_fixed}")
-        print("=" * 64)
-        return 0
-    finally:
-        session.close()
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+"""
+Backfill Pan-African citation share (and citation velocity for items missed
+by the collaboration backfill).
+
+The share is one OpenAlex request per item (filter=cites:W... grouped by
+citing-country), so run it for the most-cited works first:
+
+    python scripts/backfill_citation_velocity.py                  # DRY RUN
+    python scripts/backfill_citation_velocity.py --apply --limit 200
+
+Skips items whose share is already computed unless --force.
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from uraas.database import Item, SessionLocal
+from uraas.services.citation_tracker import CitationTracker
+from uraas.utils.analytics_cache import analytics_cache
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)")
+    parser.add_argument("--limit", type=int, default=200, help="Max items (most-cited first)")
+    parser.add_argument("--force", action="store_true", help="Recompute existing shares")
+    args = parser.parse_args()
+
+    session = SessionLocal()
+    try:
+        q = (
+            session.query(Item)
+            .filter(Item.openalex_id.isnot(None), Item.cited_by_count > 0)
+            .order_by(Item.cited_by_count.desc())
+        )
+        if not args.force:
+            q = q.filter(Item.african_citation_share.is_(None))
+        items = q.limit(args.limit).all()
+
+        print("=" * 64)
+        print(f"Items to process (most-cited first): {len(items)}")
+        print("=" * 64)
+        if not args.apply:
+            print("[DRY RUN] No API calls or writes. Re-run with --apply.")
+            return 0
+
+        updated = velocity_fixed = 0
+        for i, it in enumerate(items, 1):
+            share = CitationTracker.fetch_african_citation_share(it.openalex_id)
+            if share is not None:
+                it.african_citation_share = share
+                updated += 1
+            # Opportunistic velocity fix for items missing counts_by_year
+            if not it.counts_by_year:
+                vel = CitationTracker.fetch_work_velocity(it.openalex_id)
+                if vel and vel["counts_by_year"]:
+                    it.counts_by_year = json.dumps(vel["counts_by_year"])
+                    it.cited_by_count = vel["cited_by_count"]
+                    velocity_fixed += 1
+            if i % 25 == 0:
+                session.commit()
+                print(f"  {i}/{len(items)} processed (share set: {updated})")
+            time.sleep(1.0)
+        session.commit()
+        analytics_cache.invalidate_all()
+
+        print("\n" + "=" * 64)
+        print(f"DONE. shares set={updated}  velocity backfilled={velocity_fixed}")
+        print("=" * 64)
+        return 0
+    finally:
+        session.close()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/backfill_collaboration_data.py

@@ -1,227 +1,227 @@
-"""
-Backfill collaboration + citation data for existing items from OpenAlex.
-
-Re-fetches each item's OpenAlex record (batched 50 DOIs per request to
-conserve API quota) and populates:
-  - item_affiliations rows (institution / ROR / country per authorship)
-  - items.coauthor_countries / african_country_count / is_intra_african
-  - items.openalex_id / cited_by_count / counts_by_year
-
-Usage:
-    python scripts/backfill_collaboration_data.py                # DRY RUN
-    python scripts/backfill_collaboration_data.py --apply
-    python scripts/backfill_collaboration_data.py --apply --limit 500
-    python scripts/backfill_collaboration_data.py --apply --force   # redo enriched rows
-
-Idempotent: items that already have affiliation rows are skipped unless
---force. Respects ~1 req/sec. Set OPENALEX_API_KEY in the environment.
-"""
-
-import argparse
-import json
-import os
-import sys
-import time
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from uraas.config.african_countries import african_countries_in
-from uraas.database import Item, ItemAffiliation, SessionLocal
-from uraas.utils.analytics_cache import analytics_cache
-from uraas.utils.openalex_client import oa_get
-
-BATCH = 50
-SELECT = "id,doi,authorships,cited_by_count,counts_by_year"
-
-
-def _norm_doi(doi: str) -> str:
-    return (
-        (doi or "")
-        .replace("https://doi.org/", "")
-        .replace("http://dx.doi.org/", "")
-        .strip()
-        .lower()
-    )
-
-
-def fetch_batch_by_doi(dois):
-    """One OpenAlex call for up to 50 DOIs. Returns {normalized_doi: work}.
-
-    DOIs are pipe-joined raw — requests URL-encodes the whole filter param;
-    pre-quoting each DOI double-encodes and matches nothing."""
-    flt = "doi:" + "|".join(dois)
-    data = oa_get("/works", {"filter": flt, "select": SELECT, "per-page": BATCH})
-    out = {}
-    for work in (data or {}).get("results", []):
-        nd = _norm_doi(work.get("doi", ""))
-        if nd:
-            out[nd] = work
-    return out
-
-
-def fetch_batch_by_openalex_id(ids):
-    """One OpenAlex call for up to 50 OpenAlex work IDs."""
-    flt = "openalex_id:" + "|".join(ids)
-    data = oa_get("/works", {"filter": flt, "select": SELECT, "per-page": BATCH})
-    out = {}
-    for work in (data or {}).get("results", []):
-        wid = work.get("id", "").replace("https://openalex.org/", "")
-        if wid:
-            out[wid] = work
-    return out
-
-
-def extract_affiliations(work):
-    """(ror_short, name) -> {ror, name, country_code, author_count}."""
-    rows = {}
-    for authorship in work.get("authorships", []):
-        for inst in authorship.get("institutions", []):
-            name = inst.get("display_name", "") or ""
-            ror = (inst.get("ror") or "").replace("https://ror.org/", "")
-            cc = (inst.get("country_code") or "").upper()
-            if not (name or ror):
-                continue
-            row = rows.setdefault(
-                (ror, name),
-                {"ror": ror, "name": name, "country_code": cc, "author_count": 0},
-            )
-            row["author_count"] += 1
-            if cc and not row["country_code"]:
-                row["country_code"] = cc
-    return list(rows.values())
-
-
-def apply_work(session, item, work):
-    """Write affiliation rows + collaboration/citation columns for one item."""
-    affs = extract_affiliations(work)
-
-    # Idempotency: replace any existing affiliation rows for this item.
-    session.query(ItemAffiliation).filter_by(item_id=item.id).delete()
-    for aff in affs:
-        session.add(
-            ItemAffiliation(
-                item_id=item.id,
-                ror=(aff["ror"] or "")[:128] or None,
-                institution_name=(aff["name"] or "")[:255] or None,
-                country_code=(aff["country_code"] or "")[:2] or None,
-                author_count=aff["author_count"],
-            )
-        )
-
-    african = african_countries_in(a["country_code"] for a in affs)
-    item.coauthor_countries = ",".join(african) or None
-    item.african_country_count = len(african)
-    item.is_intra_african = len(african) >= 2
-
-    item.openalex_id = work.get("id", "").replace("https://openalex.org/", "") or None
-    item.cited_by_count = work.get("cited_by_count", 0) or 0
-    cby = work.get("counts_by_year") or []
-    item.counts_by_year = json.dumps(cby) if cby else None
-    return len(affs), item.is_intra_african
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)")
-    parser.add_argument("--limit", type=int, default=0, help="Max items to process (0 = all)")
-    parser.add_argument(
-        "--force", action="store_true", help="Re-fetch items that already have affiliation data"
-    )
-    args = parser.parse_args()
-
-    session = SessionLocal()
-    try:
-        q = session.query(Item)
-        if not args.force:
-            enriched = {i for (i,) in session.query(ItemAffiliation.item_id).distinct()}
-        else:
-            enriched = set()
-
-        items = [
-            it
-            for it in q.all()
-            if it.id not in enriched and (it.doi or "openalex.org" in (it.url or ""))
-        ]
-        skipped_no_id = q.count() - len(items) - len(enriched & {it.id for it in q})
-        if args.limit:
-            items = items[: args.limit]
-
-        print("=" * 64)
-        print(f"Items to enrich: {len(items)}  (already enriched, skipped: {len(enriched)})")
-        print("=" * 64)
-        if not args.apply:
-            print("[DRY RUN] No API calls or writes. Re-run with --apply.")
-            return 0
-
-        by_doi = [it for it in items if it.doi]
-        by_oaid = [
-            it for it in items if not it.doi and "openalex.org" in (it.url or "")
-        ]
-
-        updated = intra = not_found = 0
-
-        # ── DOI batches ──────────────────────────────────────────────────
-        doi_map = {_norm_doi(it.doi): it for it in by_doi}
-        doi_keys = list(doi_map)
-        for start in range(0, len(doi_keys), BATCH):
-            chunk = doi_keys[start : start + BATCH]
-            works = fetch_batch_by_doi(chunk)
-            for nd in chunk:
-                it = doi_map[nd]
-                work = works.get(nd)
-                if not work:
-                    not_found += 1
-                    continue
-                _, is_ia = apply_work(session, it, work)
-                updated += 1
-                intra += int(is_ia)
-            session.commit()
-            print(
-                f"  [doi {start + len(chunk)}/{len(doi_keys)}] "
-                f"updated={updated} intra_african={intra} not_found={not_found}"
-            )
-            time.sleep(1.0)
-
-        # ── OpenAlex-ID batches (items without DOI) ──────────────────────
-        oaid_map = {}
-        for it in by_oaid:
-            wid = (it.url or "").rstrip("/").split("/")[-1]
-            if wid.startswith("W"):
-                oaid_map[wid] = it
-        oaid_keys = list(oaid_map)
-        for start in range(0, len(oaid_keys), BATCH):
-            chunk = oaid_keys[start : start + BATCH]
-            works = fetch_batch_by_openalex_id(chunk)
-            for wid in chunk:
-                it = oaid_map[wid]
-                work = works.get(wid)
-                if not work:
-                    not_found += 1
-                    continue
-                _, is_ia = apply_work(session, it, work)
-                updated += 1
-                intra += int(is_ia)
-            session.commit()
-            print(
-                f"  [oaid {start + len(chunk)}/{len(oaid_keys)}] "
-                f"updated={updated} intra_african={intra} not_found={not_found}"
-            )
-            time.sleep(1.0)
-
-        analytics_cache.invalidate_all()
-        total_ia = session.query(Item).filter(Item.is_intra_african.is_(True)).count()
-        total = session.query(Item).count()
-        print("\n" + "=" * 64)
-        print(f"DONE. updated={updated}  not_found={not_found}")
-        print(
-            f"Repository intra-African collaboration: {total_ia}/{total} "
-            f"({(total_ia / total * 100) if total else 0:.1f}%) — continental baseline ~8.4%"
-        )
-        print("=" * 64)
-        return 0
-    finally:
-        session.close()
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+"""
+Backfill collaboration + citation data for existing items from OpenAlex.
+
+Re-fetches each item's OpenAlex record (batched 50 DOIs per request to
+conserve API quota) and populates:
+  - item_affiliations rows (institution / ROR / country per authorship)
+  - items.coauthor_countries / african_country_count / is_intra_african
+  - items.openalex_id / cited_by_count / counts_by_year
+
+Usage:
+    python scripts/backfill_collaboration_data.py                # DRY RUN
+    python scripts/backfill_collaboration_data.py --apply
+    python scripts/backfill_collaboration_data.py --apply --limit 500
+    python scripts/backfill_collaboration_data.py --apply --force   # redo enriched rows
+
+Idempotent: items that already have affiliation rows are skipped unless
+--force. Respects ~1 req/sec. Set OPENALEX_API_KEY in the environment.
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from uraas.config.african_countries import african_countries_in
+from uraas.database import Item, ItemAffiliation, SessionLocal
+from uraas.utils.analytics_cache import analytics_cache
+from uraas.utils.openalex_client import oa_get
+
+BATCH = 50
+SELECT = "id,doi,authorships,cited_by_count,counts_by_year"
+
+
+def _norm_doi(doi: str) -> str:
+    return (
+        (doi or "")
+        .replace("https://doi.org/", "")
+        .replace("http://dx.doi.org/", "")
+        .strip()
+        .lower()
+    )
+
+
+def fetch_batch_by_doi(dois):
+    """One OpenAlex call for up to 50 DOIs. Returns {normalized_doi: work}.
+
+    DOIs are pipe-joined raw — requests URL-encodes the whole filter param;
+    pre-quoting each DOI double-encodes and matches nothing."""
+    flt = "doi:" + "|".join(dois)
+    data = oa_get("/works", {"filter": flt, "select": SELECT, "per-page": BATCH})
+    out = {}
+    for work in (data or {}).get("results", []):
+        nd = _norm_doi(work.get("doi", ""))
+        if nd:
+            out[nd] = work
+    return out
+
+
+def fetch_batch_by_openalex_id(ids):
+    """One OpenAlex call for up to 50 OpenAlex work IDs."""
+    flt = "openalex_id:" + "|".join(ids)
+    data = oa_get("/works", {"filter": flt, "select": SELECT, "per-page": BATCH})
+    out = {}
+    for work in (data or {}).get("results", []):
+        wid = work.get("id", "").replace("https://openalex.org/", "")
+        if wid:
+            out[wid] = work
+    return out
+
+
+def extract_affiliations(work):
+    """(ror_short, name) -> {ror, name, country_code, author_count}."""
+    rows = {}
+    for authorship in work.get("authorships", []):
+        for inst in authorship.get("institutions", []):
+            name = inst.get("display_name", "") or ""
+            ror = (inst.get("ror") or "").replace("https://ror.org/", "")
+            cc = (inst.get("country_code") or "").upper()
+            if not (name or ror):
+                continue
+            row = rows.setdefault(
+                (ror, name),
+                {"ror": ror, "name": name, "country_code": cc, "author_count": 0},
+            )
+            row["author_count"] += 1
+            if cc and not row["country_code"]:
+                row["country_code"] = cc
+    return list(rows.values())
+
+
+def apply_work(session, item, work):
+    """Write affiliation rows + collaboration/citation columns for one item."""
+    affs = extract_affiliations(work)
+
+    # Idempotency: replace any existing affiliation rows for this item.
+    session.query(ItemAffiliation).filter_by(item_id=item.id).delete()
+    for aff in affs:
+        session.add(
+            ItemAffiliation(
+                item_id=item.id,
+                ror=(aff["ror"] or "")[:128] or None,
+                institution_name=(aff["name"] or "")[:255] or None,
+                country_code=(aff["country_code"] or "")[:2] or None,
+                author_count=aff["author_count"],
+            )
+        )
+
+    african = african_countries_in(a["country_code"] for a in affs)
+    item.coauthor_countries = ",".join(african) or None
+    item.african_country_count = len(african)
+    item.is_intra_african = len(african) >= 2
+
+    item.openalex_id = work.get("id", "").replace("https://openalex.org/", "") or None
+    item.cited_by_count = work.get("cited_by_count", 0) or 0
+    cby = work.get("counts_by_year") or []
+    item.counts_by_year = json.dumps(cby) if cby else None
+    return len(affs), item.is_intra_african
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)")
+    parser.add_argument("--limit", type=int, default=0, help="Max items to process (0 = all)")
+    parser.add_argument(
+        "--force", action="store_true", help="Re-fetch items that already have affiliation data"
+    )
+    args = parser.parse_args()
+
+    session = SessionLocal()
+    try:
+        q = session.query(Item)
+        if not args.force:
+            enriched = {i for (i,) in session.query(ItemAffiliation.item_id).distinct()}
+        else:
+            enriched = set()
+
+        items = [
+            it
+            for it in q.all()
+            if it.id not in enriched and (it.doi or "openalex.org" in (it.url or ""))
+        ]
+        skipped_no_id = q.count() - len(items) - len(enriched & {it.id for it in q})
+        if args.limit:
+            items = items[: args.limit]
+
+        print("=" * 64)
+        print(f"Items to enrich: {len(items)}  (already enriched, skipped: {len(enriched)})")
+        print("=" * 64)
+        if not args.apply:
+            print("[DRY RUN] No API calls or writes. Re-run with --apply.")
+            return 0
+
+        by_doi = [it for it in items if it.doi]
+        by_oaid = [
+            it for it in items if not it.doi and "openalex.org" in (it.url or "")
+        ]
+
+        updated = intra = not_found = 0
+
+        # ── DOI batches ──────────────────────────────────────────────────
+        doi_map = {_norm_doi(it.doi): it for it in by_doi}
+        doi_keys = list(doi_map)
+        for start in range(0, len(doi_keys), BATCH):
+            chunk = doi_keys[start : start + BATCH]
+            works = fetch_batch_by_doi(chunk)
+            for nd in chunk:
+                it = doi_map[nd]
+                work = works.get(nd)
+                if not work:
+                    not_found += 1
+                    continue
+                _, is_ia = apply_work(session, it, work)
+                updated += 1
+                intra += int(is_ia)
+            session.commit()
+            print(
+                f"  [doi {start + len(chunk)}/{len(doi_keys)}] "
+                f"updated={updated} intra_african={intra} not_found={not_found}"
+            )
+            time.sleep(1.0)
+
+        # ── OpenAlex-ID batches (items without DOI) ──────────────────────
+        oaid_map = {}
+        for it in by_oaid:
+            wid = (it.url or "").rstrip("/").split("/")[-1]
+            if wid.startswith("W"):
+                oaid_map[wid] = it
+        oaid_keys = list(oaid_map)
+        for start in range(0, len(oaid_keys), BATCH):
+            chunk = oaid_keys[start : start + BATCH]
+            works = fetch_batch_by_openalex_id(chunk)
+            for wid in chunk:
+                it = oaid_map[wid]
+                work = works.get(wid)
+                if not work:
+                    not_found += 1
+                    continue
+                _, is_ia = apply_work(session, it, work)
+                updated += 1
+                intra += int(is_ia)
+            session.commit()
+            print(
+                f"  [oaid {start + len(chunk)}/{len(oaid_keys)}] "
+                f"updated={updated} intra_african={intra} not_found={not_found}"
+            )
+            time.sleep(1.0)
+
+        analytics_cache.invalidate_all()
+        total_ia = session.query(Item).filter(Item.is_intra_african.is_(True)).count()
+        total = session.query(Item).count()
+        print("\n" + "=" * 64)
+        print(f"DONE. updated={updated}  not_found={not_found}")
+        print(
+            f"Repository intra-African collaboration: {total_ia}/{total} "
+            f"({(total_ia / total * 100) if total else 0:.1f}%) — continental baseline ~8.4%"
+        )
+        print("=" * 64)
+        return 0
+    finally:
+        session.close()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/backfill_pids.py

@@ -1,70 +1,70 @@
-"""
-Backfill persistent identifiers: DocID™ + ARK for items missing them.
-
-ARKs are minted deterministically from the item's DocID hash, so re-running
-is idempotent. No network access needed.
-
-Usage:
-    python scripts/backfill_pids.py            # DRY RUN
-    python scripts/backfill_pids.py --apply
-"""
-
-import argparse
-import os
-import sys
-from datetime import datetime
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from uraas.database import Item, SessionLocal
-from uraas.utils.ark_generator import ark_generator
-from uraas.utils.docid_generator import docid_generator
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)")
-    args = parser.parse_args()
-
-    session = SessionLocal()
-    try:
-        need_docid = session.query(Item).filter(Item.docid.is_(None)).count()
-        need_ark = session.query(Item).filter(Item.ark.is_(None)).count()
-        print("=" * 64)
-        print(f"Items missing DocID: {need_docid}   missing ARK: {need_ark}")
-        print("=" * 64)
-        if not args.apply:
-            print("[DRY RUN] No writes. Re-run with --apply.")
-            return 0
-
-        minted_docid = minted_ark = 0
-        now = datetime.utcnow()
-        for it in session.query(Item).filter(
-            (Item.docid.is_(None)) | (Item.ark.is_(None))
-        ):
-            if not it.docid:
-                it.docid = docid_generator.generate_docid(
-                    title=it.title or "",
-                    doi=it.doi,
-                    institution=it.institution or "Unknown",
-                    timestamp=it.publication_date,
-                )
-                it.docid_assigned_at = now
-                minted_docid += 1
-            if not it.ark:
-                it.ark = ark_generator.mint(it.docid)
-                it.ark_assigned_at = now
-                minted_ark += 1
-        session.commit()
-
-        print(f"DONE. DocIDs minted: {minted_docid}   ARKs minted: {minted_ark}")
-        sample = session.query(Item.ark).filter(Item.ark.isnot(None)).first()
-        if sample:
-            print(f"Sample ARK: {sample[0]}  (valid: {ark_generator.validate(sample[0])})")
-        return 0
-    finally:
-        session.close()
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+"""
+Backfill persistent identifiers: DocID™ + ARK for items missing them.
+
+ARKs are minted deterministically from the item's DocID hash, so re-running
+is idempotent. No network access needed.
+
+Usage:
+    python scripts/backfill_pids.py            # DRY RUN
+    python scripts/backfill_pids.py --apply
+"""
+
+import argparse
+import os
+import sys
+from datetime import datetime
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from uraas.database import Item, SessionLocal
+from uraas.utils.ark_generator import ark_generator
+from uraas.utils.docid_generator import docid_generator
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)")
+    args = parser.parse_args()
+
+    session = SessionLocal()
+    try:
+        need_docid = session.query(Item).filter(Item.docid.is_(None)).count()
+        need_ark = session.query(Item).filter(Item.ark.is_(None)).count()
+        print("=" * 64)
+        print(f"Items missing DocID: {need_docid}   missing ARK: {need_ark}")
+        print("=" * 64)
+        if not args.apply:
+            print("[DRY RUN] No writes. Re-run with --apply.")
+            return 0
+
+        minted_docid = minted_ark = 0
+        now = datetime.utcnow()
+        for it in session.query(Item).filter(
+            (Item.docid.is_(None)) | (Item.ark.is_(None))
+        ):
+            if not it.docid:
+                it.docid = docid_generator.generate_docid(
+                    title=it.title or "",
+                    doi=it.doi,
+                    institution=it.institution or "Unknown",
+                    timestamp=it.publication_date,
+                )
+                it.docid_assigned_at = now
+                minted_docid += 1
+            if not it.ark:
+                it.ark = ark_generator.mint(it.docid)
+                it.ark_assigned_at = now
+                minted_ark += 1
+        session.commit()
+
+        print(f"DONE. DocIDs minted: {minted_docid}   ARKs minted: {minted_ark}")
+        sample = session.query(Item.ark).filter(Item.ark.isnot(None)).first()
+        if sample:
+            print(f"Sample ARK: {sample[0]}  (valid: {ark_generator.validate(sample[0])})")
+        return 0
+    finally:
+        session.close()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/backfill_special_collections.py

@@ -1,69 +1,69 @@
-"""
-Backfill special_collection_score + special_collection_categories on existing items.
-
-Runs classify_special_collections() over every Item (title + abstract + dc_subject)
-and writes the score/categories. Idempotent — re-running on already-scored rows
-produces the same values.
-"""
-
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from uraas.database import Item, SessionLocal
-from uraas.utils.ai_classifier import classify_special_collections
-
-BATCH_SIZE = 500
-
-
-def main() -> int:
-    session = SessionLocal()
-    try:
-        total = session.query(Item).count()
-        print(f"Backfilling SC score for {total} items...")
-
-        scored = 0
-        hits = 0
-        offset = 0
-        while offset < total:
-            batch = (
-                session.query(Item)
-                .order_by(Item.id)
-                .offset(offset)
-                .limit(BATCH_SIZE)
-                .all()
-            )
-            if not batch:
-                break
-
-            for item in batch:
-                sc = classify_special_collections(
-                    item.title or "",
-                    item.abstract or "",
-                    item.dc_subject or "",
-                )
-                if sc:
-                    item.special_collection_score = float(sum(h["score"] for h in sc))
-                    item.special_collection_categories = ",".join(
-                        h["category"] for h in sc
-                    )
-                    hits += 1
-                else:
-                    item.special_collection_score = 0.0
-                    item.special_collection_categories = ""
-                scored += 1
-
-            session.commit()
-            offset += len(batch)
-            print(f"  {scored}/{total} scored ({hits} SC hits so far)")
-
-        print()
-        print(f"Done. {scored} items scored, {hits} matched a special collection.")
-        return 0
-    finally:
-        session.close()
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+"""
+Backfill special_collection_score + special_collection_categories on existing items.
+
+Runs classify_special_collections() over every Item (title + abstract + dc_subject)
+and writes the score/categories. Idempotent — re-running on already-scored rows
+produces the same values.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from uraas.database import Item, SessionLocal
+from uraas.utils.ai_classifier import classify_special_collections
+
+BATCH_SIZE = 500
+
+
+def main() -> int:
+    session = SessionLocal()
+    try:
+        total = session.query(Item).count()
+        print(f"Backfilling SC score for {total} items...")
+
+        scored = 0
+        hits = 0
+        offset = 0
+        while offset < total:
+            batch = (
+                session.query(Item)
+                .order_by(Item.id)
+                .offset(offset)
+                .limit(BATCH_SIZE)
+                .all()
+            )
+            if not batch:
+                break
+
+            for item in batch:
+                sc = classify_special_collections(
+                    item.title or "",
+                    item.abstract or "",
+                    item.dc_subject or "",
+                )
+                if sc:
+                    item.special_collection_score = float(sum(h["score"] for h in sc))
+                    item.special_collection_categories = ",".join(
+                        h["category"] for h in sc
+                    )
+                    hits += 1
+                else:
+                    item.special_collection_score = 0.0
+                    item.special_collection_categories = ""
+                scored += 1
+
+            session.commit()
+            offset += len(batch)
+            print(f"  {scored}/{total} scored ({hits} SC hits so far)")
+
+        print()
+        print(f"Done. {scored} items scored, {hits} matched a special collection.")
+        return 0
+    finally:
+        session.close()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/build_app.py

@@ -1,26 +1,26 @@
-import os
-import sys
-
-sys.path.insert(0, ".")
-
-APP = """import os,csv,io,subprocess,threading,re,logging
-from flask import Flask,render_template,jsonify,send_file,request,Response
-from flask_socketio import SocketIO
-from uraas.config import config
-from uraas.analytics.engine import analytics
-from uraas.database import SessionLocal,Item,File,Author,Community,Collection
-from uraas.utils.docid_generator import docid_generator
-from sqlalchemy import func,extract,desc,or_
-
-app = Flask(__name__)
-app.config["SECRET_KEY"] = config.DASHBOARD_SECRET_KEY
-socketio = SocketIO(app, cors_allowed_origins="*")
-logger = logging.getLogger(__name__)
-crawler_process = None
-crawler_lock = threading.Lock()
-docid_crawler_process = None
-docid_crawler_lock = threading.Lock()
-"""
-
-open("uraas/dashboard/app.py", "w", encoding="utf-8").write(APP)
-print("wrote", len(APP), "chars")
+import os
+import sys
+
+sys.path.insert(0, ".")
+
+APP = """import os,csv,io,subprocess,threading,re,logging
+from flask import Flask,render_template,jsonify,send_file,request,Response
+from flask_socketio import SocketIO
+from uraas.config import config
+from uraas.analytics.engine import analytics
+from uraas.database import SessionLocal,Item,File,Author,Community,Collection
+from uraas.utils.docid_generator import docid_generator
+from sqlalchemy import func,extract,desc,or_
+
+app = Flask(__name__)
+app.config["SECRET_KEY"] = config.DASHBOARD_SECRET_KEY
+socketio = SocketIO(app, cors_allowed_origins="*")
+logger = logging.getLogger(__name__)
+crawler_process = None
+crawler_lock = threading.Lock()
+docid_crawler_process = None
+docid_crawler_lock = threading.Lock()
+"""
+
+open("uraas/dashboard/app.py", "w", encoding="utf-8").write(APP)
+print("wrote", len(APP), "chars")

scripts/check_staff.py

@@ -1,9 +1,9 @@
-import os
-import sys
-
-sys.path.insert(0, os.getcwd())
-from uraas.config.institutions import get_registry
-
-registry = get_registry()
-for inst in registry.list_all():
-    print(f"{inst.short_name}: {len(inst.staff_names)} staff (File: {inst.staff_file})")
+import os
+import sys
+
+sys.path.insert(0, os.getcwd())
+from uraas.config.institutions import get_registry
+
+registry = get_registry()
+for inst in registry.list_all():
+    print(f"{inst.short_name}: {len(inst.staff_names)} staff (File: {inst.staff_file})")

scripts/clean_database.py

@@ -1,156 +1,156 @@
-"""
-Database Cleanup Script
-Removes bad/misattributed data from the URAAS database.
-
-Cleanup rules:
-1. Remove items from institutions no longer in the registry
-2. Remove items with no title, no DOI, no URL, and no authors
-3. Remove exact DOI duplicates (keep first by id)
-4. Remove items whose title is fewer than 10 chars
-5. Flag (do not delete) items with institution mismatch in affiliation
-6. Report before/after counts
-"""
-
-import logging
-import os
-import sys
-from datetime import datetime
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
-log = logging.getLogger(__name__)
-
-
-def run_cleanup(dry_run: bool = False):
-    from sqlalchemy import func
-
-    from uraas.config.institutions import get_registry
-    from uraas.database import Author, Collection, Community, Item, SessionLocal
-
-    registry = get_registry()
-    valid_inst_names = {c.name for c in registry.list_all()}
-
-    session = SessionLocal()
-    try:
-        total_before = session.query(Item).count()
-        log.info(f"Starting cleanup | Items before: {total_before}")
-
-        removed = 0
-
-        # ── Rule 1: Remove items from removed institutions ────────────────────
-        all_institutions_in_db = session.query(Item.institution).distinct().all()
-        stale_insts = [
-            r[0]
-            for r in all_institutions_in_db
-            if r[0] and r[0] not in valid_inst_names
-        ]
-
-        if stale_insts:
-            log.info(f"Found stale institutions: {stale_insts}")
-            for stale in stale_insts:
-                stale_items = (
-                    session.query(Item).filter(Item.institution == stale).all()
-                )
-                log.info(f"  [{stale}] {len(stale_items)} items to remove")
-                if not dry_run:
-                    for item in stale_items:
-                        session.delete(item)
-                    session.commit()
-                removed += len(stale_items)
-
-        # ── Rule 2: Remove items with no title ────────────────────────────────
-        no_title = (
-            session.query(Item)
-            .filter(
-                (Item.title == None) | (Item.title == "") | (Item.title == "Untitled")
-            )
-            .all()
-        )
-        log.info(f"Items with no/empty title: {len(no_title)}")
-        if not dry_run:
-            for item in no_title:
-                session.delete(item)
-            session.commit()
-        removed += len(no_title)
-
-        # ── Rule 3: Remove items with title < 10 chars and no DOI ────────────
-        all_short = session.query(Item).filter(Item.doi == None).all()
-        short_items = [i for i in all_short if i.title and len(i.title.strip()) < 10]
-        log.info(
-            f"Items with very short title (<10 chars) and no DOI: {len(short_items)}"
-        )
-        if not dry_run:
-            for item in short_items:
-                session.delete(item)
-            session.commit()
-        removed += len(short_items)
-
-        # ── Rule 4: Remove exact DOI duplicates (keep lowest id) ─────────────
-        doi_subq = (
-            session.query(Item.doi, func.min(Item.id).label("min_id"))
-            .filter(Item.doi != None)
-            .group_by(Item.doi)
-            .subquery()
-        )
-
-        dup_dois = (
-            session.query(Item)
-            .filter(Item.doi != None, Item.id.notin_(session.query(doi_subq.c.min_id)))
-            .all()
-        )
-        log.info(f"Duplicate DOI items to remove: {len(dup_dois)}")
-        if not dry_run:
-            for item in dup_dois:
-                session.delete(item)
-            session.commit()
-        removed += len(dup_dois)
-
-        # ── Rule 5: Remove items with no institution tag ──────────────────────
-        no_inst = (
-            session.query(Item)
-            .filter((Item.institution == None) | (Item.institution == ""))
-            .all()
-        )
-        # Only remove those that have no authors either
-        truly_orphan = [i for i in no_inst if not i.authors]
-        log.info(f"Items with no institution AND no authors: {len(truly_orphan)}")
-        if not dry_run:
-            for item in truly_orphan:
-                session.delete(item)
-            session.commit()
-        removed += len(truly_orphan)
-
-        total_after = session.query(Item).count()
-
-        log.info(f"\n{'='*50}")
-        log.info(f"CLEANUP COMPLETE {'(DRY RUN)' if dry_run else ''}")
-        log.info(f"  Items before: {total_before}")
-        log.info(f"  Items removed: {removed}")
-        log.info(f"  Items after:  {total_after}")
-        log.info(f"{'='*50}")
-
-        return {
-            "total_before": total_before,
-            "removed": removed,
-            "total_after": total_after,
-            "dry_run": dry_run,
-        }
-
-    except Exception as e:
-        log.error(f"Cleanup error: {e}")
-        session.rollback()
-        raise
-    finally:
-        session.close()
-
-
-if __name__ == "__main__":
-    import argparse
-
-    parser = argparse.ArgumentParser(description="URAAS Database Cleanup")
-    parser.add_argument(
-        "--dry-run", action="store_true", help="Report without deleting"
-    )
-    args = parser.parse_args()
-    run_cleanup(dry_run=args.dry_run)
+"""
+Database Cleanup Script
+Removes bad/misattributed data from the URAAS database.
+
+Cleanup rules:
+1. Remove items from institutions no longer in the registry
+2. Remove items with no title, no DOI, no URL, and no authors
+3. Remove exact DOI duplicates (keep first by id)
+4. Remove items whose title is fewer than 10 chars
+5. Flag (do not delete) items with institution mismatch in affiliation
+6. Report before/after counts
+"""
+
+import logging
+import os
+import sys
+from datetime import datetime
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+
+
+def run_cleanup(dry_run: bool = False):
+    from sqlalchemy import func
+
+    from uraas.config.institutions import get_registry
+    from uraas.database import Author, Collection, Community, Item, SessionLocal
+
+    registry = get_registry()
+    valid_inst_names = {c.name for c in registry.list_all()}
+
+    session = SessionLocal()
+    try:
+        total_before = session.query(Item).count()
+        log.info(f"Starting cleanup | Items before: {total_before}")
+
+        removed = 0
+
+        # ── Rule 1: Remove items from removed institutions ────────────────────
+        all_institutions_in_db = session.query(Item.institution).distinct().all()
+        stale_insts = [
+            r[0]
+            for r in all_institutions_in_db
+            if r[0] and r[0] not in valid_inst_names
+        ]
+
+        if stale_insts:
+            log.info(f"Found stale institutions: {stale_insts}")
+            for stale in stale_insts:
+                stale_items = (
+                    session.query(Item).filter(Item.institution == stale).all()
+                )
+                log.info(f"  [{stale}] {len(stale_items)} items to remove")
+                if not dry_run:
+                    for item in stale_items:
+                        session.delete(item)
+                    session.commit()
+                removed += len(stale_items)
+
+        # ── Rule 2: Remove items with no title ────────────────────────────────
+        no_title = (
+            session.query(Item)
+            .filter(
+                (Item.title == None) | (Item.title == "") | (Item.title == "Untitled")
+            )
+            .all()
+        )
+        log.info(f"Items with no/empty title: {len(no_title)}")
+        if not dry_run:
+            for item in no_title:
+                session.delete(item)
+            session.commit()
+        removed += len(no_title)
+
+        # ── Rule 3: Remove items with title < 10 chars and no DOI ────────────
+        all_short = session.query(Item).filter(Item.doi == None).all()
+        short_items = [i for i in all_short if i.title and len(i.title.strip()) < 10]
+        log.info(
+            f"Items with very short title (<10 chars) and no DOI: {len(short_items)}"
+        )
+        if not dry_run:
+            for item in short_items:
+                session.delete(item)
+            session.commit()
+        removed += len(short_items)
+
+        # ── Rule 4: Remove exact DOI duplicates (keep lowest id) ─────────────
+        doi_subq = (
+            session.query(Item.doi, func.min(Item.id).label("min_id"))
+            .filter(Item.doi != None)
+            .group_by(Item.doi)
+            .subquery()
+        )
+
+        dup_dois = (
+            session.query(Item)
+            .filter(Item.doi != None, Item.id.notin_(session.query(doi_subq.c.min_id)))
+            .all()
+        )
+        log.info(f"Duplicate DOI items to remove: {len(dup_dois)}")
+        if not dry_run:
+            for item in dup_dois:
+                session.delete(item)
+            session.commit()
+        removed += len(dup_dois)
+
+        # ── Rule 5: Remove items with no institution tag ──────────────────────
+        no_inst = (
+            session.query(Item)
+            .filter((Item.institution == None) | (Item.institution == ""))
+            .all()
+        )
+        # Only remove those that have no authors either
+        truly_orphan = [i for i in no_inst if not i.authors]
+        log.info(f"Items with no institution AND no authors: {len(truly_orphan)}")
+        if not dry_run:
+            for item in truly_orphan:
+                session.delete(item)
+            session.commit()
+        removed += len(truly_orphan)
+
+        total_after = session.query(Item).count()
+
+        log.info(f"\n{'='*50}")
+        log.info(f"CLEANUP COMPLETE {'(DRY RUN)' if dry_run else ''}")
+        log.info(f"  Items before: {total_before}")
+        log.info(f"  Items removed: {removed}")
+        log.info(f"  Items after:  {total_after}")
+        log.info(f"{'='*50}")
+
+        return {
+            "total_before": total_before,
+            "removed": removed,
+            "total_after": total_after,
+            "dry_run": dry_run,
+        }
+
+    except Exception as e:
+        log.error(f"Cleanup error: {e}")
+        session.rollback()
+        raise
+    finally:
+        session.close()
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="URAAS Database Cleanup")
+    parser.add_argument(
+        "--dry-run", action="store_true", help="Report without deleting"
+    )
+    args = parser.parse_args()
+    run_cleanup(dry_run=args.dry_run)

scripts/crawl_multi_institution.py

@@ -1,242 +1,242 @@
-"""
-Multi-Institution Crawler
-Crawls papers for multiple Nigerian universities simultaneously
-"""
-
-import argparse
-import os
-import subprocess
-import sys
-
-from scrapy.crawler import CrawlerProcess
-from scrapy.utils.project import get_project_settings
-
-# Force unbuffered output so terminal log is in correct order
-sys.stdout.reconfigure(line_buffering=True)
-
-# Add project root to path (parent of scripts/)
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from uraas.config.institutions import get_registry
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Multi-institution research paper crawler"
-    )
-    parser.add_argument(
-        "--institutions",
-        type=str,
-        default="all",
-        help='Comma-separated list of institution short names, or "all" (default: all)',
-    )
-    parser.add_argument(
-        "--target", type=int, default=20, help="Target number of papers per institution"
-    )
-    parser.add_argument(
-        "--spider",
-        type=str,
-        default="openalex",
-        choices=["openalex", "crossref", "arxiv", "orcid", "oai",
-                 "semantic_scholar", "europepmc", "core", "pubmed",
-                 "openaire", "doaj", "ajol", "all"],
-        help=(
-            "Spider to use for crawling. "
-            "'all' fans out across every web source (openalex + crossref + "
-            "semantic_scholar + europepmc + arxiv + orcid) for maximum coverage."
-        ),
-    )
-    parser.add_argument(
-        "--from-date",
-        dest="from_date",
-        type=str,
-        default=None,
-        help="OAI harvest lower bound YYYY-MM-DD (oai spider only; "
-        "defaults to a recent look-back window)",
-    )
-    parser.add_argument(
-        "--until-date",
-        dest="until_date",
-        type=str,
-        default=None,
-        help="OAI harvest upper bound YYYY-MM-DD (oai spider only; optional)",
-    )
-    parser.add_argument(
-        "--clean",
-        action="store_true",
-        help="Run database cleanup script before crawling",
-    )
-    parser.add_argument(
-        "--no-boost-special",
-        dest="boost_special",
-        action="store_false",
-        help="Disable Special Collections boost waves (default: boost ON)",
-    )
-    parser.add_argument(
-        "--sc-only",
-        action="store_true",
-        help="Crawl ONLY Special Collections seed waves (skip generic ROR pass)",
-    )
-    parser.set_defaults(boost_special=True)
-
-    args = parser.parse_args()
-
-    if args.clean:
-        print("\n" + "=" * 60)
-        print("RUNNING DATABASE CLEANUP")
-        print("=" * 60)
-        try:
-            subprocess.run([sys.executable, "scripts/clean_database.py"], check=True)
-            print("Cleanup completed successfully.")
-        except subprocess.CalledProcessError as e:
-            print(f"Cleanup failed: {e}")
-            return 1
-
-    registry = get_registry()
-
-    if args.institutions.lower() == "all":
-        valid_institutions = [inst.short_name.lower() for inst in registry.list_all()]
-    else:
-        # Parse institutions
-        institution_list = [inst.strip() for inst in args.institutions.split(",")]
-        valid_institutions = []
-        for inst in institution_list:
-            config = registry.get(inst)
-            if config:
-                valid_institutions.append(config.short_name.lower())
-            else:
-                print(f"  [NOT FOUND] '{inst}' not found in registry")
-
-    print("\n" + "=" * 60, flush=True)
-    print("MULTI-INSTITUTION CRAWLER", flush=True)
-    print("=" * 60, flush=True)
-    print(f"\nTarget: {args.target} papers total per institution", flush=True)
-    print(f"Spider: {args.spider}", flush=True)
-    print(f"\nValidating institutions...", flush=True)
-
-    # Map spider names to classes (defined early so we can validate)
-    spider_map = {
-        "openalex":        "uraas.spiders.sources.openalex_spider.OpenAlexSpider",
-        "crossref":        "uraas.spiders.sources.crossref_spider.CrossrefSpider",
-        "arxiv":           "uraas.spiders.sources.arxiv_spider.ArxivSpider",
-        "orcid":           "uraas.spiders.sources.orcid_spider.ORCIDSpider",
-        "oai":             "uraas.spiders.sources.oai_spider.OAISpider",
-        "semantic_scholar":"uraas.spiders.sources.semantic_scholar_spider.SemanticScholarSpider",
-        "europepmc":       "uraas.spiders.sources.europepmc_spider.EuropePMCSpider",
-        "core":            "uraas.spiders.sources.core_spider.CORESpider",
-        "pubmed":          "uraas.spiders.sources.pubmed_spider.PubMedSpider",
-        "openaire":        "uraas.spiders.sources.openaire_spider.OpenAIRESpider",
-        "doaj":            "uraas.spiders.sources.doaj_spider.DOAJSpider",
-        "ajol":            "uraas.spiders.sources.ajol_spider.AJOLSpider",
-    }
-
-    # "all" = every web-discovery spider (excludes "oai" which reads FROM the IR)
-    ALL_WEB_SPIDERS = [
-        "openalex", "crossref", "semantic_scholar", "europepmc",
-        "core", "pubmed", "openaire", "doaj", "ajol", "arxiv", "orcid",
-    ]
-
-    if args.spider == "all":
-        spider_names_to_run = ALL_WEB_SPIDERS
-        # Divide target across spiders so total ≈ requested target
-        per_spider_target = max(1, args.target // len(spider_names_to_run))
-    else:
-        spider_names_to_run = [args.spider]
-        per_spider_target = args.target
-
-    # Validate + import all spider classes up front so errors appear early
-    spider_classes = {}
-    for sname in spider_names_to_run:
-        path = spider_map.get(sname)
-        if not path:
-            print(f"\n[ERR] Spider '{sname}' not supported", flush=True)
-            return 1
-        mod_path, cls_name = path.rsplit(".", 1)
-        mod = __import__(mod_path, fromlist=[cls_name])
-        spider_classes[sname] = getattr(mod, cls_name)
-
-    # Legacy single-spider variable (used below)
-    spider_class = spider_classes.get(spider_names_to_run[0])
-
-    for inst in valid_institutions:
-        config = registry.get(inst)
-        print(f"  [VALID] {config.name} ({config.short_name})", flush=True)
-        print(f"    ROR: {config.ror}", flush=True)
-        print(f"    Staff: {len(config.staff_names)}", flush=True)
-
-    if not valid_institutions:
-        print("\n[ERR] No valid institutions found. Exiting.", flush=True)
-        return 1
-
-    print(f"\n{len(valid_institutions)} institution(s) validated", flush=True)
-    print("=" * 60, flush=True)
-
-    # Schedule crawls — ONE CrawlerProcess for ALL institutions
-    print(f"\nScheduling crawls...", flush=True)
-    settings = get_project_settings()
-    settings.set(
-        "ITEM_PIPELINES",
-        {
-            "uraas.pipelines.database.DatabaseStoragePipeline": 300,
-        },
-    )
-    settings.set("LOG_LEVEL", "INFO")
-    settings.set("LOG_SCRAPED_ITEMS", False)
-    settings.set("TELNETCONSOLE_ENABLED", False)
-
-    process = CrawlerProcess(settings)
-
-    print(f"  Boost special collections: {args.boost_special}", flush=True)
-    print(f"  SC-only mode: {args.sc_only}", flush=True)
-    print(f"  Spiders: {', '.join(spider_names_to_run)}", flush=True)
-
-    for inst in valid_institutions:
-        cfg = registry.get(inst)
-        print(f"  -> {cfg.name}", flush=True)
-        for sname in spider_names_to_run:
-            scls = spider_classes[sname]
-            if sname == "oai":
-                process.crawl(
-                    scls,
-                    institution=inst,
-                    target=per_spider_target,
-                    from_date=args.from_date,
-                    until_date=args.until_date,
-                )
-            else:
-                process.crawl(
-                    scls,
-                    institution=inst,
-                    target=per_spider_target,
-                    boost_special=args.boost_special,
-                    sc_only=args.sc_only,
-                )
-
-    print(
-        f"\nStarting crawl for {len(valid_institutions)} institution(s)...", flush=True
-    )
-    print("=" * 60, flush=True)
-    sys.stdout.flush()
-
-    # Start crawling
-    try:
-        process.start()
-        print("\n" + "=" * 60)
-        print("CRAWL COMPLETED")
-        print("=" * 60)
-        return 0
-
-    except KeyboardInterrupt:
-        print("\n\n[ERR] Crawl interrupted by user")
-        return 1
-
-    except Exception as e:
-        print(f"\n\n[ERR] Crawl failed: {e}")
-        import traceback
-
-        traceback.print_exc()
-        return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+"""
+Multi-Institution Crawler
+Crawls papers for multiple Nigerian universities simultaneously
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+
+from scrapy.crawler import CrawlerProcess
+from scrapy.utils.project import get_project_settings
+
+# Force unbuffered output so terminal log is in correct order
+sys.stdout.reconfigure(line_buffering=True)
+
+# Add project root to path (parent of scripts/)
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from uraas.config.institutions import get_registry
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Multi-institution research paper crawler"
+    )
+    parser.add_argument(
+        "--institutions",
+        type=str,
+        default="all",
+        help='Comma-separated list of institution short names, or "all" (default: all)',
+    )
+    parser.add_argument(
+        "--target", type=int, default=20, help="Target number of papers per institution"
+    )
+    parser.add_argument(
+        "--spider",
+        type=str,
+        default="openalex",
+        choices=["openalex", "crossref", "arxiv", "orcid", "oai",
+                 "semantic_scholar", "europepmc", "core", "pubmed",
+                 "openaire", "doaj", "ajol", "all"],
+        help=(
+            "Spider to use for crawling. "
+            "'all' fans out across every web source (openalex + crossref + "
+            "semantic_scholar + europepmc + arxiv + orcid) for maximum coverage."
+        ),
+    )
+    parser.add_argument(
+        "--from-date",
+        dest="from_date",
+        type=str,
+        default=None,
+        help="OAI harvest lower bound YYYY-MM-DD (oai spider only; "
+        "defaults to a recent look-back window)",
+    )
+    parser.add_argument(
+        "--until-date",
+        dest="until_date",
+        type=str,
+        default=None,
+        help="OAI harvest upper bound YYYY-MM-DD (oai spider only; optional)",
+    )
+    parser.add_argument(
+        "--clean",
+        action="store_true",
+        help="Run database cleanup script before crawling",
+    )
+    parser.add_argument(
+        "--no-boost-special",
+        dest="boost_special",
+        action="store_false",
+        help="Disable Special Collections boost waves (default: boost ON)",
+    )
+    parser.add_argument(
+        "--sc-only",
+        action="store_true",
+        help="Crawl ONLY Special Collections seed waves (skip generic ROR pass)",
+    )
+    parser.set_defaults(boost_special=True)
+
+    args = parser.parse_args()
+
+    if args.clean:
+        print("\n" + "=" * 60)
+        print("RUNNING DATABASE CLEANUP")
+        print("=" * 60)
+        try:
+            subprocess.run([sys.executable, "scripts/clean_database.py"], check=True)
+            print("Cleanup completed successfully.")
+        except subprocess.CalledProcessError as e:
+            print(f"Cleanup failed: {e}")
+            return 1
+
+    registry = get_registry()
+
+    if args.institutions.lower() == "all":
+        valid_institutions = [inst.short_name.lower() for inst in registry.list_all()]
+    else:
+        # Parse institutions
+        institution_list = [inst.strip() for inst in args.institutions.split(",")]
+        valid_institutions = []
+        for inst in institution_list:
+            config = registry.get(inst)
+            if config:
+                valid_institutions.append(config.short_name.lower())
+            else:
+                print(f"  [NOT FOUND] '{inst}' not found in registry")
+
+    print("\n" + "=" * 60, flush=True)
+    print("MULTI-INSTITUTION CRAWLER", flush=True)
+    print("=" * 60, flush=True)
+    print(f"\nTarget: {args.target} papers total per institution", flush=True)
+    print(f"Spider: {args.spider}", flush=True)
+    print(f"\nValidating institutions...", flush=True)
+
+    # Map spider names to classes (defined early so we can validate)
+    spider_map = {
+        "openalex":        "uraas.spiders.sources.openalex_spider.OpenAlexSpider",
+        "crossref":        "uraas.spiders.sources.crossref_spider.CrossrefSpider",
+        "arxiv":           "uraas.spiders.sources.arxiv_spider.ArxivSpider",
+        "orcid":           "uraas.spiders.sources.orcid_spider.ORCIDSpider",
+        "oai":             "uraas.spiders.sources.oai_spider.OAISpider",
+        "semantic_scholar":"uraas.spiders.sources.semantic_scholar_spider.SemanticScholarSpider",
+        "europepmc":       "uraas.spiders.sources.europepmc_spider.EuropePMCSpider",
+        "core":            "uraas.spiders.sources.core_spider.CORESpider",
+        "pubmed":          "uraas.spiders.sources.pubmed_spider.PubMedSpider",
+        "openaire":        "uraas.spiders.sources.openaire_spider.OpenAIRESpider",
+        "doaj":            "uraas.spiders.sources.doaj_spider.DOAJSpider",
+        "ajol":            "uraas.spiders.sources.ajol_spider.AJOLSpider",
+    }
+
+    # "all" = every web-discovery spider (excludes "oai" which reads FROM the IR)
+    ALL_WEB_SPIDERS = [
+        "openalex", "crossref", "semantic_scholar", "europepmc",
+        "core", "pubmed", "openaire", "doaj", "ajol", "arxiv", "orcid",
+    ]
+
+    if args.spider == "all":
+        spider_names_to_run = ALL_WEB_SPIDERS
+        # Divide target across spiders so total ≈ requested target
+        per_spider_target = max(1, args.target // len(spider_names_to_run))
+    else:
+        spider_names_to_run = [args.spider]
+        per_spider_target = args.target
+
+    # Validate + import all spider classes up front so errors appear early
+    spider_classes = {}
+    for sname in spider_names_to_run:
+        path = spider_map.get(sname)
+        if not path:
+            print(f"\n[ERR] Spider '{sname}' not supported", flush=True)
+            return 1
+        mod_path, cls_name = path.rsplit(".", 1)
+        mod = __import__(mod_path, fromlist=[cls_name])
+        spider_classes[sname] = getattr(mod, cls_name)
+
+    # Legacy single-spider variable (used below)
+    spider_class = spider_classes.get(spider_names_to_run[0])
+
+    for inst in valid_institutions:
+        config = registry.get(inst)
+        print(f"  [VALID] {config.name} ({config.short_name})", flush=True)
+        print(f"    ROR: {config.ror}", flush=True)
+        print(f"    Staff: {len(config.staff_names)}", flush=True)
+
+    if not valid_institutions:
+        print("\n[ERR] No valid institutions found. Exiting.", flush=True)
+        return 1
+
+    print(f"\n{len(valid_institutions)} institution(s) validated", flush=True)
+    print("=" * 60, flush=True)
+
+    # Schedule crawls — ONE CrawlerProcess for ALL institutions
+    print(f"\nScheduling crawls...", flush=True)
+    settings = get_project_settings()
+    settings.set(
+        "ITEM_PIPELINES",
+        {
+            "uraas.pipelines.database.DatabaseStoragePipeline": 300,
+        },
+    )
+    settings.set("LOG_LEVEL", "INFO")
+    settings.set("LOG_SCRAPED_ITEMS", False)
+    settings.set("TELNETCONSOLE_ENABLED", False)
+
+    process = CrawlerProcess(settings)
+
+    print(f"  Boost special collections: {args.boost_special}", flush=True)
+    print(f"  SC-only mode: {args.sc_only}", flush=True)
+    print(f"  Spiders: {', '.join(spider_names_to_run)}", flush=True)
+
+    for inst in valid_institutions:
+        cfg = registry.get(inst)
+        print(f"  -> {cfg.name}", flush=True)
+        for sname in spider_names_to_run:
+            scls = spider_classes[sname]
+            if sname == "oai":
+                process.crawl(
+                    scls,
+                    institution=inst,
+                    target=per_spider_target,
+                    from_date=args.from_date,
+                    until_date=args.until_date,
+                )
+            else:
+                process.crawl(
+                    scls,
+                    institution=inst,
+                    target=per_spider_target,
+                    boost_special=args.boost_special,
+                    sc_only=args.sc_only,
+                )
+
+    print(
+        f"\nStarting crawl for {len(valid_institutions)} institution(s)...", flush=True
+    )
+    print("=" * 60, flush=True)
+    sys.stdout.flush()
+
+    # Start crawling
+    try:
+        process.start()
+        print("\n" + "=" * 60)
+        print("CRAWL COMPLETED")
+        print("=" * 60)
+        return 0
+
+    except KeyboardInterrupt:
+        print("\n\n[ERR] Crawl interrupted by user")
+        return 1
+
+    except Exception as e:
+        print(f"\n\n[ERR] Crawl failed: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/deploy.sh

@@ -1,110 +1,110 @@
-#!/usr/bin/env bash
-# ──────────────────────────────────────────────────────────────────────────────
-#  URAAS — One-shot deployment script for Ubuntu 22.04 / Debian 12
-#
-#  Run on a fresh VPS as root or a sudo user:
-#    curl -sSL https://raw.githubusercontent.com/YOUR/repo/main/scripts/deploy.sh | bash
-#  OR after cloning:
-#    bash scripts/deploy.sh
-#
-#  What it does:
-#    1. Install Docker + Docker Compose plugin
-#    2. Generate password hashes interactively
-#    3. Build and start all containers (postgres, redis, app, nginx)
-#    4. Print the URL to reach the dashboard
-# ──────────────────────────────────────────────────────────────────────────────
-set -euo pipefail
-
-REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-cd "$REPO_DIR"
-
-echo ""
-echo "═══════════════════════════════════════════════════"
-echo "  URAAS Deployment — $(date +%Y-%m-%d)"
-echo "═══════════════════════════════════════════════════"
-
-# ── 1. Docker ─────────────────────────────────────────────────────────────────
-if ! command -v docker &>/dev/null; then
-  echo ""
-  echo "▶  Installing Docker..."
-  curl -fsSL https://get.docker.com | sh
-  usermod -aG docker "$USER" || true
-  echo "   Docker installed. You may need to log out and back in."
-fi
-
-if ! docker compose version &>/dev/null 2>&1; then
-  echo ""
-  echo "▶  Installing Docker Compose plugin..."
-  apt-get install -y docker-compose-plugin 2>/dev/null || \
-    pip install docker-compose 2>/dev/null || \
-    echo "   Install docker-compose manually from docs.docker.com/compose/install/"
-fi
-
-# ── 2. .env.prod ──────────────────────────────────────────────────────────────
-if [ ! -f .env.prod ]; then
-  echo ""
-  echo "▶  Creating .env.prod from example..."
-  cp .env.prod.example .env.prod
-
-  # Get server IP
-  SERVER_IP=$(curl -s https://ifconfig.me || curl -s https://api.ipify.org || echo "YOUR_SERVER_IP")
-  sed -i "s/YOUR_SERVER_IP/$SERVER_IP/g" .env.prod
-
-  # Generate secret key
-  SECRET_KEY=$(python3 -c "import secrets; print(secrets.token_hex(32))" 2>/dev/null || \
-               openssl rand -hex 32)
-  sed -i "s/REPLACE_WITH_STRONG_RANDOM_KEY/$SECRET_KEY/" .env.prod
-
-  # Generate password hashes
-  echo ""
-  echo "Enter the ADMIN password (for dashboard login):"
-  read -rs ADMIN_PASS
-  ADMIN_HASH=$(python3 -c "from werkzeug.security import generate_password_hash as g; print(g('$ADMIN_PASS'))" 2>/dev/null || \
-               python3 -c "import hashlib, os; print('pbkdf2:sha256:' + hashlib.pbkdf2_hmac('sha256', b'$ADMIN_PASS', os.urandom(16), 150000).hex())")
-  sed -i "s|REPLACE_WITH_WERKZEUG_HASH|$ADMIN_HASH|g" .env.prod
-
-  echo ""
-  echo "   .env.prod created. Edit it to add SMTP_PASSWORD and API keys before running."
-  echo ""
-  echo "   IMPORTANT: Set these in .env.prod before the demo:"
-  echo "     SMTP_PASSWORD=<your-gmail-app-password>"
-  echo "     S2_API_KEY=<from semanticscholar.org>"
-  echo "     CORE_API_KEY=<from core.ac.uk/api-keys>"
-fi
-
-# ── 3. Required directories ───────────────────────────────────────────────────
-mkdir -p storage/pdfs data logs backups nginx/ssl
-
-# ── 4. Build + Start ──────────────────────────────────────────────────────────
-echo ""
-echo "▶  Building and starting containers (this takes ~3 min first time)..."
-echo ""
-# docker-compose.demo.yml = HTTP-only, works on bare IP (no SSL cert needed).
-# Switch to docker-compose.prod.yml once you have a domain + SSL certificate.
-docker compose --env-file .env.prod -f docker-compose.demo.yml up --build -d
-
-# ── 5. Wait for health ────────────────────────────────────────────────────────
-echo ""
-echo "▶  Waiting for app to become healthy..."
-for i in $(seq 1 20); do
-  STATUS=$(docker inspect --format='{{.State.Health.Status}}' uraas-app 2>/dev/null || echo "starting")
-  if [ "$STATUS" = "healthy" ]; then
-    echo "   App is healthy!"
-    break
-  fi
-  echo "   [$i/20] Status: $STATUS — waiting 5s..."
-  sleep 5
-done
-
-# ── 6. Done ───────────────────────────────────────────────────────────────────
-SERVER_IP=$(curl -s https://ifconfig.me 2>/dev/null || echo "YOUR_SERVER_IP")
-echo ""
-echo "═══════════════════════════════════════════════════"
-echo "  URAAS is running!"
-echo ""
-echo "  Dashboard (direct):  http://$SERVER_IP:8080"
-echo "  Dashboard (nginx):   http://$SERVER_IP"
-echo ""
-echo "  Logs:  docker compose -f docker-compose.prod.yml logs -f app"
-echo "  Stop:  docker compose -f docker-compose.prod.yml down"
-echo "═══════════════════════════════════════════════════"
+#!/usr/bin/env bash
+# ──────────────────────────────────────────────────────────────────────────────
+#  URAAS — One-shot deployment script for Ubuntu 22.04 / Debian 12
+#
+#  Run on a fresh VPS as root or a sudo user:
+#    curl -sSL https://raw.githubusercontent.com/YOUR/repo/main/scripts/deploy.sh | bash
+#  OR after cloning:
+#    bash scripts/deploy.sh
+#
+#  What it does:
+#    1. Install Docker + Docker Compose plugin
+#    2. Generate password hashes interactively
+#    3. Build and start all containers (postgres, redis, app, nginx)
+#    4. Print the URL to reach the dashboard
+# ──────────────────────────────────────────────────────────────────────────────
+set -euo pipefail
+
+REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$REPO_DIR"
+
+echo ""
+echo "═══════════════════════════════════════════════════"
+echo "  URAAS Deployment — $(date +%Y-%m-%d)"
+echo "═══════════════════════════════════════════════════"
+
+# ── 1. Docker ─────────────────────────────────────────────────────────────────
+if ! command -v docker &>/dev/null; then
+  echo ""
+  echo "▶  Installing Docker..."
+  curl -fsSL https://get.docker.com | sh
+  usermod -aG docker "$USER" || true
+  echo "   Docker installed. You may need to log out and back in."
+fi
+
+if ! docker compose version &>/dev/null 2>&1; then
+  echo ""
+  echo "▶  Installing Docker Compose plugin..."
+  apt-get install -y docker-compose-plugin 2>/dev/null || \
+    pip install docker-compose 2>/dev/null || \
+    echo "   Install docker-compose manually from docs.docker.com/compose/install/"
+fi
+
+# ── 2. .env.prod ──────────────────────────────────────────────────────────────
+if [ ! -f .env.prod ]; then
+  echo ""
+  echo "▶  Creating .env.prod from example..."
+  cp .env.prod.example .env.prod
+
+  # Get server IP
+  SERVER_IP=$(curl -s https://ifconfig.me || curl -s https://api.ipify.org || echo "YOUR_SERVER_IP")
+  sed -i "s/YOUR_SERVER_IP/$SERVER_IP/g" .env.prod
+
+  # Generate secret key
+  SECRET_KEY=$(python3 -c "import secrets; print(secrets.token_hex(32))" 2>/dev/null || \
+               openssl rand -hex 32)
+  sed -i "s/REPLACE_WITH_STRONG_RANDOM_KEY/$SECRET_KEY/" .env.prod
+
+  # Generate password hashes
+  echo ""
+  echo "Enter the ADMIN password (for dashboard login):"
+  read -rs ADMIN_PASS
+  ADMIN_HASH=$(python3 -c "from werkzeug.security import generate_password_hash as g; print(g('$ADMIN_PASS'))" 2>/dev/null || \
+               python3 -c "import hashlib, os; print('pbkdf2:sha256:' + hashlib.pbkdf2_hmac('sha256', b'$ADMIN_PASS', os.urandom(16), 150000).hex())")
+  sed -i "s|REPLACE_WITH_WERKZEUG_HASH|$ADMIN_HASH|g" .env.prod
+
+  echo ""
+  echo "   .env.prod created. Edit it to add SMTP_PASSWORD and API keys before running."
+  echo ""
+  echo "   IMPORTANT: Set these in .env.prod before the demo:"
+  echo "     SMTP_PASSWORD=<your-gmail-app-password>"
+  echo "     S2_API_KEY=<from semanticscholar.org>"
+  echo "     CORE_API_KEY=<from core.ac.uk/api-keys>"
+fi
+
+# ── 3. Required directories ───────────────────────────────────────────────────
+mkdir -p storage/pdfs data logs backups nginx/ssl
+
+# ── 4. Build + Start ���─────────────────────────────────────────────────────────
+echo ""
+echo "▶  Building and starting containers (this takes ~3 min first time)..."
+echo ""
+# docker-compose.demo.yml = HTTP-only, works on bare IP (no SSL cert needed).
+# Switch to docker-compose.prod.yml once you have a domain + SSL certificate.
+docker compose --env-file .env.prod -f docker-compose.demo.yml up --build -d
+
+# ── 5. Wait for health ────────────────────────────────────────────────────────
+echo ""
+echo "▶  Waiting for app to become healthy..."
+for i in $(seq 1 20); do
+  STATUS=$(docker inspect --format='{{.State.Health.Status}}' uraas-app 2>/dev/null || echo "starting")
+  if [ "$STATUS" = "healthy" ]; then
+    echo "   App is healthy!"
+    break
+  fi
+  echo "   [$i/20] Status: $STATUS — waiting 5s..."
+  sleep 5
+done
+
+# ── 6. Done ───────────────────────────────────────────────────────────────────
+SERVER_IP=$(curl -s https://ifconfig.me 2>/dev/null || echo "YOUR_SERVER_IP")
+echo ""
+echo "═══════════════════════════════════════════════════"
+echo "  URAAS is running!"
+echo ""
+echo "  Dashboard (direct):  http://$SERVER_IP:8080"
+echo "  Dashboard (nginx):   http://$SERVER_IP"
+echo ""
+echo "  Logs:  docker compose -f docker-compose.prod.yml logs -f app"
+echo "  Stop:  docker compose -f docker-compose.prod.yml down"
+echo "═══════════════════════════════════════════════════"

scripts/fix_rors.py

@@ -1,26 +1,26 @@
-import glob
-import json
-import os
-import urllib.parse
-import urllib.request
-
-files = glob.glob("config/institutions/*.json")
-
-for fpath in files:
-    with open(fpath, "r", encoding="utf-8") as f:
-        data = json.load(f)
-    name = data.get("name")
-    if not name:
-        continue
-
-    url = f"https://api.openalex.org/institutions?search={urllib.parse.quote(name)}&per-page=1"
-    try:
-        res = json.loads(urllib.request.urlopen(url).read().decode())["results"][0]
-        correct_ror = res.get("ror")
-        if correct_ror and correct_ror != data.get("ror"):
-            print(f"Updating {name}: {data.get('ror')} -> {correct_ror}")
-            data["ror"] = correct_ror
-            with open(fpath, "w", encoding="utf-8") as f:
-                json.dump(data, f, indent=2)
-    except Exception as e:
-        print(f"Error for {name}: {e}")
+import glob
+import json
+import os
+import urllib.parse
+import urllib.request
+
+files = glob.glob("config/institutions/*.json")
+
+for fpath in files:
+    with open(fpath, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    name = data.get("name")
+    if not name:
+        continue
+
+    url = f"https://api.openalex.org/institutions?search={urllib.parse.quote(name)}&per-page=1"
+    try:
+        res = json.loads(urllib.request.urlopen(url).read().decode())["results"][0]
+        correct_ror = res.get("ror")
+        if correct_ror and correct_ror != data.get("ror"):
+            print(f"Updating {name}: {data.get('ror')} -> {correct_ror}")
+            data["ror"] = correct_ror
+            with open(fpath, "w", encoding="utf-8") as f:
+                json.dump(data, f, indent=2)
+    except Exception as e:
+        print(f"Error for {name}: {e}")

scripts/generate_registry.py

@@ -1,444 +1,444 @@
-import json
-import os
-
-# Sub-regions and countries
-subregions = {
-    "North Africa": ["Egypt", "Morocco", "Algeria", "Tunisia", "Libya", "Sudan"],
-    "West Africa": [
-        "Nigeria",
-        "Ghana",
-        "Senegal",
-        "Cote d'Ivoire",
-        "Benin",
-        "Burkina Faso",
-        "Cape Verde",
-        "Gambia",
-        "Guinea",
-        "Guinea-Bissau",
-        "Liberia",
-        "Mali",
-        "Mauritania",
-        "Niger",
-        "Sierra Leone",
-        "Togo",
-    ],
-    "East Africa": [
-        "Kenya",
-        "Uganda",
-        "Tanzania",
-        "Ethiopia",
-        "Rwanda",
-        "Burundi",
-        "Djibouti",
-        "Eritrea",
-        "Somalia",
-        "South Sudan",
-        "Madagascar",
-        "Mauritius",
-        "Seychelles",
-        "Comoros",
-    ],
-    "Southern Africa": [
-        "South Africa",
-        "Zimbabwe",
-        "Zambia",
-        "Namibia",
-        "Botswana",
-        "Lesotho",
-        "Eswatini",
-        "Malawi",
-        "Mozambique",
-    ],
-    "Central Africa": [
-        "Cameroon",
-        "DR Congo",
-        "Angola",
-        "Gabon",
-        "Republic of the Congo",
-        "Central African Republic",
-        "Chad",
-        "Equatorial Guinea",
-        "Sao Tome and Principe",
-    ],
-}
-
-# Major cities for generation if needed
-capitals = {
-    "Egypt": "Cairo",
-    "Morocco": "Rabat",
-    "Algeria": "Algiers",
-    "Tunisia": "Tunis",
-    "Libya": "Tripoli",
-    "Sudan": "Khartoum",
-    "Nigeria": "Abuja",
-    "Ghana": "Accra",
-    "Senegal": "Dakar",
-    "Cote d'Ivoire": "Yamoussoukro",
-    "Benin": "Porto-Novo",
-    "Burkina Faso": "Ouagadougou",
-    "Cape Verde": "Praia",
-    "Gambia": "Banjul",
-    "Guinea": "Conakry",
-    "Guinea-Bissau": "Bissau",
-    "Liberia": "Monrovia",
-    "Mali": "Bamako",
-    "Mauritania": "Nouakchott",
-    "Niger": "Niamey",
-    "Sierra Leone": "Freetown",
-    "Togo": "Lome",
-    "Kenya": "Nairobi",
-    "Uganda": "Kampala",
-    "Tanzania": "Dodoma",
-    "Ethiopia": "Addis Ababa",
-    "Rwanda": "Kigali",
-    "Burundi": "Gitega",
-    "Djibouti": "Djibouti",
-    "Eritrea": "Asmara",
-    "Somalia": "Mogadishu",
-    "South Sudan": "Juba",
-    "Madagascar": "Antananarivo",
-    "Mauritius": "Port Louis",
-    "Seychelles": "Victoria",
-    "Comoros": "Moroni",
-    "South Africa": "Pretoria",
-    "Zimbabwe": "Harare",
-    "Zambia": "Lusaka",
-    "Namibia": "Windhoek",
-    "Botswana": "Gaborone",
-    "Lesotho": "Maseru",
-    "Eswatini": "Mbabane",
-    "Malawi": "Lilongwe",
-    "Mozambique": "Maputo",
-    "Cameroon": "Yaounde",
-    "DR Congo": "Kinshasa",
-    "Angola": "Luanda",
-    "Gabon": "Libreville",
-    "Republic of the Congo": "Brazzaville",
-    "Central African Republic": "Bangui",
-    "Chad": "N'Djamena",
-    "Equatorial Guinea": "Malabo",
-    "Sao Tome and Principe": "Sao Tome",
-}
-
-# Hand-curated top universities to include (demo universities)
-curated_universities = {
-    # North Africa
-    "Egypt": [
-        {"name": "Cairo University", "ror": "https://ror.org/03c4mpy73"},
-        {"name": "Ain Shams University", "ror": "https://ror.org/034x7p097"},
-        {"name": "Alexandria University", "ror": "https://ror.org/02078r490"},
-        {"name": "Mansoura University", "ror": "https://ror.org/032p18087"},
-        {"name": "Assiut University", "ror": "https://ror.org/047fpp722"},
-    ],
-    "Morocco": [
-        {"name": "Université Mohammed V de Rabat", "ror": "https://ror.org/03vpy3v17"},
-        {"name": "Université Cadi Ayyad", "ror": "https://ror.org/0154pcf71"},
-        {
-            "name": "Université Hassan II de Casablanca",
-            "ror": "https://ror.org/013y27r38",
-        },
-    ],
-    "Tunisia": [
-        {"name": "Université de Tunis El Manar", "ror": "https://ror.org/050j3a172"},
-        {"name": "Université de Sfax", "ror": "https://ror.org/02157p641"},
-        {"name": "Université de Carthage", "ror": "https://ror.org/011yvpy71"},
-    ],
-    # Central Africa
-    "Cameroon": [
-        {"name": "Université de Yaoundé I", "ror": "https://ror.org/04h7g6177"},
-        {"name": "Université de Dschang", "ror": "https://ror.org/012y7p041"},
-        {"name": "Université de Douala", "ror": "https://ror.org/041y27r28"},
-    ],
-    "DR Congo": [
-        {"name": "Université de Kinshasa", "ror": "https://ror.org/05vzwad88"},
-        {"name": "Université de Lubumbashi", "ror": "https://ror.org/02rry3m21"},
-    ],
-    "Angola": [
-        {"name": "Université Agostinho Neto", "ror": "https://ror.org/00z2bpt98"}
-    ],
-    "Gabon": [
-        {
-            "name": "Université des Sciences et Techniques de Masuku",
-            "ror": "https://ror.org/059gqse72",
-        },
-        {"name": "Université Omar Bongo", "ror": "https://ror.org/041yp1812"},
-    ],
-    "Republic of the Congo": [
-        {"name": "Université Marien Ngouabi", "ror": "https://ror.org/02y1sra05"}
-    ],
-    # West Africa
-    "Nigeria": [
-        {"name": "University of Lagos", "ror": "https://ror.org/05rk03822"},
-        {"name": "University of Ibadan", "ror": "https://ror.org/01es5me90"},
-        {"name": "Covenant University", "ror": "https://ror.org/02n05rk12"},
-        {"name": "Obafemi Awolowo University", "ror": "https://ror.org/013pcr241"},
-        {"name": "University of Nigeria Nsukka", "ror": "https://ror.org/02kpy5732"},
-    ],
-    "Ghana": [
-        {"name": "University of Ghana", "ror": "https://ror.org/00zpy3v12"},
-        {
-            "name": "Kwame Nkrumah University of Science and Technology",
-            "ror": "https://ror.org/00x4mpy73",
-        },
-        {"name": "University of Cape Coast", "ror": "https://ror.org/01es3v123"},
-    ],
-    # Southern Africa
-    "South Africa": [
-        {"name": "University of Cape Town", "ror": "https://ror.org/017620319"},
-        {"name": "Stellenbosch University", "ror": "https://ror.org/05777p686"},
-        {"name": "University of the Witwatersrand", "ror": "https://ror.org/039482g93"},
-        {"name": "University of Pretoria", "ror": "https://ror.org/047fpp722"},
-        {"name": "University of KwaZulu-Natal", "ror": "https://ror.org/01267r312"},
-    ],
-    "Zimbabwe": [
-        {"name": "University of Zimbabwe", "ror": "https://ror.org/03w489125"},
-        {
-            "name": "National University of Science and Technology",
-            "ror": "https://ror.org/01y6mpy73",
-        },
-    ],
-    # East Africa
-    "Kenya": [
-        {"name": "University of Nairobi", "ror": "https://ror.org/01078r490"},
-        {"name": "Kenyatta University", "ror": "https://ror.org/01py3v171"},
-        {
-            "name": "Jomo Kenyatta University of Agriculture and Technology",
-            "ror": "https://ror.org/03pyvpy71",
-        },
-    ],
-    "Uganda": [
-        {"name": "Makerere University", "ror": "https://ror.org/05vzwad88"},
-        {
-            "name": "Mbarara University of Science and Technology",
-            "ror": "https://ror.org/0155pcf71",
-        },
-    ],
-    "Tanzania": [
-        {"name": "University of Dar es Salaam", "ror": "https://ror.org/0199e1957"},
-        {
-            "name": "Sokoine University of Agriculture",
-            "ror": "https://ror.org/011y27r38",
-        },
-    ],
-    "Ethiopia": [
-        {"name": "Addis Ababa University", "ror": "https://ror.org/01py3v171"}
-    ],
-    "Rwanda": [{"name": "University of Rwanda", "ror": "https://ror.org/02yr01r27"}],
-}
-
-# Generate 15-20 universities for each country
-registry_data = {}
-for subregion, countries in subregions.items():
-    registry_data[subregion] = {}
-    for country in countries:
-        cap = capitals.get(country, "City")
-        # Start with curated list or empty
-        unis = curated_universities.get(country, []).copy()
-
-        # Add generated universities to reach 15
-        existing_names = {u["name"] for u in unis}
-        templates = [
-            f"University of {cap}",
-            f"National University of {country}",
-            f"{country} University of Science and Technology",
-            f"{cap} Institute of Technology",
-            f"State University of {cap}",
-            f"{country} International University",
-            f"Pan-African University, {cap} Campus",
-            f"Catholic University of {country}",
-            f"Technical University of {cap}",
-            (
-                f"Ahmadu Bello University of {cap}"
-                if country == "Nigeria"
-                else f"Federal University of {cap}"
-            ),
-            f"Metropolitan University of {cap}",
-            f"Central University of {country}",
-            f"Presbyterian University of {country}",
-            f"Adventist University of {country}",
-            f"Islamic University of {country}",
-            f"Methodist University of {country}",
-            f"Covenant University of {cap}",
-            f"{cap} College of Medicine and Health Sciences",
-            f"Regional Institute of Information Technology, {cap}",
-            f"Greenfield University, {cap}",
-        ]
-
-        idx = 0
-        while len(unis) < 18:
-            name = templates[idx % len(templates)]
-            # ensure uniqueness
-            if name not in existing_names:
-                unis.append({"name": name, "ror": ""})
-                existing_names.add(name)
-            idx += 1
-
-        registry_data[subregion][country] = unis
-
-# Write university_registry.json
-os.makedirs("data", exist_ok=True)
-with open("data/university_registry.json", "w", encoding="utf-8") as f:
-    json.dump(registry_data, f, indent=2, ensure_ascii=False)
-print(
-    "Generated data/university_registry.json with 52 countries and 18 universities each."
-)
-
-# Configurations for the 15 new universities to make a total of 25 demo universities
-new_universities = [
-    # North (5)
-    {
-        "file": "cairo.json",
-        "ror": "https://ror.org/03c4mpy73",
-        "name": "Cairo University",
-        "short_name": "Cairo Univ",
-        "country": "Egypt",
-        "sub_region": "North Africa",
-    },
-    {
-        "file": "ainshams.json",
-        "ror": "https://ror.org/034x7p097",
-        "name": "Ain Shams University",
-        "short_name": "Ain Shams",
-        "country": "Egypt",
-        "sub_region": "North Africa",
-    },
-    {
-        "file": "alexandria.json",
-        "ror": "https://ror.org/02078r490",
-        "name": "Alexandria University",
-        "short_name": "Alexandria",
-        "country": "Egypt",
-        "sub_region": "North Africa",
-    },
-    {
-        "file": "tunis.json",
-        "ror": "https://ror.org/050j3a172",
-        "name": "Université de Tunis El Manar",
-        "short_name": "Tunis El Manar",
-        "country": "Tunisia",
-        "sub_region": "North Africa",
-    },
-    {
-        "file": "mohammedv.json",
-        "ror": "https://ror.org/03vpy3v17",
-        "name": "Université Mohammed V de Rabat",
-        "short_name": "Mohammed V",
-        "country": "Morocco",
-        "sub_region": "North Africa",
-    },
-    # Central (5)
-    {
-        "file": "yaoundei.json",
-        "ror": "https://ror.org/04h7g6177",
-        "name": "Université de Yaoundé I",
-        "short_name": "Yaoundé I",
-        "country": "Cameroon",
-        "sub_region": "Central Africa",
-    },
-    {
-        "file": "kinshasa.json",
-        "ror": "https://ror.org/05vzwad88",
-        "name": "Université de Kinshasa",
-        "short_name": "UNIKIN",
-        "country": "DR Congo",
-        "sub_region": "Central Africa",
-    },
-    {
-        "file": "agostinhoneto.json",
-        "ror": "https://ror.org/00z2bpt98",
-        "name": "Université Agostinho Neto",
-        "short_name": "Agostinho Neto",
-        "country": "Angola",
-        "sub_region": "Central Africa",
-    },
-    {
-        "file": "marienngouabi.json",
-        "ror": "https://ror.org/02y1sra05",
-        "name": "Université Marien Ngouabi",
-        "short_name": "Marien Ngouabi",
-        "country": "Republic of the Congo",
-        "sub_region": "Central Africa",
-    },
-    {
-        "file": "masuku.json",
-        "ror": "https://ror.org/059gqse72",
-        "name": "Université des Sciences et Techniques de Masuku",
-        "short_name": "USTM Masuku",
-        "country": "Gabon",
-        "sub_region": "Central Africa",
-    },
-    # Southern (+3 new ones)
-    {
-        "file": "wits.json",
-        "ror": "https://ror.org/039482g93",
-        "name": "University of the Witwatersrand",
-        "short_name": "Wits",
-        "country": "South Africa",
-        "sub_region": "Southern Africa",
-    },
-    {
-        "file": "pretoria.json",
-        "ror": "https://ror.org/047fpp722",
-        "name": "University of Pretoria",
-        "short_name": "UP",
-        "country": "South Africa",
-        "sub_region": "Southern Africa",
-    },
-    {
-        "file": "zimbabwe.json",
-        "ror": "https://ror.org/03w489125",
-        "name": "University of Zimbabwe",
-        "short_name": "UZ",
-        "country": "Zimbabwe",
-        "sub_region": "Southern Africa",
-    },
-    # East (+2 new ones)
-    {
-        "file": "daressalaam.json",
-        "ror": "https://ror.org/0199e1957",
-        "name": "University of Dar es Salaam",
-        "short_name": "UDSM",
-        "country": "Tanzania",
-        "sub_region": "East Africa",
-    },
-    {
-        "file": "rwanda.json",
-        "ror": "https://ror.org/02yr01r27",
-        "name": "University of Rwanda",
-        "short_name": "UR",
-        "country": "Rwanda",
-        "sub_region": "East Africa",
-    },
-]
-
-# Write institutional configs
-os.makedirs("config/institutions", exist_ok=True)
-for u in new_universities:
-    cfg = {
-        "ror": u["ror"],
-        "name": u["name"],
-        "short_name": u["short_name"],
-        "country": u["country"],
-        "sub_region": u["sub_region"],  # explicitly add sub-region to config files
-        "staff_file": f"data/{u['short_name'].lower().replace(' ', '_')}_staff.json",
-        "affiliation_patterns": [u["name"], u["short_name"], f"{u['name']} Department"],
-        "faculties": [
-            "Science",
-            "Humanities",
-            "Engineering",
-            "Medicine",
-            "Social Sciences",
-            "Arts",
-            "Law",
-        ],
-        "crawler_settings": {
-            "rate_limit": 2.0,
-            "concurrent_requests": 8,
-            "retry_times": 3,
-            "download_delay": 2.0,
-        },
-    }
-    with open(f"config/institutions/{u['file']}", "w", encoding="utf-8") as f:
-        json.dump(cfg, f, indent=2, ensure_ascii=False)
-
-print("Generated 15 new institutional config files.")
+import json
+import os
+
+# Sub-regions and countries
+subregions = {
+    "North Africa": ["Egypt", "Morocco", "Algeria", "Tunisia", "Libya", "Sudan"],
+    "West Africa": [
+        "Nigeria",
+        "Ghana",
+        "Senegal",
+        "Cote d'Ivoire",
+        "Benin",
+        "Burkina Faso",
+        "Cape Verde",
+        "Gambia",
+        "Guinea",
+        "Guinea-Bissau",
+        "Liberia",
+        "Mali",
+        "Mauritania",
+        "Niger",
+        "Sierra Leone",
+        "Togo",
+    ],
+    "East Africa": [
+        "Kenya",
+        "Uganda",
+        "Tanzania",
+        "Ethiopia",
+        "Rwanda",
+        "Burundi",
+        "Djibouti",
+        "Eritrea",
+        "Somalia",
+        "South Sudan",
+        "Madagascar",
+        "Mauritius",
+        "Seychelles",
+        "Comoros",
+    ],
+    "Southern Africa": [
+        "South Africa",
+        "Zimbabwe",
+        "Zambia",
+        "Namibia",
+        "Botswana",
+        "Lesotho",
+        "Eswatini",
+        "Malawi",
+        "Mozambique",
+    ],
+    "Central Africa": [
+        "Cameroon",
+        "DR Congo",
+        "Angola",
+        "Gabon",
+        "Republic of the Congo",
+        "Central African Republic",
+        "Chad",
+        "Equatorial Guinea",
+        "Sao Tome and Principe",
+    ],
+}
+
+# Major cities for generation if needed
+capitals = {
+    "Egypt": "Cairo",
+    "Morocco": "Rabat",
+    "Algeria": "Algiers",
+    "Tunisia": "Tunis",
+    "Libya": "Tripoli",
+    "Sudan": "Khartoum",
+    "Nigeria": "Abuja",
+    "Ghana": "Accra",
+    "Senegal": "Dakar",
+    "Cote d'Ivoire": "Yamoussoukro",
+    "Benin": "Porto-Novo",
+    "Burkina Faso": "Ouagadougou",
+    "Cape Verde": "Praia",
+    "Gambia": "Banjul",
+    "Guinea": "Conakry",
+    "Guinea-Bissau": "Bissau",
+    "Liberia": "Monrovia",
+    "Mali": "Bamako",
+    "Mauritania": "Nouakchott",
+    "Niger": "Niamey",
+    "Sierra Leone": "Freetown",
+    "Togo": "Lome",
+    "Kenya": "Nairobi",
+    "Uganda": "Kampala",
+    "Tanzania": "Dodoma",
+    "Ethiopia": "Addis Ababa",
+    "Rwanda": "Kigali",
+    "Burundi": "Gitega",
+    "Djibouti": "Djibouti",
+    "Eritrea": "Asmara",
+    "Somalia": "Mogadishu",
+    "South Sudan": "Juba",
+    "Madagascar": "Antananarivo",
+    "Mauritius": "Port Louis",
+    "Seychelles": "Victoria",
+    "Comoros": "Moroni",
+    "South Africa": "Pretoria",
+    "Zimbabwe": "Harare",
+    "Zambia": "Lusaka",
+    "Namibia": "Windhoek",
+    "Botswana": "Gaborone",
+    "Lesotho": "Maseru",
+    "Eswatini": "Mbabane",
+    "Malawi": "Lilongwe",
+    "Mozambique": "Maputo",
+    "Cameroon": "Yaounde",
+    "DR Congo": "Kinshasa",
+    "Angola": "Luanda",
+    "Gabon": "Libreville",
+    "Republic of the Congo": "Brazzaville",
+    "Central African Republic": "Bangui",
+    "Chad": "N'Djamena",
+    "Equatorial Guinea": "Malabo",
+    "Sao Tome and Principe": "Sao Tome",
+}
+
+# Hand-curated top universities to include (demo universities)
+curated_universities = {
+    # North Africa
+    "Egypt": [
+        {"name": "Cairo University", "ror": "https://ror.org/03c4mpy73"},
+        {"name": "Ain Shams University", "ror": "https://ror.org/034x7p097"},
+        {"name": "Alexandria University", "ror": "https://ror.org/02078r490"},
+        {"name": "Mansoura University", "ror": "https://ror.org/032p18087"},
+        {"name": "Assiut University", "ror": "https://ror.org/047fpp722"},
+    ],
+    "Morocco": [
+        {"name": "Université Mohammed V de Rabat", "ror": "https://ror.org/03vpy3v17"},
+        {"name": "Université Cadi Ayyad", "ror": "https://ror.org/0154pcf71"},
+        {
+            "name": "Université Hassan II de Casablanca",
+            "ror": "https://ror.org/013y27r38",
+        },
+    ],
+    "Tunisia": [
+        {"name": "Université de Tunis El Manar", "ror": "https://ror.org/050j3a172"},
+        {"name": "Université de Sfax", "ror": "https://ror.org/02157p641"},
+        {"name": "Université de Carthage", "ror": "https://ror.org/011yvpy71"},
+    ],
+    # Central Africa
+    "Cameroon": [
+        {"name": "Université de Yaoundé I", "ror": "https://ror.org/04h7g6177"},
+        {"name": "Université de Dschang", "ror": "https://ror.org/012y7p041"},
+        {"name": "Université de Douala", "ror": "https://ror.org/041y27r28"},
+    ],
+    "DR Congo": [
+        {"name": "Université de Kinshasa", "ror": "https://ror.org/05vzwad88"},
+        {"name": "Université de Lubumbashi", "ror": "https://ror.org/02rry3m21"},
+    ],
+    "Angola": [
+        {"name": "Université Agostinho Neto", "ror": "https://ror.org/00z2bpt98"}
+    ],
+    "Gabon": [
+        {
+            "name": "Université des Sciences et Techniques de Masuku",
+            "ror": "https://ror.org/059gqse72",
+        },
+        {"name": "Université Omar Bongo", "ror": "https://ror.org/041yp1812"},
+    ],
+    "Republic of the Congo": [
+        {"name": "Université Marien Ngouabi", "ror": "https://ror.org/02y1sra05"}
+    ],
+    # West Africa
+    "Nigeria": [
+        {"name": "University of Lagos", "ror": "https://ror.org/05rk03822"},
+        {"name": "University of Ibadan", "ror": "https://ror.org/01es5me90"},
+        {"name": "Covenant University", "ror": "https://ror.org/02n05rk12"},
+        {"name": "Obafemi Awolowo University", "ror": "https://ror.org/013pcr241"},
+        {"name": "University of Nigeria Nsukka", "ror": "https://ror.org/02kpy5732"},
+    ],
+    "Ghana": [
+        {"name": "University of Ghana", "ror": "https://ror.org/00zpy3v12"},
+        {
+            "name": "Kwame Nkrumah University of Science and Technology",
+            "ror": "https://ror.org/00x4mpy73",
+        },
+        {"name": "University of Cape Coast", "ror": "https://ror.org/01es3v123"},
+    ],
+    # Southern Africa
+    "South Africa": [
+        {"name": "University of Cape Town", "ror": "https://ror.org/017620319"},
+        {"name": "Stellenbosch University", "ror": "https://ror.org/05777p686"},
+        {"name": "University of the Witwatersrand", "ror": "https://ror.org/039482g93"},
+        {"name": "University of Pretoria", "ror": "https://ror.org/047fpp722"},
+        {"name": "University of KwaZulu-Natal", "ror": "https://ror.org/01267r312"},
+    ],
+    "Zimbabwe": [
+        {"name": "University of Zimbabwe", "ror": "https://ror.org/03w489125"},
+        {
+            "name": "National University of Science and Technology",
+            "ror": "https://ror.org/01y6mpy73",
+        },
+    ],
+    # East Africa
+    "Kenya": [
+        {"name": "University of Nairobi", "ror": "https://ror.org/01078r490"},
+        {"name": "Kenyatta University", "ror": "https://ror.org/01py3v171"},
+        {
+            "name": "Jomo Kenyatta University of Agriculture and Technology",
+            "ror": "https://ror.org/03pyvpy71",
+        },
+    ],
+    "Uganda": [
+        {"name": "Makerere University", "ror": "https://ror.org/05vzwad88"},
+        {
+            "name": "Mbarara University of Science and Technology",
+            "ror": "https://ror.org/0155pcf71",
+        },
+    ],
+    "Tanzania": [
+        {"name": "University of Dar es Salaam", "ror": "https://ror.org/0199e1957"},
+        {
+            "name": "Sokoine University of Agriculture",
+            "ror": "https://ror.org/011y27r38",
+        },
+    ],
+    "Ethiopia": [
+        {"name": "Addis Ababa University", "ror": "https://ror.org/01py3v171"}
+    ],
+    "Rwanda": [{"name": "University of Rwanda", "ror": "https://ror.org/02yr01r27"}],
+}
+
+# Generate 15-20 universities for each country
+registry_data = {}
+for subregion, countries in subregions.items():
+    registry_data[subregion] = {}
+    for country in countries:
+        cap = capitals.get(country, "City")
+        # Start with curated list or empty
+        unis = curated_universities.get(country, []).copy()
+
+        # Add generated universities to reach 15
+        existing_names = {u["name"] for u in unis}
+        templates = [
+            f"University of {cap}",
+            f"National University of {country}",
+            f"{country} University of Science and Technology",
+            f"{cap} Institute of Technology",
+            f"State University of {cap}",
+            f"{country} International University",
+            f"Pan-African University, {cap} Campus",
+            f"Catholic University of {country}",
+            f"Technical University of {cap}",
+            (
+                f"Ahmadu Bello University of {cap}"
+                if country == "Nigeria"
+                else f"Federal University of {cap}"
+            ),
+            f"Metropolitan University of {cap}",
+            f"Central University of {country}",
+            f"Presbyterian University of {country}",
+            f"Adventist University of {country}",
+            f"Islamic University of {country}",
+            f"Methodist University of {country}",
+            f"Covenant University of {cap}",
+            f"{cap} College of Medicine and Health Sciences",
+            f"Regional Institute of Information Technology, {cap}",
+            f"Greenfield University, {cap}",
+        ]
+
+        idx = 0
+        while len(unis) < 18:
+            name = templates[idx % len(templates)]
+            # ensure uniqueness
+            if name not in existing_names:
+                unis.append({"name": name, "ror": ""})
+                existing_names.add(name)
+            idx += 1
+
+        registry_data[subregion][country] = unis
+
+# Write university_registry.json
+os.makedirs("data", exist_ok=True)
+with open("data/university_registry.json", "w", encoding="utf-8") as f:
+    json.dump(registry_data, f, indent=2, ensure_ascii=False)
+print(
+    "Generated data/university_registry.json with 52 countries and 18 universities each."
+)
+
+# Configurations for the 15 new universities to make a total of 25 demo universities
+new_universities = [
+    # North (5)
+    {
+        "file": "cairo.json",
+        "ror": "https://ror.org/03c4mpy73",
+        "name": "Cairo University",
+        "short_name": "Cairo Univ",
+        "country": "Egypt",
+        "sub_region": "North Africa",
+    },
+    {
+        "file": "ainshams.json",
+        "ror": "https://ror.org/034x7p097",
+        "name": "Ain Shams University",
+        "short_name": "Ain Shams",
+        "country": "Egypt",
+        "sub_region": "North Africa",
+    },
+    {
+        "file": "alexandria.json",
+        "ror": "https://ror.org/02078r490",
+        "name": "Alexandria University",
+        "short_name": "Alexandria",
+        "country": "Egypt",
+        "sub_region": "North Africa",
+    },
+    {
+        "file": "tunis.json",
+        "ror": "https://ror.org/050j3a172",
+        "name": "Université de Tunis El Manar",
+        "short_name": "Tunis El Manar",
+        "country": "Tunisia",
+        "sub_region": "North Africa",
+    },
+    {
+        "file": "mohammedv.json",
+        "ror": "https://ror.org/03vpy3v17",
+        "name": "Université Mohammed V de Rabat",
+        "short_name": "Mohammed V",
+        "country": "Morocco",
+        "sub_region": "North Africa",
+    },
+    # Central (5)
+    {
+        "file": "yaoundei.json",
+        "ror": "https://ror.org/04h7g6177",
+        "name": "Université de Yaoundé I",
+        "short_name": "Yaoundé I",
+        "country": "Cameroon",
+        "sub_region": "Central Africa",
+    },
+    {
+        "file": "kinshasa.json",
+        "ror": "https://ror.org/05vzwad88",
+        "name": "Université de Kinshasa",
+        "short_name": "UNIKIN",
+        "country": "DR Congo",
+        "sub_region": "Central Africa",
+    },
+    {
+        "file": "agostinhoneto.json",
+        "ror": "https://ror.org/00z2bpt98",
+        "name": "Université Agostinho Neto",
+        "short_name": "Agostinho Neto",
+        "country": "Angola",
+        "sub_region": "Central Africa",
+    },
+    {
+        "file": "marienngouabi.json",
+        "ror": "https://ror.org/02y1sra05",
+        "name": "Université Marien Ngouabi",
+        "short_name": "Marien Ngouabi",
+        "country": "Republic of the Congo",
+        "sub_region": "Central Africa",
+    },
+    {
+        "file": "masuku.json",
+        "ror": "https://ror.org/059gqse72",
+        "name": "Université des Sciences et Techniques de Masuku",
+        "short_name": "USTM Masuku",
+        "country": "Gabon",
+        "sub_region": "Central Africa",
+    },
+    # Southern (+3 new ones)
+    {
+        "file": "wits.json",
+        "ror": "https://ror.org/039482g93",
+        "name": "University of the Witwatersrand",
+        "short_name": "Wits",
+        "country": "South Africa",
+        "sub_region": "Southern Africa",
+    },
+    {
+        "file": "pretoria.json",
+        "ror": "https://ror.org/047fpp722",
+        "name": "University of Pretoria",
+        "short_name": "UP",
+        "country": "South Africa",
+        "sub_region": "Southern Africa",
+    },
+    {
+        "file": "zimbabwe.json",
+        "ror": "https://ror.org/03w489125",
+        "name": "University of Zimbabwe",
+        "short_name": "UZ",
+        "country": "Zimbabwe",
+        "sub_region": "Southern Africa",
+    },
+    # East (+2 new ones)
+    {
+        "file": "daressalaam.json",
+        "ror": "https://ror.org/0199e1957",
+        "name": "University of Dar es Salaam",
+        "short_name": "UDSM",
+        "country": "Tanzania",
+        "sub_region": "East Africa",
+    },
+    {
+        "file": "rwanda.json",
+        "ror": "https://ror.org/02yr01r27",
+        "name": "University of Rwanda",
+        "short_name": "UR",
+        "country": "Rwanda",
+        "sub_region": "East Africa",
+    },
+]
+
+# Write institutional configs
+os.makedirs("config/institutions", exist_ok=True)
+for u in new_universities:
+    cfg = {
+        "ror": u["ror"],
+        "name": u["name"],
+        "short_name": u["short_name"],
+        "country": u["country"],
+        "sub_region": u["sub_region"],  # explicitly add sub-region to config files
+        "staff_file": f"data/{u['short_name'].lower().replace(' ', '_')}_staff.json",
+        "affiliation_patterns": [u["name"], u["short_name"], f"{u['name']} Department"],
+        "faculties": [
+            "Science",
+            "Humanities",
+            "Engineering",
+            "Medicine",
+            "Social Sciences",
+            "Arts",
+            "Law",
+        ],
+        "crawler_settings": {
+            "rate_limit": 2.0,
+            "concurrent_requests": 8,
+            "retry_times": 3,
+            "download_delay": 2.0,
+        },
+    }
+    with open(f"config/institutions/{u['file']}", "w", encoding="utf-8") as f:
+        json.dump(cfg, f, indent=2, ensure_ascii=False)
+
+print("Generated 15 new institutional config files.")

scripts/harvest_staff_openalex.py

@@ -1,315 +1,315 @@
-"""
-Staff Harvester — fetches real staff names, ORCIDs, departments from OpenAlex
-for every configured institution. Saves enriched JSON to data/{inst}_staff.json.
-
-Usage:
-    python scripts/harvest_staff_openalex.py                  # all institutions
-    python scripts/harvest_staff_openalex.py --institution unilag
-    python scripts/harvest_staff_openalex.py --dry-run        # just print counts
-"""
-
-import argparse
-import json
-import logging
-import os
-import sys
-import time
-import urllib.error
-import urllib.parse
-import urllib.request
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from uraas.config.institutions import get_registry, reset_registry
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
-log = logging.getLogger(__name__)
-
-OPENALEX_BASE = "https://api.openalex.org"
-MAILTO = "uraas-bot@research.edu.ng"
-MAX_AUTHORS = 500  # cap per institution to avoid very long runs
-DELAY = 0.5  # seconds between requests (polite)
-
-
-def _get(url: str, retries: int = 3) -> dict:
-    """Simple urllib GET with retries."""
-    for attempt in range(retries):
-        try:
-            req = urllib.request.Request(
-                url, headers={"User-Agent": f"URAAS/1.0 (mailto:{MAILTO})"}
-            )
-            with urllib.request.urlopen(req, timeout=20) as resp:
-                return json.loads(resp.read().decode())
-        except urllib.error.HTTPError as e:
-            if e.code == 429:
-                wait = 5 * (attempt + 1)
-                log.warning(f"Rate limited, waiting {wait}s …")
-                time.sleep(wait)
-            else:
-                log.error(f"HTTP {e.code} for {url}")
-                break
-        except Exception as e:
-            log.error(f"Request error ({attempt+1}/{retries}): {e}")
-            time.sleep(2)
-    return {}
-
-
-def harvest_institution(inst_config, dry_run: bool = False) -> list:
-    """
-    Harvest staff by looking at recent works from the institution on OpenAlex.
-    Extracts unique authors from the authorships array.
-    Returns list of rich staff dicts: {name, orcid, department, faculty, openalex_id, paper_count}
-    """
-    ror_url = inst_config.ror
-    inst_name = inst_config.name
-    log.info(f"Harvesting staff for {inst_name} (ROR: {ror_url}) …")
-
-    unique_staff = {}
-    cursor = "*"
-    page = 0
-    max_pages = 50  # Limit to 50 pages (10k works max) to avoid running forever
-
-    while len(unique_staff) < MAX_AUTHORS and page < max_pages:
-        # We query the works endpoint using the exact ROR url
-        url = (
-            f"{OPENALEX_BASE}/works"
-            f"?filter=institutions.ror:{urllib.parse.quote(ror_url)}"
-            f"&select=authorships"
-            f"&per-page=200"
-            f"&cursor={urllib.parse.quote(cursor)}"
-            f"&mailto={MAILTO}"
-        )
-        data = _get(url)
-        if not data:
-            break
-
-        results = data.get("results", [])
-        if not results:
-            break
-
-        for work in results:
-            for authorship in work.get("authorships", []):
-                # Ensure the author is affiliated with our target institution for this work
-                is_affiliated = False
-                for inst in authorship.get("institutions", []):
-                    if inst.get("ror") == ror_url:
-                        is_affiliated = True
-                        break
-
-                if not is_affiliated:
-                    continue
-
-                author = authorship.get("author", {})
-                aid = author.get("id")
-                if not aid or aid in unique_staff:
-                    if aid in unique_staff:
-                        unique_staff[aid]["paper_count"] += 1
-                    continue
-
-                name = author.get("display_name", "").strip()
-                if not name:
-                    continue
-
-                orcid_url = author.get("orcid", "")
-                orcid = (
-                    orcid_url.replace("https://orcid.org/", "") if orcid_url else None
-                )
-
-                # We can't get concepts easily from works authorships without extra queries,
-                # so we will leave faculty and department empty for now.
-
-                unique_staff[aid] = {
-                    "name": name,
-                    "orcid": orcid,
-                    "department": None,
-                    "faculty": None,
-                    "openalex_id": aid.replace("https://openalex.org/", ""),
-                    "paper_count": 1,
-                }
-
-                if len(unique_staff) >= MAX_AUTHORS:
-                    break
-
-            if len(unique_staff) >= MAX_AUTHORS:
-                break
-
-        log.info(
-            f"  Page {page+1}: Processed {len(results)} works | Unique staff so far: {len(unique_staff)}"
-        )
-        page += 1
-        time.sleep(DELAY)
-
-        meta = data.get("meta", {})
-        cursor = meta.get("next_cursor")
-        if not cursor:
-            break
-
-    staff_list = list(unique_staff.values())
-    log.info(f"  Harvested {len(staff_list)} staff for {inst_name}")
-    return staff_list
-
-
-def _map_concept_to_faculty(concept: str, faculties: list) -> str:
-    """Rough concept→faculty mapping via keyword overlap."""
-    concept_lower = concept.lower()
-    faculty_map = {
-        "medicine": ["health", "medicine", "clinical", "nursing", "pharmacy", "dental"],
-        "engineering": [
-            "engineering",
-            "technology",
-            "mechanical",
-            "electrical",
-            "civil",
-            "chemical",
-        ],
-        "science": [
-            "biology",
-            "chemistry",
-            "physics",
-            "mathematics",
-            "statistics",
-            "computer",
-        ],
-        "arts": [
-            "literature",
-            "linguistics",
-            "language",
-            "history",
-            "philosophy",
-            "arts",
-        ],
-        "social": [
-            "sociology",
-            "economics",
-            "political",
-            "psychology",
-            "anthropology",
-            "social",
-        ],
-        "law": ["law", "legal", "jurisprudence", "criminology"],
-        "education": ["education", "pedagogy", "teaching", "curriculum"],
-        "agriculture": ["agriculture", "botany", "zoology", "ecology", "forestry"],
-        "management": ["business", "management", "accounting", "finance", "marketing"],
-        "environmental": [
-            "environment",
-            "urban",
-            "planning",
-            "geography",
-            "architecture",
-        ],
-    }
-    for fac_key, keywords in faculty_map.items():
-        if any(kw in concept_lower for kw in keywords):
-            # Try to match to actual faculty names
-            for f in faculties:
-                if fac_key in f.lower() or any(kw in f.lower() for kw in keywords):
-                    return f
-    return None
-
-
-def get_orcid_details(orcid: str) -> dict:
-    """Fetch name and affiliation details from ORCID public API."""
-    url = f"https://pub.orcid.org/v3.0/{orcid}/person"
-    try:
-        req = urllib.request.Request(
-            url,
-            headers={
-                "Accept": "application/json",
-                "User-Agent": f"URAAS/1.0 (mailto:{MAILTO})",
-            },
-        )
-        with urllib.request.urlopen(req, timeout=15) as resp:
-            data = json.loads(resp.read().decode())
-        affiliations = data.get("activities-summary", {})
-        return {"orcid": orcid}
-    except Exception:
-        return {}
-
-
-def save_staff(inst_config, staff: list, dry_run: bool = False):
-    """Save staff list to data/{short_name_lower}_staff.json"""
-    short = inst_config.short_name.lower()
-    # Resolve base directory
-    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-    out_path = os.path.join(base_dir, "data", f"{short}_staff.json")
-
-    if dry_run:
-        log.info(f"[DRY-RUN] Would save {len(staff)} staff records to {out_path}")
-        return
-
-    os.makedirs(os.path.dirname(out_path), exist_ok=True)
-    with open(out_path, "w", encoding="utf-8") as f:
-        json.dump(staff, f, indent=2, ensure_ascii=False)
-    log.info(f"Saved {len(staff)} staff records → {out_path}")
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Harvest staff from OpenAlex for URAAS institutions"
-    )
-    parser.add_argument(
-        "--institution",
-        type=str,
-        default=None,
-        help="Single institution short name (default: all)",
-    )
-    parser.add_argument(
-        "--dry-run", action="store_true", help="Print counts without saving"
-    )
-    args = parser.parse_args()
-
-    reset_registry()
-    registry = get_registry()
-    all_insts = registry.list_all()
-
-    if args.institution:
-        inst = registry.get(args.institution)
-        if not inst:
-            print(f"ERROR: Institution '{args.institution}' not found")
-            sys.exit(1)
-        target_insts = [inst]
-    else:
-        target_insts = all_insts
-
-    print(f"\n{'='*60}")
-    print(f"URAAS Staff Harvester — OpenAlex")
-    print(f"Institutions: {len(target_insts)}")
-    print(f"{'='*60}\n")
-
-    summary = []
-    for inst in target_insts:
-        try:
-            staff = harvest_institution(inst, dry_run=args.dry_run)
-            orcid_count = sum(1 for s in staff if s.get("orcid"))
-            save_staff(inst, staff, dry_run=args.dry_run)
-            summary.append(
-                {
-                    "institution": inst.name,
-                    "staff_total": len(staff),
-                    "with_orcid": orcid_count,
-                }
-            )
-        except Exception as e:
-            log.error(f"Failed harvesting {inst.name}: {e}")
-            summary.append(
-                {"institution": inst.name, "staff_total": 0, "with_orcid": 0}
-            )
-        time.sleep(1)
-
-    print(f"\n{'='*60}")
-    print("HARVEST SUMMARY")
-    print(f"{'='*60}")
-    total_staff = 0
-    total_orcid = 0
-    for s in summary:
-        print(
-            f"  {s['institution']:<45} {s['staff_total']:>5} staff  {s['with_orcid']:>4} ORCID"
-        )
-        total_staff += s["staff_total"]
-        total_orcid += s["with_orcid"]
-    print(f"{'-'*60}")
-    print(f"  {'TOTAL':<45} {total_staff:>5} staff  {total_orcid:>4} ORCID")
-    print(f"{'='*60}\n")
-
-
-if __name__ == "__main__":
-    main()
+"""
+Staff Harvester — fetches real staff names, ORCIDs, departments from OpenAlex
+for every configured institution. Saves enriched JSON to data/{inst}_staff.json.
+
+Usage:
+    python scripts/harvest_staff_openalex.py                  # all institutions
+    python scripts/harvest_staff_openalex.py --institution unilag
+    python scripts/harvest_staff_openalex.py --dry-run        # just print counts
+"""
+
+import argparse
+import json
+import logging
+import os
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from uraas.config.institutions import get_registry, reset_registry
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+
+OPENALEX_BASE = "https://api.openalex.org"
+MAILTO = "uraas-bot@research.edu.ng"
+MAX_AUTHORS = 500  # cap per institution to avoid very long runs
+DELAY = 0.5  # seconds between requests (polite)
+
+
+def _get(url: str, retries: int = 3) -> dict:
+    """Simple urllib GET with retries."""
+    for attempt in range(retries):
+        try:
+            req = urllib.request.Request(
+                url, headers={"User-Agent": f"URAAS/1.0 (mailto:{MAILTO})"}
+            )
+            with urllib.request.urlopen(req, timeout=20) as resp:
+                return json.loads(resp.read().decode())
+        except urllib.error.HTTPError as e:
+            if e.code == 429:
+                wait = 5 * (attempt + 1)
+                log.warning(f"Rate limited, waiting {wait}s …")
+                time.sleep(wait)
+            else:
+                log.error(f"HTTP {e.code} for {url}")
+                break
+        except Exception as e:
+            log.error(f"Request error ({attempt+1}/{retries}): {e}")
+            time.sleep(2)
+    return {}
+
+
+def harvest_institution(inst_config, dry_run: bool = False) -> list:
+    """
+    Harvest staff by looking at recent works from the institution on OpenAlex.
+    Extracts unique authors from the authorships array.
+    Returns list of rich staff dicts: {name, orcid, department, faculty, openalex_id, paper_count}
+    """
+    ror_url = inst_config.ror
+    inst_name = inst_config.name
+    log.info(f"Harvesting staff for {inst_name} (ROR: {ror_url}) …")
+
+    unique_staff = {}
+    cursor = "*"
+    page = 0
+    max_pages = 50  # Limit to 50 pages (10k works max) to avoid running forever
+
+    while len(unique_staff) < MAX_AUTHORS and page < max_pages:
+        # We query the works endpoint using the exact ROR url
+        url = (
+            f"{OPENALEX_BASE}/works"
+            f"?filter=institutions.ror:{urllib.parse.quote(ror_url)}"
+            f"&select=authorships"
+            f"&per-page=200"
+            f"&cursor={urllib.parse.quote(cursor)}"
+            f"&mailto={MAILTO}"
+        )
+        data = _get(url)
+        if not data:
+            break
+
+        results = data.get("results", [])
+        if not results:
+            break
+
+        for work in results:
+            for authorship in work.get("authorships", []):
+                # Ensure the author is affiliated with our target institution for this work
+                is_affiliated = False
+                for inst in authorship.get("institutions", []):
+                    if inst.get("ror") == ror_url:
+                        is_affiliated = True
+                        break
+
+                if not is_affiliated:
+                    continue
+
+                author = authorship.get("author", {})
+                aid = author.get("id")
+                if not aid or aid in unique_staff:
+                    if aid in unique_staff:
+                        unique_staff[aid]["paper_count"] += 1
+                    continue
+
+                name = author.get("display_name", "").strip()
+                if not name:
+                    continue
+
+                orcid_url = author.get("orcid", "")
+                orcid = (
+                    orcid_url.replace("https://orcid.org/", "") if orcid_url else None
+                )
+
+                # We can't get concepts easily from works authorships without extra queries,
+                # so we will leave faculty and department empty for now.
+
+                unique_staff[aid] = {
+                    "name": name,
+                    "orcid": orcid,
+                    "department": None,
+                    "faculty": None,
+                    "openalex_id": aid.replace("https://openalex.org/", ""),
+                    "paper_count": 1,
+                }
+
+                if len(unique_staff) >= MAX_AUTHORS:
+                    break
+
+            if len(unique_staff) >= MAX_AUTHORS:
+                break
+
+        log.info(
+            f"  Page {page+1}: Processed {len(results)} works | Unique staff so far: {len(unique_staff)}"
+        )
+        page += 1
+        time.sleep(DELAY)
+
+        meta = data.get("meta", {})
+        cursor = meta.get("next_cursor")
+        if not cursor:
+            break
+
+    staff_list = list(unique_staff.values())
+    log.info(f"  Harvested {len(staff_list)} staff for {inst_name}")
+    return staff_list
+
+
+def _map_concept_to_faculty(concept: str, faculties: list) -> str:
+    """Rough concept→faculty mapping via keyword overlap."""
+    concept_lower = concept.lower()
+    faculty_map = {
+        "medicine": ["health", "medicine", "clinical", "nursing", "pharmacy", "dental"],
+        "engineering": [
+            "engineering",
+            "technology",
+            "mechanical",
+            "electrical",
+            "civil",
+            "chemical",
+        ],
+        "science": [
+            "biology",
+            "chemistry",
+            "physics",
+            "mathematics",
+            "statistics",
+            "computer",
+        ],
+        "arts": [
+            "literature",
+            "linguistics",
+            "language",
+            "history",
+            "philosophy",
+            "arts",
+        ],
+        "social": [
+            "sociology",
+            "economics",
+            "political",
+            "psychology",
+            "anthropology",
+            "social",
+        ],
+        "law": ["law", "legal", "jurisprudence", "criminology"],
+        "education": ["education", "pedagogy", "teaching", "curriculum"],
+        "agriculture": ["agriculture", "botany", "zoology", "ecology", "forestry"],
+        "management": ["business", "management", "accounting", "finance", "marketing"],
+        "environmental": [
+            "environment",
+            "urban",
+            "planning",
+            "geography",
+            "architecture",
+        ],
+    }
+    for fac_key, keywords in faculty_map.items():
+        if any(kw in concept_lower for kw in keywords):
+            # Try to match to actual faculty names
+            for f in faculties:
+                if fac_key in f.lower() or any(kw in f.lower() for kw in keywords):
+                    return f
+    return None
+
+
+def get_orcid_details(orcid: str) -> dict:
+    """Fetch name and affiliation details from ORCID public API."""
+    url = f"https://pub.orcid.org/v3.0/{orcid}/person"
+    try:
+        req = urllib.request.Request(
+            url,
+            headers={
+                "Accept": "application/json",
+                "User-Agent": f"URAAS/1.0 (mailto:{MAILTO})",
+            },
+        )
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            data = json.loads(resp.read().decode())
+        affiliations = data.get("activities-summary", {})
+        return {"orcid": orcid}
+    except Exception:
+        return {}
+
+
+def save_staff(inst_config, staff: list, dry_run: bool = False):
+    """Save staff list to data/{short_name_lower}_staff.json"""
+    short = inst_config.short_name.lower()
+    # Resolve base directory
+    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    out_path = os.path.join(base_dir, "data", f"{short}_staff.json")
+
+    if dry_run:
+        log.info(f"[DRY-RUN] Would save {len(staff)} staff records to {out_path}")
+        return
+
+    os.makedirs(os.path.dirname(out_path), exist_ok=True)
+    with open(out_path, "w", encoding="utf-8") as f:
+        json.dump(staff, f, indent=2, ensure_ascii=False)
+    log.info(f"Saved {len(staff)} staff records → {out_path}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Harvest staff from OpenAlex for URAAS institutions"
+    )
+    parser.add_argument(
+        "--institution",
+        type=str,
+        default=None,
+        help="Single institution short name (default: all)",
+    )
+    parser.add_argument(
+        "--dry-run", action="store_true", help="Print counts without saving"
+    )
+    args = parser.parse_args()
+
+    reset_registry()
+    registry = get_registry()
+    all_insts = registry.list_all()
+
+    if args.institution:
+        inst = registry.get(args.institution)
+        if not inst:
+            print(f"ERROR: Institution '{args.institution}' not found")
+            sys.exit(1)
+        target_insts = [inst]
+    else:
+        target_insts = all_insts
+
+    print(f"\n{'='*60}")
+    print(f"URAAS Staff Harvester — OpenAlex")
+    print(f"Institutions: {len(target_insts)}")
+    print(f"{'='*60}\n")
+
+    summary = []
+    for inst in target_insts:
+        try:
+            staff = harvest_institution(inst, dry_run=args.dry_run)
+            orcid_count = sum(1 for s in staff if s.get("orcid"))
+            save_staff(inst, staff, dry_run=args.dry_run)
+            summary.append(
+                {
+                    "institution": inst.name,
+                    "staff_total": len(staff),
+                    "with_orcid": orcid_count,
+                }
+            )
+        except Exception as e:
+            log.error(f"Failed harvesting {inst.name}: {e}")
+            summary.append(
+                {"institution": inst.name, "staff_total": 0, "with_orcid": 0}
+            )
+        time.sleep(1)
+
+    print(f"\n{'='*60}")
+    print("HARVEST SUMMARY")
+    print(f"{'='*60}")
+    total_staff = 0
+    total_orcid = 0
+    for s in summary:
+        print(
+            f"  {s['institution']:<45} {s['staff_total']:>5} staff  {s['with_orcid']:>4} ORCID"
+        )
+        total_staff += s["staff_total"]
+        total_orcid += s["with_orcid"]
+    print(f"{'-'*60}")
+    print(f"  {'TOTAL':<45} {total_staff:>5} staff  {total_orcid:>4} ORCID")
+    print(f"{'='*60}\n")
+
+
+if __name__ == "__main__":
+    main()

scripts/init_db.py

@@ -1,78 +1,78 @@
-"""
-Database Initialization Script
-Creates all tables and seeds Communities and Collections based on UNILAG structure.
-"""
-
-import os
-import sys
-
-# Add project root to path (parent of scripts/)
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from uraas.database import Collection, Community, SessionLocal, init_db
-from uraas.utils.unilag_classifier import UNILAG_STRUCTURE
-
-
-def seed_communities_and_collections():
-    """Seed the database with UNILAG faculty and department structure."""
-    session = SessionLocal()
-
-    try:
-        print("Seeding Communities (Faculties) and Collections (Departments)...")
-
-        for faculty_name, departments in UNILAG_STRUCTURE.items():
-            # Check if community exists
-            community = session.query(Community).filter_by(name=faculty_name).first()
-            if not community:
-                community = Community(name=faculty_name)
-                session.add(community)
-                session.flush()
-                print(f"  Created Community: {faculty_name}")
-
-            # Create collections (departments) under this community
-            for dept_name, keywords in departments.items():
-                collection = session.query(Collection).filter_by(name=dept_name).first()
-                if not collection:
-                    collection = Collection(
-                        community_id=community.id,
-                        name=dept_name,
-                        keywords=", ".join(keywords),
-                    )
-                    session.add(collection)
-                    print(f"    Created Collection: {dept_name}")
-
-        session.commit()
-        print("\n[OK] Database seeding completed successfully!")
-        print(f"  Total Communities: {session.query(Community).count()}")
-        print(f"  Total Collections: {session.query(Collection).count()}")
-
-    except Exception as e:
-        print(f"\n[ERR] Error seeding database: {e}")
-        session.rollback()
-        raise
-    finally:
-        session.close()
-
-
-def main():
-    print("=" * 60)
-    print("URAAS Database Initialization")
-    print("=" * 60)
-    print()
-
-    # Create all tables
-    print("Creating database tables...")
-    init_db()
-    print("[OK] Tables created successfully!")
-    print()
-
-    # Seed communities and collections
-    seed_communities_and_collections()
-    print()
-    print("=" * 60)
-    print("Database is ready for use!")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    main()
+"""
+Database Initialization Script
+Creates all tables and seeds Communities and Collections based on UNILAG structure.
+"""
+
+import os
+import sys
+
+# Add project root to path (parent of scripts/)
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from uraas.database import Collection, Community, SessionLocal, init_db
+from uraas.utils.unilag_classifier import UNILAG_STRUCTURE
+
+
+def seed_communities_and_collections():
+    """Seed the database with UNILAG faculty and department structure."""
+    session = SessionLocal()
+
+    try:
+        print("Seeding Communities (Faculties) and Collections (Departments)...")
+
+        for faculty_name, departments in UNILAG_STRUCTURE.items():
+            # Check if community exists
+            community = session.query(Community).filter_by(name=faculty_name).first()
+            if not community:
+                community = Community(name=faculty_name)
+                session.add(community)
+                session.flush()
+                print(f"  Created Community: {faculty_name}")
+
+            # Create collections (departments) under this community
+            for dept_name, keywords in departments.items():
+                collection = session.query(Collection).filter_by(name=dept_name).first()
+                if not collection:
+                    collection = Collection(
+                        community_id=community.id,
+                        name=dept_name,
+                        keywords=", ".join(keywords),
+                    )
+                    session.add(collection)
+                    print(f"    Created Collection: {dept_name}")
+
+        session.commit()
+        print("\n[OK] Database seeding completed successfully!")
+        print(f"  Total Communities: {session.query(Community).count()}")
+        print(f"  Total Collections: {session.query(Collection).count()}")
+
+    except Exception as e:
+        print(f"\n[ERR] Error seeding database: {e}")
+        session.rollback()
+        raise
+    finally:
+        session.close()
+
+
+def main():
+    print("=" * 60)
+    print("URAAS Database Initialization")
+    print("=" * 60)
+    print()
+
+    # Create all tables
+    print("Creating database tables...")
+    init_db()
+    print("[OK] Tables created successfully!")
+    print()
+
+    # Seed communities and collections
+    seed_communities_and_collections()
+    print()
+    print("=" * 60)
+    print("Database is ready for use!")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

scripts/migrate.py

@@ -1,44 +1,44 @@
-"""Run once to add new APA columns to the existing SQLite database."""
-
-import sys
-
-sys.path.insert(0, ".")
-from sqlalchemy import inspect, text
-
-from uraas.database import engine
-
-NEW_COLS = [
-    ("items", "dc_type", "TEXT"),
-    ("items", "dc_language", "TEXT"),
-    ("items", "dc_subject", "TEXT"),
-    ("items", "docid", "TEXT UNIQUE"),
-    ("items", "docid_assigned_at", "DATETIME"),
-    ("items", "content_type", 'TEXT DEFAULT "research_paper"'),
-    ("items", "tk_label", "TEXT"),
-    ("items", "tk_community", "TEXT"),
-    ("items", "patent_id", "TEXT"),
-    ("items", "patent_date", "DATETIME"),
-    ("items", "language_code", "TEXT"),
-    ("items", "is_african_language", "INTEGER DEFAULT 0"),
-    ("items", "sdg_tags", "TEXT"),
-    ("items", "ai_keywords", "TEXT"),
-    ("authors", "orcid", "TEXT"),
-    ("authors", "ror", "TEXT"),
-    ("communities", "ror_id", "TEXT"),
-    ("communities", "institution", "TEXT"),
-]
-
-inspector = inspect(engine)
-with engine.connect() as conn:
-    for table, col, col_type in NEW_COLS:
-        existing = [c["name"] for c in inspector.get_columns(table)]
-        if col not in existing:
-            # SQLite doesn't support UNIQUE in ALTER TABLE — skip constraint
-            safe_type = col_type.replace(" UNIQUE", "")
-            conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {col} {safe_type}"))
-            print(f"  + {table}.{col}")
-        else:
-            print(f"  . {table}.{col} (exists)")
-    conn.commit()
-
-print("Migration complete.")
+"""Run once to add new APA columns to the existing SQLite database."""
+
+import sys
+
+sys.path.insert(0, ".")
+from sqlalchemy import inspect, text
+
+from uraas.database import engine
+
+NEW_COLS = [
+    ("items", "dc_type", "TEXT"),
+    ("items", "dc_language", "TEXT"),
+    ("items", "dc_subject", "TEXT"),
+    ("items", "docid", "TEXT UNIQUE"),
+    ("items", "docid_assigned_at", "DATETIME"),
+    ("items", "content_type", 'TEXT DEFAULT "research_paper"'),
+    ("items", "tk_label", "TEXT"),
+    ("items", "tk_community", "TEXT"),
+    ("items", "patent_id", "TEXT"),
+    ("items", "patent_date", "DATETIME"),
+    ("items", "language_code", "TEXT"),
+    ("items", "is_african_language", "INTEGER DEFAULT 0"),
+    ("items", "sdg_tags", "TEXT"),
+    ("items", "ai_keywords", "TEXT"),
+    ("authors", "orcid", "TEXT"),
+    ("authors", "ror", "TEXT"),
+    ("communities", "ror_id", "TEXT"),
+    ("communities", "institution", "TEXT"),
+]
+
+inspector = inspect(engine)
+with engine.connect() as conn:
+    for table, col, col_type in NEW_COLS:
+        existing = [c["name"] for c in inspector.get_columns(table)]
+        if col not in existing:
+            # SQLite doesn't support UNIQUE in ALTER TABLE — skip constraint
+            safe_type = col_type.replace(" UNIQUE", "")
+            conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {col} {safe_type}"))
+            print(f"  + {table}.{col}")
+        else:
+            print(f"  . {table}.{col} (exists)")
+    conn.commit()
+
+print("Migration complete.")

scripts/migrate_2026_upgrade.py

@@ -1,86 +1,86 @@
-"""
-Schema migration: 2026 UNESCO upgrade. Idempotent; safe on SQLite + Postgres.
-
-Adds to items:
-  - alignment_scores (JSON TEXT) + alignment_version  (framework alignment)
-  - coauthor_countries / african_country_count / is_intra_african  (collaboration)
-  - openalex_id / counts_by_year / cited_by_count / african_citation_share  (citations)
-  - ark / ark_assigned_at  (ARK persistent identifiers)
-
-New tables (item_affiliations, alignment_aggregates) are created by
-scripts/init_db.py via Base.metadata.create_all — run init_db.py first.
-"""
-
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from sqlalchemy import inspect, text
-
-from uraas.database import engine
-
-# (column, DDL type clause) — types chosen to work on both SQLite and Postgres.
-ITEMS_COLUMNS = [
-    ("alignment_scores", "TEXT"),
-    ("alignment_version", "INTEGER DEFAULT 0"),
-    ("coauthor_countries", "TEXT"),
-    ("african_country_count", "INTEGER DEFAULT 0"),
-    ("is_intra_african", "BOOLEAN DEFAULT FALSE"),
-    ("openalex_id", "VARCHAR(64)"),
-    ("counts_by_year", "TEXT"),
-    ("cited_by_count", "INTEGER DEFAULT 0"),
-    ("african_citation_share", "FLOAT"),
-    ("ark", "VARCHAR(128)"),
-    ("ark_assigned_at", "TIMESTAMP"),
-]
-
-INDEXES = [
-    "CREATE INDEX IF NOT EXISTS ix_items_is_intra_african ON items (is_intra_african)",
-    "CREATE UNIQUE INDEX IF NOT EXISTS ux_items_ark ON items (ark)",
-]
-
-
-def column_exists(insp, table: str, column: str) -> bool:
-    return column in {c["name"] for c in insp.get_columns(table)}
-
-
-def main() -> int:
-    print("Migration: 2026 upgrade (alignment / collaboration / citations / ARK)")
-
-    dialect = engine.dialect.name
-    print(f"Dialect: {dialect}")
-
-    insp = inspect(engine)
-    statements = []
-    for column, ddl_type in ITEMS_COLUMNS:
-        if column_exists(insp, "items", column):
-            print(f"  {column} already present, skipping")
-            continue
-        if dialect == "sqlite" and "BOOLEAN" in ddl_type:
-            # SQLite stores booleans as integers
-            ddl_type = ddl_type.replace("BOOLEAN", "INTEGER").replace("FALSE", "0")
-        statements.append(f"ALTER TABLE items ADD COLUMN {column} {ddl_type}")
-
-    if statements:
-        with engine.begin() as conn:
-            for stmt in statements:
-                print(f"  -> {stmt}")
-                conn.execute(text(stmt))
-    else:
-        print("  All columns already present.")
-
-    for stmt in INDEXES:
-        try:
-            with engine.begin() as conn:
-                conn.execute(text(stmt))
-                print(f"  -> {stmt.split(' ON ')[0].replace('CREATE ', '').strip()} ensured")
-        except Exception as e:
-            print(f"  (index creation skipped: {e})")
-
-    print("Done.")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+"""
+Schema migration: 2026 UNESCO upgrade. Idempotent; safe on SQLite + Postgres.
+
+Adds to items:
+  - alignment_scores (JSON TEXT) + alignment_version  (framework alignment)
+  - coauthor_countries / african_country_count / is_intra_african  (collaboration)
+  - openalex_id / counts_by_year / cited_by_count / african_citation_share  (citations)
+  - ark / ark_assigned_at  (ARK persistent identifiers)
+
+New tables (item_affiliations, alignment_aggregates) are created by
+scripts/init_db.py via Base.metadata.create_all — run init_db.py first.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from sqlalchemy import inspect, text
+
+from uraas.database import engine
+
+# (column, DDL type clause) — types chosen to work on both SQLite and Postgres.
+ITEMS_COLUMNS = [
+    ("alignment_scores", "TEXT"),
+    ("alignment_version", "INTEGER DEFAULT 0"),
+    ("coauthor_countries", "TEXT"),
+    ("african_country_count", "INTEGER DEFAULT 0"),
+    ("is_intra_african", "BOOLEAN DEFAULT FALSE"),
+    ("openalex_id", "VARCHAR(64)"),
+    ("counts_by_year", "TEXT"),
+    ("cited_by_count", "INTEGER DEFAULT 0"),
+    ("african_citation_share", "FLOAT"),
+    ("ark", "VARCHAR(128)"),
+    ("ark_assigned_at", "TIMESTAMP"),
+]
+
+INDEXES = [
+    "CREATE INDEX IF NOT EXISTS ix_items_is_intra_african ON items (is_intra_african)",
+    "CREATE UNIQUE INDEX IF NOT EXISTS ux_items_ark ON items (ark)",
+]
+
+
+def column_exists(insp, table: str, column: str) -> bool:
+    return column in {c["name"] for c in insp.get_columns(table)}
+
+
+def main() -> int:
+    print("Migration: 2026 upgrade (alignment / collaboration / citations / ARK)")
+
+    dialect = engine.dialect.name
+    print(f"Dialect: {dialect}")
+
+    insp = inspect(engine)
+    statements = []
+    for column, ddl_type in ITEMS_COLUMNS:
+        if column_exists(insp, "items", column):
+            print(f"  {column} already present, skipping")
+            continue
+        if dialect == "sqlite" and "BOOLEAN" in ddl_type:
+            # SQLite stores booleans as integers
+            ddl_type = ddl_type.replace("BOOLEAN", "INTEGER").replace("FALSE", "0")
+        statements.append(f"ALTER TABLE items ADD COLUMN {column} {ddl_type}")
+
+    if statements:
+        with engine.begin() as conn:
+            for stmt in statements:
+                print(f"  -> {stmt}")
+                conn.execute(text(stmt))
+    else:
+        print("  All columns already present.")
+
+    for stmt in INDEXES:
+        try:
+            with engine.begin() as conn:
+                conn.execute(text(stmt))
+                print(f"  -> {stmt.split(' ON ')[0].replace('CREATE ', '').strip()} ensured")
+        except Exception as e:
+            print(f"  (index creation skipped: {e})")
+
+    print("Done.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/migrate_add_ror.py

@@ -1,69 +1,69 @@
-"""
-Database migration: Add ROR support for multi-institution comparison
-"""
-
-from sqlalchemy import text
-
-from uraas.database import Item, SessionLocal, engine
-
-
-def migrate():
-    print("Adding ROR columns to items table...")
-
-    with engine.connect() as conn:
-        try:
-            # Add ror column
-            conn.execute(text("ALTER TABLE items ADD COLUMN ror VARCHAR(128)"))
-            conn.execute(text("CREATE INDEX ix_items_ror ON items(ror)"))
-            print("✓ Added ror column and index")
-        except Exception as e:
-            if (
-                "duplicate column" in str(e).lower()
-                or "already exists" in str(e).lower()
-            ):
-                print("✓ ROR column already exists")
-            else:
-                print(f"Error: {e}")
-
-        try:
-            # Add institution column if not exists
-            conn.execute(text("ALTER TABLE items ADD COLUMN institution VARCHAR(255)"))
-            print("✓ Added institution column")
-        except Exception as e:
-            if (
-                "duplicate column" in str(e).lower()
-                or "already exists" in str(e).lower()
-            ):
-                print("✓ Institution column already exists")
-            else:
-                print(f"Error: {e}")
-
-        conn.commit()
-
-    # Set default ROR for UNILAG papers
-    print("\nSetting default ROR for existing UNILAG papers...")
-    session = SessionLocal()
-    try:
-        unilag_ror = "https://ror.org/03qcnxw14"
-        count = (
-            session.query(Item)
-            .filter(Item.ror.is_(None))
-            .update(
-                {Item.ror: unilag_ror, Item.institution: "University of Lagos"},
-                synchronize_session=False,
-            )
-        )
-        session.commit()
-        print(f"✓ Updated {count} papers with UNILAG ROR")
-    finally:
-        session.close()
-
-    print("\nMigration complete!")
-    print("\nNext steps:")
-    print("1. Add Comparator tab to dashboard")
-    print("2. Test multi-institution comparison")
-    print("3. Add more institutions to database")
-
-
-if __name__ == "__main__":
-    migrate()
+"""
+Database migration: Add ROR support for multi-institution comparison
+"""
+
+from sqlalchemy import text
+
+from uraas.database import Item, SessionLocal, engine
+
+
+def migrate():
+    print("Adding ROR columns to items table...")
+
+    with engine.connect() as conn:
+        try:
+            # Add ror column
+            conn.execute(text("ALTER TABLE items ADD COLUMN ror VARCHAR(128)"))
+            conn.execute(text("CREATE INDEX ix_items_ror ON items(ror)"))
+            print("✓ Added ror column and index")
+        except Exception as e:
+            if (
+                "duplicate column" in str(e).lower()
+                or "already exists" in str(e).lower()
+            ):
+                print("✓ ROR column already exists")
+            else:
+                print(f"Error: {e}")
+
+        try:
+            # Add institution column if not exists
+            conn.execute(text("ALTER TABLE items ADD COLUMN institution VARCHAR(255)"))
+            print("✓ Added institution column")
+        except Exception as e:
+            if (
+                "duplicate column" in str(e).lower()
+                or "already exists" in str(e).lower()
+            ):
+                print("✓ Institution column already exists")
+            else:
+                print(f"Error: {e}")
+
+        conn.commit()
+
+    # Set default ROR for UNILAG papers
+    print("\nSetting default ROR for existing UNILAG papers...")
+    session = SessionLocal()
+    try:
+        unilag_ror = "https://ror.org/03qcnxw14"
+        count = (
+            session.query(Item)
+            .filter(Item.ror.is_(None))
+            .update(
+                {Item.ror: unilag_ror, Item.institution: "University of Lagos"},
+                synchronize_session=False,
+            )
+        )
+        session.commit()
+        print(f"✓ Updated {count} papers with UNILAG ROR")
+    finally:
+        session.close()
+
+    print("\nMigration complete!")
+    print("\nNext steps:")
+    print("1. Add Comparator tab to dashboard")
+    print("2. Test multi-institution comparison")
+    print("3. Add more institutions to database")
+
+
+if __name__ == "__main__":
+    migrate()

scripts/migrate_add_sc_columns.py

@@ -1,72 +1,72 @@
-"""
-Schema migration: add special_collection_score + special_collection_categories
-columns to items table. Idempotent.
-"""
-
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from sqlalchemy import inspect, text
-
-from uraas.database import engine
-
-
-def column_exists(table: str, column: str) -> bool:
-    insp = inspect(engine)
-    return column in {c["name"] for c in insp.get_columns(table)}
-
-
-def main() -> int:
-    print(
-        "Migration: adding special_collection_score + special_collection_categories to items"
-    )
-
-    dialect = engine.dialect.name
-    print(f"Dialect: {dialect}")
-
-    statements = []
-    if not column_exists("items", "special_collection_score"):
-        statements.append(
-            "ALTER TABLE items ADD COLUMN special_collection_score FLOAT DEFAULT 0.0"
-        )
-    else:
-        print("  special_collection_score already present, skipping")
-
-    if not column_exists("items", "special_collection_categories"):
-        # TEXT for both sqlite + postgres
-        statements.append(
-            "ALTER TABLE items ADD COLUMN special_collection_categories TEXT"
-        )
-    else:
-        print("  special_collection_categories already present, skipping")
-
-    if not statements:
-        print("Nothing to do.")
-        return 0
-
-    with engine.begin() as conn:
-        for stmt in statements:
-            print(f"  -> {stmt}")
-            conn.execute(text(stmt))
-
-    # Index on score so ORDER BY score DESC is fast
-    try:
-        with engine.begin() as conn:
-            conn.execute(
-                text(
-                    "CREATE INDEX IF NOT EXISTS ix_items_special_collection_score "
-                    "ON items (special_collection_score)"
-                )
-            )
-            print("  -> index ix_items_special_collection_score ensured")
-    except Exception as e:
-        print(f"  (index creation skipped: {e})")
-
-    print("Done.")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+"""
+Schema migration: add special_collection_score + special_collection_categories
+columns to items table. Idempotent.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from sqlalchemy import inspect, text
+
+from uraas.database import engine
+
+
+def column_exists(table: str, column: str) -> bool:
+    insp = inspect(engine)
+    return column in {c["name"] for c in insp.get_columns(table)}
+
+
+def main() -> int:
+    print(
+        "Migration: adding special_collection_score + special_collection_categories to items"
+    )
+
+    dialect = engine.dialect.name
+    print(f"Dialect: {dialect}")
+
+    statements = []
+    if not column_exists("items", "special_collection_score"):
+        statements.append(
+            "ALTER TABLE items ADD COLUMN special_collection_score FLOAT DEFAULT 0.0"
+        )
+    else:
+        print("  special_collection_score already present, skipping")
+
+    if not column_exists("items", "special_collection_categories"):
+        # TEXT for both sqlite + postgres
+        statements.append(
+            "ALTER TABLE items ADD COLUMN special_collection_categories TEXT"
+        )
+    else:
+        print("  special_collection_categories already present, skipping")
+
+    if not statements:
+        print("Nothing to do.")
+        return 0
+
+    with engine.begin() as conn:
+        for stmt in statements:
+            print(f"  -> {stmt}")
+            conn.execute(text(stmt))
+
+    # Index on score so ORDER BY score DESC is fast
+    try:
+        with engine.begin() as conn:
+            conn.execute(
+                text(
+                    "CREATE INDEX IF NOT EXISTS ix_items_special_collection_score "
+                    "ON items (special_collection_score)"
+                )
+            )
+            print("  -> index ix_items_special_collection_score ensured")
+    except Exception as e:
+        print(f"  (index creation skipped: {e})")
+
+    print("Done.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/migrate_sqlite_to_postgres.py

@@ -1,202 +1,202 @@
-"""
-Migration script: Copy all data from local SQLite database (uraas.db)
-to the production PostgreSQL database.
-"""
-
-import os
-import sys
-from sqlalchemy import create_engine, MetaData, text
-from sqlalchemy.orm import sessionmaker
-
-# Add project root to path
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from uraas.database import Base, Community, Collection, Author, Item, File, item_authors, item_collections
-
-def migrate():
-    # SQLite URL
-    sqlite_url = "sqlite:///uraas.db"
-    
-    # Postgres URL (get from environment variable)
-    postgres_url = os.getenv("DATABASE_URL")
-    if not postgres_url:
-        print("[ERR] DATABASE_URL environment variable is not set!")
-        print("Please run this command with DATABASE_URL set, for example:")
-        print("DATABASE_URL=postgresql://user:pass@host:port/dbname python scripts/migrate_sqlite_to_postgres.py")
-        sys.exit(1)
-        
-    # Standardize Render's postgres:// prefix to postgresql:// if needed
-    if postgres_url.startswith("postgres://"):
-        postgres_url = postgres_url.replace("postgres://", "postgresql://", 1)
-
-    print(f"Source SQLite database: {sqlite_url}")
-    print(f"Destination PostgreSQL database: {postgres_url.split('@')[-1] if '@' in postgres_url else postgres_url}")
-    print("\nInitializing connections...")
-    
-    sqlite_engine = create_engine(sqlite_url)
-    postgres_engine = create_engine(postgres_url)
-    
-    SqliteSession = sessionmaker(bind=sqlite_engine)
-    PostgresSession = sessionmaker(bind=postgres_engine)
-    
-    sqlite_session = SqliteSession()
-    postgres_session = PostgresSession()
-    
-    try:
-        print("Recreating destination database tables if they do not exist...")
-        Base.metadata.create_all(bind=postgres_engine)
-        
-        print("Clearing existing data in PostgreSQL tables to prevent collisions...")
-        # Order matters for foreign key constraints
-        postgres_session.execute(text("TRUNCATE TABLE files, item_authors, item_collections, items, authors, collections, communities CASCADE"))
-        postgres_session.commit()
-        
-        # 1. Migrate Communities
-        print("Migrating Communities...")
-        communities = sqlite_session.query(Community).all()
-        for comm in communities:
-            new_comm = Community(
-                id=comm.id,
-                name=comm.name,
-                ror_id=comm.ror_id,
-                institution=comm.institution,
-                ror=comm.ror
-            )
-            postgres_session.add(new_comm)
-        postgres_session.flush()
-        print(f"  Migrated {len(communities)} communities.")
-        
-        # 2. Migrate Collections
-        print("Migrating Collections...")
-        collections = sqlite_session.query(Collection).all()
-        for coll in collections:
-            new_coll = Collection(
-                id=coll.id,
-                community_id=coll.community_id,
-                name=coll.name,
-                email_domains=coll.email_domains,
-                keywords=coll.keywords
-            )
-            postgres_session.add(new_coll)
-        postgres_session.flush()
-        print(f"  Migrated {len(collections)} collections.")
-        
-        # 3. Migrate Authors
-        print("Migrating Authors...")
-        authors = sqlite_session.query(Author).all()
-        for auth in authors:
-            new_auth = Author(
-                id=auth.id,
-                name=auth.name,
-                normalized_name=auth.normalized_name,
-                profile_url=auth.profile_url,
-                orcid=auth.orcid,
-                ror=auth.ror
-            )
-            postgres_session.add(new_auth)
-        postgres_session.flush()
-        print(f"  Migrated {len(authors)} authors.")
-        
-        # 4. Migrate Items
-        print("Migrating Items...")
-        items = sqlite_session.query(Item).all()
-        for item in items:
-            new_item = Item(
-                id=item.id,
-                title=item.title,
-                abstract=item.abstract,
-                doi=item.doi,
-                publication_date=item.publication_date,
-                url=item.url,
-                source_repository=item.source_repository,
-                pdf_url=item.pdf_url,
-                dc_title=item.dc_title,
-                dc_date_issued=item.dc_date_issued,
-                dc_identifier_uri=item.dc_identifier_uri,
-                dc_identifier_doi=item.dc_identifier_doi,
-                dc_description_provenance=item.dc_description_provenance,
-                dc_rights=item.dc_rights,
-                dc_type=item.dc_type,
-                dc_language=item.dc_language,
-                dc_subject=item.dc_subject,
-                docid=item.docid,
-                docid_assigned_at=item.docid_assigned_at,
-                ror=item.ror,
-                institution=item.institution,
-                content_type=item.content_type,
-                tk_label=item.tk_label,
-                tk_community=item.tk_community,
-                patent_id=item.patent_id,
-                patent_date=item.patent_date,
-                language_code=item.language_code,
-                is_african_language=item.is_african_language,
-                sdg_tags=item.sdg_tags,
-                ai_keywords=item.ai_keywords,
-                special_collection_score=item.special_collection_score,
-                special_collection_categories=item.special_collection_categories,
-                created_at=item.created_at
-            )
-            postgres_session.add(new_item)
-        postgres_session.flush()
-        print(f"  Migrated {len(items)} items.")
-        
-        # 5. Migrate Files
-        print("Migrating Files...")
-        files = sqlite_session.query(File).all()
-        for file in files:
-            new_file = File(
-                id=file.id,
-                item_id=file.item_id,
-                file_path=file.file_path,
-                sha256_hash=file.sha256_hash,
-                access_policy=file.access_policy,
-                downloaded_at=file.downloaded_at
-            )
-            postgres_session.add(new_file)
-        postgres_session.flush()
-        print(f"  Migrated {len(files)} files.")
-        
-        # 6. Migrate association tables (item_authors and item_collections)
-        print("Migrating Item-Author associations...")
-        item_author_rows = sqlite_session.execute(item_authors.select()).all()
-        for row in item_author_rows:
-            postgres_session.execute(
-                item_authors.insert().values(item_id=row.item_id, author_id=row.author_id)
-            )
-        print(f"  Migrated {len(item_author_rows)} item-author mappings.")
-            
-        print("Migrating Item-Collection associations...")
-        item_coll_rows = sqlite_session.execute(item_collections.select()).all()
-        for row in item_coll_rows:
-            postgres_session.execute(
-                item_collections.insert().values(
-                    item_id=row.item_id, 
-                    collection_id=row.collection_id,
-                    confidence_score=row.confidence_score
-                )
-            )
-        print(f"  Migrated {len(item_coll_rows)} item-collection mappings.")
-        
-        postgres_session.commit()
-        print("[SUCCESS] Data migrated to PostgreSQL successfully!")
-        
-        # Reset sequences in Postgres so future inserts don't collide
-        print("Resetting PostgreSQL primary key sequences...")
-        tables = ["communities", "collections", "authors", "items", "files"]
-        for table in tables:
-            postgres_session.execute(text(
-                f"SELECT setval(pg_get_serial_sequence('{table}', 'id'), COALESCE(MAX(id), 1) + 1) FROM {table}"
-            ))
-        postgres_session.commit()
-        print("[SUCCESS] Sequences advanced.")
-        
-    except Exception as e:
-        print(f"[ERR] Migration failed: {e}")
-        postgres_session.rollback()
-        raise
-    finally:
-        sqlite_session.close()
-        postgres_session.close()
-
-if __name__ == "__main__":
-    migrate()
+"""
+Migration script: Copy all data from local SQLite database (uraas.db)
+to the production PostgreSQL database.
+"""
+
+import os
+import sys
+from sqlalchemy import create_engine, MetaData, text
+from sqlalchemy.orm import sessionmaker
+
+# Add project root to path
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from uraas.database import Base, Community, Collection, Author, Item, File, item_authors, item_collections
+
+def migrate():
+    # SQLite URL
+    sqlite_url = "sqlite:///uraas.db"
+    
+    # Postgres URL (get from environment variable)
+    postgres_url = os.getenv("DATABASE_URL")
+    if not postgres_url:
+        print("[ERR] DATABASE_URL environment variable is not set!")
+        print("Please run this command with DATABASE_URL set, for example:")
+        print("DATABASE_URL=postgresql://user:pass@host:port/dbname python scripts/migrate_sqlite_to_postgres.py")
+        sys.exit(1)
+        
+    # Standardize Render's postgres:// prefix to postgresql:// if needed
+    if postgres_url.startswith("postgres://"):
+        postgres_url = postgres_url.replace("postgres://", "postgresql://", 1)
+
+    print(f"Source SQLite database: {sqlite_url}")
+    print(f"Destination PostgreSQL database: {postgres_url.split('@')[-1] if '@' in postgres_url else postgres_url}")
+    print("\nInitializing connections...")
+    
+    sqlite_engine = create_engine(sqlite_url)
+    postgres_engine = create_engine(postgres_url)
+    
+    SqliteSession = sessionmaker(bind=sqlite_engine)
+    PostgresSession = sessionmaker(bind=postgres_engine)
+    
+    sqlite_session = SqliteSession()
+    postgres_session = PostgresSession()
+    
+    try:
+        print("Recreating destination database tables if they do not exist...")
+        Base.metadata.create_all(bind=postgres_engine)
+        
+        print("Clearing existing data in PostgreSQL tables to prevent collisions...")
+        # Order matters for foreign key constraints
+        postgres_session.execute(text("TRUNCATE TABLE files, item_authors, item_collections, items, authors, collections, communities CASCADE"))
+        postgres_session.commit()
+        
+        # 1. Migrate Communities
+        print("Migrating Communities...")
+        communities = sqlite_session.query(Community).all()
+        for comm in communities:
+            new_comm = Community(
+                id=comm.id,
+                name=comm.name,
+                ror_id=comm.ror_id,
+                institution=comm.institution,
+                ror=comm.ror
+            )
+            postgres_session.add(new_comm)
+        postgres_session.flush()
+        print(f"  Migrated {len(communities)} communities.")
+        
+        # 2. Migrate Collections
+        print("Migrating Collections...")
+        collections = sqlite_session.query(Collection).all()
+        for coll in collections:
+            new_coll = Collection(
+                id=coll.id,
+                community_id=coll.community_id,
+                name=coll.name,
+                email_domains=coll.email_domains,
+                keywords=coll.keywords
+            )
+            postgres_session.add(new_coll)
+        postgres_session.flush()
+        print(f"  Migrated {len(collections)} collections.")
+        
+        # 3. Migrate Authors
+        print("Migrating Authors...")
+        authors = sqlite_session.query(Author).all()
+        for auth in authors:
+            new_auth = Author(
+                id=auth.id,
+                name=auth.name,
+                normalized_name=auth.normalized_name,
+                profile_url=auth.profile_url,
+                orcid=auth.orcid,
+                ror=auth.ror
+            )
+            postgres_session.add(new_auth)
+        postgres_session.flush()
+        print(f"  Migrated {len(authors)} authors.")
+        
+        # 4. Migrate Items
+        print("Migrating Items...")
+        items = sqlite_session.query(Item).all()
+        for item in items:
+            new_item = Item(
+                id=item.id,
+                title=item.title,
+                abstract=item.abstract,
+                doi=item.doi,
+                publication_date=item.publication_date,
+                url=item.url,
+                source_repository=item.source_repository,
+                pdf_url=item.pdf_url,
+                dc_title=item.dc_title,
+                dc_date_issued=item.dc_date_issued,
+                dc_identifier_uri=item.dc_identifier_uri,
+                dc_identifier_doi=item.dc_identifier_doi,
+                dc_description_provenance=item.dc_description_provenance,
+                dc_rights=item.dc_rights,
+                dc_type=item.dc_type,
+                dc_language=item.dc_language,
+                dc_subject=item.dc_subject,
+                docid=item.docid,
+                docid_assigned_at=item.docid_assigned_at,
+                ror=item.ror,
+                institution=item.institution,
+                content_type=item.content_type,
+                tk_label=item.tk_label,
+                tk_community=item.tk_community,
+                patent_id=item.patent_id,
+                patent_date=item.patent_date,
+                language_code=item.language_code,
+                is_african_language=item.is_african_language,
+                sdg_tags=item.sdg_tags,
+                ai_keywords=item.ai_keywords,
+                special_collection_score=item.special_collection_score,
+                special_collection_categories=item.special_collection_categories,
+                created_at=item.created_at
+            )
+            postgres_session.add(new_item)
+        postgres_session.flush()
+        print(f"  Migrated {len(items)} items.")
+        
+        # 5. Migrate Files
+        print("Migrating Files...")
+        files = sqlite_session.query(File).all()
+        for file in files:
+            new_file = File(
+                id=file.id,
+                item_id=file.item_id,
+                file_path=file.file_path,
+                sha256_hash=file.sha256_hash,
+                access_policy=file.access_policy,
+                downloaded_at=file.downloaded_at
+            )
+            postgres_session.add(new_file)
+        postgres_session.flush()
+        print(f"  Migrated {len(files)} files.")
+        
+        # 6. Migrate association tables (item_authors and item_collections)
+        print("Migrating Item-Author associations...")
+        item_author_rows = sqlite_session.execute(item_authors.select()).all()
+        for row in item_author_rows:
+            postgres_session.execute(
+                item_authors.insert().values(item_id=row.item_id, author_id=row.author_id)
+            )
+        print(f"  Migrated {len(item_author_rows)} item-author mappings.")
+            
+        print("Migrating Item-Collection associations...")
+        item_coll_rows = sqlite_session.execute(item_collections.select()).all()
+        for row in item_coll_rows:
+            postgres_session.execute(
+                item_collections.insert().values(
+                    item_id=row.item_id, 
+                    collection_id=row.collection_id,
+                    confidence_score=row.confidence_score
+                )
+            )
+        print(f"  Migrated {len(item_coll_rows)} item-collection mappings.")
+        
+        postgres_session.commit()
+        print("[SUCCESS] Data migrated to PostgreSQL successfully!")
+        
+        # Reset sequences in Postgres so future inserts don't collide
+        print("Resetting PostgreSQL primary key sequences...")
+        tables = ["communities", "collections", "authors", "items", "files"]
+        for table in tables:
+            postgres_session.execute(text(
+                f"SELECT setval(pg_get_serial_sequence('{table}', 'id'), COALESCE(MAX(id), 1) + 1) FROM {table}"
+            ))
+        postgres_session.commit()
+        print("[SUCCESS] Sequences advanced.")
+        
+    except Exception as e:
+        print(f"[ERR] Migration failed: {e}")
+        postgres_session.rollback()
+        raise
+    finally:
+        sqlite_session.close()
+        postgres_session.close()
+
+if __name__ == "__main__":
+    migrate()

scripts/migrate_unilag_ror.py

@@ -1,55 +1,55 @@
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from uraas.database import Author, Community, Item, SessionLocal
-
-
-def migrate():
-    session = SessionLocal()
-    try:
-        old_ror = "https://ror.org/03qcnxw14"
-        new_ror = "https://ror.org/05rk03822"
-
-        print("Migrating UNILAG RORs in database...")
-
-        # 1. Update items
-        item_count = (
-            session.query(Item)
-            .filter(Item.ror == old_ror)
-            .update({Item.ror: new_ror}, synchronize_session=False)
-        )
-        print(f"[OK] Updated {item_count} items")
-
-        # 2. Update communities
-        comm_count = (
-            session.query(Community)
-            .filter((Community.ror == old_ror) | (Community.ror_id == old_ror))
-            .update(
-                {Community.ror: new_ror, Community.ror_id: new_ror},
-                synchronize_session=False,
-            )
-        )
-        print(f"[OK] Updated {comm_count} communities")
-
-        # 3. Update authors
-        author_count = (
-            session.query(Author)
-            .filter(Author.ror == old_ror)
-            .update({Author.ror: new_ror}, synchronize_session=False)
-        )
-        print(f"[OK] Updated {author_count} authors")
-
-        session.commit()
-        print("Migration complete successfully!")
-
-    except Exception as e:
-        session.rollback()
-        print(f"Error during migration: {e}")
-    finally:
-        session.close()
-
-
-if __name__ == "__main__":
-    migrate()
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from uraas.database import Author, Community, Item, SessionLocal
+
+
+def migrate():
+    session = SessionLocal()
+    try:
+        old_ror = "https://ror.org/03qcnxw14"
+        new_ror = "https://ror.org/05rk03822"
+
+        print("Migrating UNILAG RORs in database...")
+
+        # 1. Update items
+        item_count = (
+            session.query(Item)
+            .filter(Item.ror == old_ror)
+            .update({Item.ror: new_ror}, synchronize_session=False)
+        )
+        print(f"[OK] Updated {item_count} items")
+
+        # 2. Update communities
+        comm_count = (
+            session.query(Community)
+            .filter((Community.ror == old_ror) | (Community.ror_id == old_ror))
+            .update(
+                {Community.ror: new_ror, Community.ror_id: new_ror},
+                synchronize_session=False,
+            )
+        )
+        print(f"[OK] Updated {comm_count} communities")
+
+        # 3. Update authors
+        author_count = (
+            session.query(Author)
+            .filter(Author.ror == old_ror)
+            .update({Author.ror: new_ror}, synchronize_session=False)
+        )
+        print(f"[OK] Updated {author_count} authors")
+
+        session.commit()
+        print("Migration complete successfully!")
+
+    except Exception as e:
+        session.rollback()
+        print(f"Error during migration: {e}")
+    finally:
+        session.close()
+
+
+if __name__ == "__main__":
+    migrate()

scripts/patch_html.py

@@ -1,74 +1,74 @@
-import re
-
-with open("uraas/dashboard/templates/index.html", "r", encoding="utf-8") as f:
-    content = f.read()
-
-# Find the special collections section and replace it
-pattern = r"(    \s*<!--\s+SPECIAL COLLECTIONS.*?</div>\n\n  </div><!-- end analytics tab -->)"
-replacement = """    
-    <!--  SPECIAL COLLECTIONS  -->
-    <div id="atab-special" class="atab-content hidden">
-      <div class="mb-5 flex items-start justify-between gap-4">
-        <div>
-          <h2 class="text-xl font-bold mb-1" style="color:var(--text)">Special Collections: African Literature &amp; Indigenous Knowledge</h2>
-          <p class="text-sm" style="color:var(--text-muted)">AI-powered classification: Postcolonial Studies, Pan-African Studies, Ethnomusicology &amp; more.</p>
-        </div>
-        <a href="/api/analytics/special-collections/export.csv" class="btn-ghost text-xs flex-shrink-0">
-          <svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-4l-4 4m0 0l-4-4m4 4V4"/></svg>
-          Download CSV
-        </a>
-      </div>
-      <div id="special-loading" class="flex items-center gap-3 py-10 justify-center">
-        <div class="w-6 h-6 border-2 rounded-full animate-spin" style="border-color:var(--accent);border-top-color:transparent"></div>
-        <p class="text-sm" style="color:var(--text-muted)">Analyzing special collections...</p>
-      </div>
-      <div id="special-content" class="hidden">
-        <div class="grid grid-cols-1 lg:grid-cols-3 gap-5 mb-6" id="special-stats"></div>
-        <div class="grid grid-cols-1 lg:grid-cols-2 gap-5" id="special-categories"></div>
-      </div>
-    </div>
-
-    <!--  STAFF DIRECTORY  -->
-    <div id="atab-staff" class="atab-content hidden">
-      <div class="mb-5 flex items-start justify-between gap-4">
-        <div>
-          <h2 class="text-xl font-bold mb-1" style="color:var(--text)">Staff Directory</h2>
-          <p class="text-sm" style="color:var(--text-muted)">Real staff members with departments and ORCID identifiers, harvested from OpenAlex and ORCID APIs.</p>
-        </div>
-        <select id="staff-inst-filter" onchange="loadStaffDirectory()" style="width:auto;font-size:12px;padding:5px 10px">
-          <option value="">All Institutions</option>
-          <option value="unilag">UNILAG</option><option value="covenant">Covenant</option>
-          <option value="ui">Univ. Ibadan</option><option value="uct">UCT</option>
-          <option value="stellenbosch">Stellenbosch</option><option value="nairobi">Nairobi</option>
-          <option value="makerere">Makerere</option><option value="ghana">Univ. Ghana</option>
-          <option value="addisababa">Addis Ababa</option><option value="knust">KNUST</option>
-        </select>
-      </div>
-      <div id="staff-loading" class="flex items-center gap-3 py-10 justify-center">
-        <div class="w-6 h-6 border-2 rounded-full animate-spin" style="border-color:var(--accent);border-top-color:transparent"></div>
-        <p class="text-sm" style="color:var(--text-muted)">Loading staff directory...</p>
-      </div>
-      <div id="staff-content" class="hidden">
-        <div id="staff-institutions-list" class="space-y-6"></div>
-      </div>
-    </div>
-
-  </div><!-- end analytics tab -->"""
-
-new_content = re.sub(pattern, replacement, content, flags=re.DOTALL)
-if new_content == content:
-    print("ERROR: Pattern not matched. Trying direct string replace...")
-    # Try to find where special collections starts
-    idx = content.find("<!--  SPECIAL COLLECTIONS  -->")
-    print(f"Found at index: {idx}")
-    if idx >= 0:
-        end_marker = "  </div><!-- end analytics tab -->"
-        end_idx = content.find(end_marker, idx)
-        print(f"End marker at: {end_idx}")
-        segment = content[idx : end_idx + len(end_marker)]
-        print(f"Segment length: {len(segment)}")
-        print("First 200 chars:", repr(segment[:200]))
-else:
-    with open("uraas/dashboard/templates/index.html", "w", encoding="utf-8") as f:
-        f.write(new_content)
-    print("SUCCESS")
+import re
+
+with open("uraas/dashboard/templates/index.html", "r", encoding="utf-8") as f:
+    content = f.read()
+
+# Find the special collections section and replace it
+pattern = r"(    \s*<!--\s+SPECIAL COLLECTIONS.*?</div>\n\n  </div><!-- end analytics tab -->)"
+replacement = """    
+    <!--  SPECIAL COLLECTIONS  -->
+    <div id="atab-special" class="atab-content hidden">
+      <div class="mb-5 flex items-start justify-between gap-4">
+        <div>
+          <h2 class="text-xl font-bold mb-1" style="color:var(--text)">Special Collections: African Literature &amp; Indigenous Knowledge</h2>
+          <p class="text-sm" style="color:var(--text-muted)">AI-powered classification: Postcolonial Studies, Pan-African Studies, Ethnomusicology &amp; more.</p>
+        </div>
+        <a href="/api/analytics/special-collections/export.csv" class="btn-ghost text-xs flex-shrink-0">
+          <svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-4l-4 4m0 0l-4-4m4 4V4"/></svg>
+          Download CSV
+        </a>
+      </div>
+      <div id="special-loading" class="flex items-center gap-3 py-10 justify-center">
+        <div class="w-6 h-6 border-2 rounded-full animate-spin" style="border-color:var(--accent);border-top-color:transparent"></div>
+        <p class="text-sm" style="color:var(--text-muted)">Analyzing special collections...</p>
+      </div>
+      <div id="special-content" class="hidden">
+        <div class="grid grid-cols-1 lg:grid-cols-3 gap-5 mb-6" id="special-stats"></div>
+        <div class="grid grid-cols-1 lg:grid-cols-2 gap-5" id="special-categories"></div>
+      </div>
+    </div>
+
+    <!--  STAFF DIRECTORY  -->
+    <div id="atab-staff" class="atab-content hidden">
+      <div class="mb-5 flex items-start justify-between gap-4">
+        <div>
+          <h2 class="text-xl font-bold mb-1" style="color:var(--text)">Staff Directory</h2>
+          <p class="text-sm" style="color:var(--text-muted)">Real staff members with departments and ORCID identifiers, harvested from OpenAlex and ORCID APIs.</p>
+        </div>
+        <select id="staff-inst-filter" onchange="loadStaffDirectory()" style="width:auto;font-size:12px;padding:5px 10px">
+          <option value="">All Institutions</option>
+          <option value="unilag">UNILAG</option><option value="covenant">Covenant</option>
+          <option value="ui">Univ. Ibadan</option><option value="uct">UCT</option>
+          <option value="stellenbosch">Stellenbosch</option><option value="nairobi">Nairobi</option>
+          <option value="makerere">Makerere</option><option value="ghana">Univ. Ghana</option>
+          <option value="addisababa">Addis Ababa</option><option value="knust">KNUST</option>
+        </select>
+      </div>
+      <div id="staff-loading" class="flex items-center gap-3 py-10 justify-center">
+        <div class="w-6 h-6 border-2 rounded-full animate-spin" style="border-color:var(--accent);border-top-color:transparent"></div>
+        <p class="text-sm" style="color:var(--text-muted)">Loading staff directory...</p>
+      </div>
+      <div id="staff-content" class="hidden">
+        <div id="staff-institutions-list" class="space-y-6"></div>
+      </div>
+    </div>
+
+  </div><!-- end analytics tab -->"""
+
+new_content = re.sub(pattern, replacement, content, flags=re.DOTALL)
+if new_content == content:
+    print("ERROR: Pattern not matched. Trying direct string replace...")
+    # Try to find where special collections starts
+    idx = content.find("<!--  SPECIAL COLLECTIONS  -->")
+    print(f"Found at index: {idx}")
+    if idx >= 0:
+        end_marker = "  </div><!-- end analytics tab -->"
+        end_idx = content.find(end_marker, idx)
+        print(f"End marker at: {end_idx}")
+        segment = content[idx : end_idx + len(end_marker)]
+        print(f"Segment length: {len(segment)}")
+        print("First 200 chars:", repr(segment[:200]))
+else:
+    with open("uraas/dashboard/templates/index.html", "w", encoding="utf-8") as f:
+        f.write(new_content)
+    print("SUCCESS")

scripts/push_to_hf.py

@@ -1,195 +1,195 @@
-"""
-Push URAAS to Hugging Face Spaces — Lordkiki/APA-URAAS
-
-Usage:
-    python scripts/push_to_hf.py
-
-What it does:
-  1. Logs you into HF (paste your write token when prompted)
-  2. Stages a clean copy of the project:
-       Dockerfile.hf  → Dockerfile   (HF Spaces Dockerfile, not the prod one)
-       README.hf.md   → README.md    (has the HF Space frontmatter)
-  3. Uploads everything to the Space via huggingface_hub.upload_folder
-  4. HF triggers an auto-build — app is live in ~5 minutes
-"""
-
-import os
-import shutil
-import sys
-import tempfile
-
-# ── Config ────────────────────────────────────────────────────────────────────
-REPO_ID   = "Lordkiki/APA-URAAS"
-REPO_TYPE = "space"
-REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-# Directories/files to never push
-IGNORE_DIRS = {
-    ".git", ".claude", "__pycache__", ".pytest_cache", ".mypy_cache",
-    "storage", "logs", "data", "backups", "node_modules",
-    ".venv", "venv", "env",
-}
-IGNORE_FILES = {
-    ".env", ".env.prod", ".env.prod.example",
-    "uraas.db",                  # DB lives on /data in HF, not in image
-    "Dockerfile",                # replaced by Dockerfile.hf
-    "README.md",                 # replaced by README.hf.md (has HF frontmatter)
-    "docker-compose.yml",
-    "docker-compose.prod.yml",
-    "docker-compose.replica.yml",
-    "docker-compose.demo.yml",
-}
-IGNORE_EXTS = {".pyc", ".pyo", ".pyd"}
-
-
-def should_skip(rel_path: str, is_dir: bool) -> bool:
-    parts = rel_path.replace("\\", "/").split("/")
-    name  = parts[-1]
-    if is_dir:
-        return name in IGNORE_DIRS
-    return (
-        name in IGNORE_FILES
-        or os.path.splitext(name)[1] in IGNORE_EXTS
-    )
-
-
-def stage_project(src: str, dst: str) -> int:
-    """Copy src → dst with HF-specific renames and exclusions."""
-    count = 0
-    for root, dirs, files in os.walk(src):
-        rel_root = os.path.relpath(root, src)
-
-        # Prune excluded directories in-place so os.walk doesn't recurse
-        dirs[:] = [
-            d for d in dirs
-            if not should_skip(
-                os.path.join(rel_root, d) if rel_root != "." else d,
-                is_dir=True,
-            )
-        ]
-
-        for fname in files:
-            rel = os.path.join(rel_root, fname) if rel_root != "." else fname
-            if should_skip(rel, is_dir=False):
-                continue
-
-            src_file = os.path.join(root, fname)
-
-            # HF-specific renames
-            if fname == "Dockerfile.hf":
-                dest_rel = os.path.join(os.path.dirname(rel), "Dockerfile") if os.path.dirname(rel) else "Dockerfile"
-            elif fname == "README.hf.md":
-                dest_rel = "README.md"
-            else:
-                dest_rel = rel
-
-            dst_file = os.path.join(dst, dest_rel)
-            os.makedirs(os.path.dirname(dst_file), exist_ok=True)
-            shutil.copy2(src_file, dst_file)
-            count += 1
-    return count
-
-
-def main():
-    # ── Ensure huggingface_hub is available ────────────────────────────────
-    try:
-        from huggingface_hub import HfApi, login
-    except ImportError:
-        print("Installing huggingface_hub…")
-        os.system(f"{sys.executable} -m pip install huggingface_hub -q")
-        from huggingface_hub import HfApi, login  # type: ignore
-
-    print()
-    print("═" * 55)
-    print("  URAAS → Hugging Face Spaces")
-    print(f"  Space: {REPO_ID}")
-    print("═" * 55)
-
-    # ── Auth ───────────────────────────────────────────────────────────────
-    token = os.getenv("HF_TOKEN")
-    if token:
-        login(token=token, add_to_git_credential=True)
-        print("  Logged in via HF_TOKEN env var.")
-    else:
-        print()
-        print("  Paste your HF write token below.")
-        print("  (Get one at: https://huggingface.co/settings/tokens)")
-        print()
-        login(add_to_git_credential=True)
-
-    api = HfApi()
-
-    # ── Stage files ────────────────────────────────────────────────────────
-    print()
-    print("Staging project files…")
-    with tempfile.TemporaryDirectory() as staging:
-        n = stage_project(REPO_ROOT, staging)
-        staged_names = os.listdir(staging)
-        print(f"  {n} files staged across {len(staged_names)} top-level items")
-
-        # Sanity checks
-        has_dockerfile = "Dockerfile" in staged_names
-        has_readme     = "README.md" in staged_names
-        has_start_sh   = os.path.exists(os.path.join(staging, "scripts", "start_hf.sh"))
-
-        print(f"  Dockerfile   : {'✓' if has_dockerfile else '✗ MISSING — check Dockerfile.hf exists'}")
-        print(f"  README.md    : {'✓' if has_readme else '✗ MISSING — check README.hf.md exists'}")
-        print(f"  start_hf.sh  : {'✓' if has_start_sh else '✗ MISSING — check scripts/start_hf.sh'}")
-
-        if not has_dockerfile:
-            print()
-            print("ERROR: Dockerfile missing from staging. Aborting.")
-            sys.exit(1)
-
-        # ── Upload ─────────────────────────────────────────────────────────
-        print()
-        print(f"Uploading to {REPO_ID}…")
-        api.upload_folder(
-            folder_path=staging,
-            repo_id=REPO_ID,
-            repo_type=REPO_TYPE,
-            commit_message="Deploy URAAS — African Research Archival & Analytics System",
-        )
-
-    # ── Done ───────────────────────────────────────────────────────────────
-    print()
-    print("═" * 55)
-    print("  Upload complete! Build starting on HF (~5 min).")
-    print()
-    print("  Watch build: https://huggingface.co/spaces/Lordkiki/APA-URAAS")
-    print("  App URL    : https://lordkiki-apa-uraas.hf.space")
-    print()
-    print("  ─── Secrets to set in Space Settings → Variables & Secrets ───")
-    secrets = [
-        ("URAAS_ENV",              "production"),
-        ("DASHBOARD_SECRET_KEY",   "307790fc5aff3fe1e766303f6b94e2fc28c831582bfba5b34802e2c9cbbac0ce"),
-        ("ADMIN_USERNAME",         "admin"),
-        ("ADMIN_PASSWORD_HASH",    "scrypt:32768:8:1$r2KZFX32rJ2twbfV$16f394a253c2b505a215ff2747f7dafbb098eb0eb8b4e6bb9fb521f0ea38af8ce71f33d2354245d68cc383678257367bf410aa45f835b5e848bff95a746878c8"),
-        ("VIEWER_USERNAME",        "viewer"),
-        ("VIEWER_PASSWORD_HASH",   "scrypt:32768:8:1$M8OjWxX64B38akos$2803ad5b29508c4d69df115579630b2a4cbdf2c8406157598c088d5511a7b3d79f91399035040660705413c63fdc41aa0d020cbd7b45038c8ed51d20092ea609"),
-        ("SMTP_HOST",              "smtp.gmail.com"),
-        ("SMTP_PORT",              "587"),
-        ("SMTP_USE_TLS",           "true"),
-        ("SMTP_USER",              "lawalgiyath200716@gmail.com"),
-        ("SMTP_PASSWORD",          "ufwqbdrecpfrzppn"),
-        ("SMTP_FROM",              "URAAS UNILAG <lawalgiyath200716@gmail.com>"),
-        ("DASHBOARD_BASE_URL",     "https://lordkiki-apa-uraas.hf.space"),
-        ("DASHBOARD_CORS_ORIGINS", "https://lordkiki-apa-uraas.hf.space"),
-        ("ARK_NAAN",               "99999"),
-        ("ARK_SHOULDER",           "z1"),
-        ("OPENALEX_MAILTO",        "lawalgiyath200716@gmail.com"),
-        ("DSPACE_API_URL",         "https://api-ir.unilag.edu.ng/server"),
-        ("DSPACE_USERNAME",        "<professor email — set as Secret, not Variable>"),
-        ("DSPACE_PASSWORD",        "<professor password — set as Secret, not Variable>"),
-    ]
-    max_k = max(len(k) for k, _ in secrets)
-    for k, v in secrets:
-        print(f"  {k:<{max_k}} = {v}")
-    print()
-    print("  Admin login : admin / URAAS2024demo")
-    print("  Viewer login: viewer / view2024")
-    print("═" * 55)
-
-
-if __name__ == "__main__":
-    main()
+"""
+Push URAAS to Hugging Face Spaces — Lordkiki/APA-URAAS
+
+Usage:
+    python scripts/push_to_hf.py
+
+What it does:
+  1. Logs you into HF (paste your write token when prompted)
+  2. Stages a clean copy of the project:
+       Dockerfile.hf  → Dockerfile   (HF Spaces Dockerfile, not the prod one)
+       README.hf.md   → README.md    (has the HF Space frontmatter)
+  3. Uploads everything to the Space via huggingface_hub.upload_folder
+  4. HF triggers an auto-build — app is live in ~5 minutes
+"""
+
+import os
+import shutil
+import sys
+import tempfile
+
+# ── Config ────────────────────────────────────────────────────────────────────
+REPO_ID   = "Lordkiki/APA-URAAS"
+REPO_TYPE = "space"
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# Directories/files to never push
+IGNORE_DIRS = {
+    ".git", ".claude", "__pycache__", ".pytest_cache", ".mypy_cache",
+    "storage", "logs", "data", "backups", "node_modules",
+    ".venv", "venv", "env",
+}
+IGNORE_FILES = {
+    ".env", ".env.prod", ".env.prod.example",
+    "uraas.db",                  # DB lives on /data in HF, not in image
+    "Dockerfile",                # replaced by Dockerfile.hf
+    "README.md",                 # replaced by README.hf.md (has HF frontmatter)
+    "docker-compose.yml",
+    "docker-compose.prod.yml",
+    "docker-compose.replica.yml",
+    "docker-compose.demo.yml",
+}
+IGNORE_EXTS = {".pyc", ".pyo", ".pyd"}
+
+
+def should_skip(rel_path: str, is_dir: bool) -> bool:
+    parts = rel_path.replace("\\", "/").split("/")
+    name  = parts[-1]
+    if is_dir:
+        return name in IGNORE_DIRS
+    return (
+        name in IGNORE_FILES
+        or os.path.splitext(name)[1] in IGNORE_EXTS
+    )
+
+
+def stage_project(src: str, dst: str) -> int:
+    """Copy src → dst with HF-specific renames and exclusions."""
+    count = 0
+    for root, dirs, files in os.walk(src):
+        rel_root = os.path.relpath(root, src)
+
+        # Prune excluded directories in-place so os.walk doesn't recurse
+        dirs[:] = [
+            d for d in dirs
+            if not should_skip(
+                os.path.join(rel_root, d) if rel_root != "." else d,
+                is_dir=True,
+            )
+        ]
+
+        for fname in files:
+            rel = os.path.join(rel_root, fname) if rel_root != "." else fname
+            if should_skip(rel, is_dir=False):
+                continue
+
+            src_file = os.path.join(root, fname)
+
+            # HF-specific renames
+            if fname == "Dockerfile.hf":
+                dest_rel = os.path.join(os.path.dirname(rel), "Dockerfile") if os.path.dirname(rel) else "Dockerfile"
+            elif fname == "README.hf.md":
+                dest_rel = "README.md"
+            else:
+                dest_rel = rel
+
+            dst_file = os.path.join(dst, dest_rel)
+            os.makedirs(os.path.dirname(dst_file), exist_ok=True)
+            shutil.copy2(src_file, dst_file)
+            count += 1
+    return count
+
+
+def main():
+    # ── Ensure huggingface_hub is available ────────────────────────────────
+    try:
+        from huggingface_hub import HfApi, login
+    except ImportError:
+        print("Installing huggingface_hub…")
+        os.system(f"{sys.executable} -m pip install huggingface_hub -q")
+        from huggingface_hub import HfApi, login  # type: ignore
+
+    print()
+    print("═" * 55)
+    print("  URAAS → Hugging Face Spaces")
+    print(f"  Space: {REPO_ID}")
+    print("═" * 55)
+
+    # ── Auth ───────────────────────────────────────────────────────────────
+    token = os.getenv("HF_TOKEN")
+    if token:
+        login(token=token, add_to_git_credential=True)
+        print("  Logged in via HF_TOKEN env var.")
+    else:
+        print()
+        print("  Paste your HF write token below.")
+        print("  (Get one at: https://huggingface.co/settings/tokens)")
+        print()
+        login(add_to_git_credential=True)
+
+    api = HfApi()
+
+    # ── Stage files ────────────────────────────────────────────────────────
+    print()
+    print("Staging project files…")
+    with tempfile.TemporaryDirectory() as staging:
+        n = stage_project(REPO_ROOT, staging)
+        staged_names = os.listdir(staging)
+        print(f"  {n} files staged across {len(staged_names)} top-level items")
+
+        # Sanity checks
+        has_dockerfile = "Dockerfile" in staged_names
+        has_readme     = "README.md" in staged_names
+        has_start_sh   = os.path.exists(os.path.join(staging, "scripts", "start_hf.sh"))
+
+        print(f"  Dockerfile   : {'✓' if has_dockerfile else '✗ MISSING — check Dockerfile.hf exists'}")
+        print(f"  README.md    : {'✓' if has_readme else '✗ MISSING — check README.hf.md exists'}")
+        print(f"  start_hf.sh  : {'✓' if has_start_sh else '✗ MISSING — check scripts/start_hf.sh'}")
+
+        if not has_dockerfile:
+            print()
+            print("ERROR: Dockerfile missing from staging. Aborting.")
+            sys.exit(1)
+
+        # ── Upload ─────────────────────────────────────────────────────────
+        print()
+        print(f"Uploading to {REPO_ID}…")
+        api.upload_folder(
+            folder_path=staging,
+            repo_id=REPO_ID,
+            repo_type=REPO_TYPE,
+            commit_message="Deploy URAAS — African Research Archival & Analytics System",
+        )
+
+    # ── Done ───────────────────────────────────────────────────────────────
+    print()
+    print("═" * 55)
+    print("  Upload complete! Build starting on HF (~5 min).")
+    print()
+    print("  Watch build: https://huggingface.co/spaces/Lordkiki/APA-URAAS")
+    print("  App URL    : https://lordkiki-apa-uraas.hf.space")
+    print()
+    print("  ─── Secrets to set in Space Settings → Variables & Secrets ───")
+    secrets = [
+        ("URAAS_ENV",              "production"),
+        ("DASHBOARD_SECRET_KEY",   "307790fc5aff3fe1e766303f6b94e2fc28c831582bfba5b34802e2c9cbbac0ce"),
+        ("ADMIN_USERNAME",         "admin"),
+        ("ADMIN_PASSWORD_HASH",    "scrypt:32768:8:1$r2KZFX32rJ2twbfV$16f394a253c2b505a215ff2747f7dafbb098eb0eb8b4e6bb9fb521f0ea38af8ce71f33d2354245d68cc383678257367bf410aa45f835b5e848bff95a746878c8"),
+        ("VIEWER_USERNAME",        "viewer"),
+        ("VIEWER_PASSWORD_HASH",   "scrypt:32768:8:1$M8OjWxX64B38akos$2803ad5b29508c4d69df115579630b2a4cbdf2c8406157598c088d5511a7b3d79f91399035040660705413c63fdc41aa0d020cbd7b45038c8ed51d20092ea609"),
+        ("SMTP_HOST",              "smtp.gmail.com"),
+        ("SMTP_PORT",              "587"),
+        ("SMTP_USE_TLS",           "true"),
+        ("SMTP_USER",              "lawalgiyath200716@gmail.com"),
+        ("SMTP_PASSWORD",          "ufwqbdrecpfrzppn"),
+        ("SMTP_FROM",              "URAAS UNILAG <lawalgiyath200716@gmail.com>"),
+        ("DASHBOARD_BASE_URL",     "https://lordkiki-apa-uraas.hf.space"),
+        ("DASHBOARD_CORS_ORIGINS", "https://lordkiki-apa-uraas.hf.space"),
+        ("ARK_NAAN",               "99999"),
+        ("ARK_SHOULDER",           "z1"),
+        ("OPENALEX_MAILTO",        "lawalgiyath200716@gmail.com"),
+        ("DSPACE_API_URL",         "https://api-ir.unilag.edu.ng/server"),
+        ("DSPACE_USERNAME",        "<professor email — set as Secret, not Variable>"),
+        ("DSPACE_PASSWORD",        "<professor password — set as Secret, not Variable>"),
+    ]
+    max_k = max(len(k) for k, _ in secrets)
+    for k, v in secrets:
+        print(f"  {k:<{max_k}} = {v}")
+    print()
+    print("  Admin login : admin / URAAS2024demo")
+    print("  Viewer login: viewer / view2024")
+    print("═" * 55)
+
+
+if __name__ == "__main__":
+    main()

scripts/reclassify_and_prune_sc.py

@@ -1,199 +1,199 @@
-"""
-Re-classify every Item with the Special Collections decision engine and prune
-everything that is not a genuine special collection.
-
-The platform is Special-Collections-only: papers that the engine scores 0 are
-research noise (STEM/medical/jargon) and must be removed.
-
-Usage:
-    python scripts/reclassify_and_prune_sc.py            # DRY RUN (default) — no writes
-    python scripts/reclassify_and_prune_sc.py --apply    # re-score + delete non-SC
-
-The --apply pass:
-  1. Backs up uraas.db -> uraas.db.bak (SQLite only).
-  2. Re-scores all items, writing special_collection_score / _categories.
-  3. Deletes items with score == 0 (ORM delete so association/file rows cascade),
-     then removes orphan authors / empty collections / empty communities.
-  4. Flushes the analytics cache.
-
-Run with the dashboard and any crawler STOPPED to avoid SQLite write locks.
-"""
-
-import argparse
-import os
-import shutil
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from sqlalchemy import text
-
-from uraas.config import config
-from uraas.database import (
-    Author,
-    Collection,
-    Community,
-    Item,
-    SessionLocal,
-    engine,
-    item_authors,
-)
-from uraas.services.sc_engine import is_special_collection
-from uraas.utils.analytics_cache import analytics_cache
-
-
-def backup_sqlite():
-    url = (config.DATABASE_URL or "").lower()
-    if not url.startswith("sqlite"):
-        print(f"[backup] Non-SQLite DB ({url[:30]}...) — skipping file backup.")
-        return
-    db_path = config.DATABASE_URL.split("///")[-1]
-    if not os.path.exists(db_path):
-        print(f"[backup] DB file not found at {db_path}; nothing to back up.")
-        return
-    bak = db_path + ".bak"
-    shutil.copy2(db_path, bak)
-    print(f"[backup] {db_path} -> {bak}")
-
-
-def rescore(session, apply: bool):
-    """Re-score every item. Returns (keep_ids, drop_ids)."""
-    items = session.query(Item).all()
-    keep_ids, drop_ids = [], []
-    for it in items:
-        is_sc, score, cats = is_special_collection(
-            it.title or "", it.abstract or "", it.dc_subject or ""
-        )
-        if apply:
-            it.special_collection_score = float(score)
-            it.special_collection_categories = ",".join(cats) if is_sc else ""
-        (keep_ids if is_sc else drop_ids).append(it.id)
-    if apply:
-        session.commit()
-    return keep_ids, drop_ids
-
-
-def prune(session, drop_ids):
-    """Delete non-SC items + orphan authors/collections/communities."""
-    # Enforce FK cascade for this SQLite connection (default is OFF).
-    session.execute(text("PRAGMA foreign_keys=ON"))
-
-    deleted = 0
-    for chunk_start in range(0, len(drop_ids), 500):
-        chunk = drop_ids[chunk_start : chunk_start + 500]
-        for it in session.query(Item).filter(Item.id.in_(chunk)).all():
-            session.delete(it)  # ORM delete -> association + file rows cascade
-            deleted += 1
-        session.commit()
-    print(f"[prune] deleted {deleted} non-SC items")
-
-    # Sweep stray association rows that referenced deleted items (SQLite FK
-    # cascade is unreliable for raw association tables across chunked deletes).
-    session.execute(
-        text(
-            "DELETE FROM item_authors WHERE item_id NOT IN (SELECT id FROM items) "
-            "OR author_id NOT IN (SELECT id FROM authors)"
-        )
-    )
-    session.execute(
-        text(
-            "DELETE FROM item_collections WHERE item_id NOT IN (SELECT id FROM items) "
-            "OR collection_id NOT IN (SELECT id FROM collections)"
-        )
-    )
-    session.commit()
-
-    # Orphan authors: no remaining item associations.
-    orphan_authors = (
-        session.query(Author)
-        .filter(~Author.id.in_(session.query(item_authors.c.author_id)))
-        .all()
-    )
-    for a in orphan_authors:
-        session.delete(a)
-    print(f"[prune] deleted {len(orphan_authors)} orphan authors")
-
-    # Empty collections (no items) and then empty communities (no collections).
-    empty_colls = [c for c in session.query(Collection).all() if not c.items]
-    for c in empty_colls:
-        session.delete(c)
-    session.commit()
-    print(f"[prune] deleted {len(empty_colls)} empty collections")
-
-    empty_comms = [c for c in session.query(Community).all() if not c.collections]
-    for c in empty_comms:
-        session.delete(c)
-    session.commit()
-    print(f"[prune] deleted {len(empty_comms)} empty communities")
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Re-classify & prune non-SC papers")
-    parser.add_argument(
-        "--apply", action="store_true", help="Actually re-score and delete (default: dry run)"
-    )
-    parser.add_argument(
-        "--samples", type=int, default=20, help="How many borderline drops to print"
-    )
-    args = parser.parse_args()
-
-    session = SessionLocal()
-    try:
-        total = session.query(Item).count()
-        old_sc = session.query(Item).filter(Item.special_collection_score > 0).count()
-        print("=" * 64)
-        print(f"Total items: {total}   (old score>0: {old_sc})")
-        print("=" * 64)
-
-        if args.apply:
-            backup_sqlite()
-
-        keep_ids, drop_ids = rescore(session, apply=args.apply)
-        print(f"\nKEEP (special collections): {len(keep_ids)}")
-        print(f"DROP (not special collections): {len(drop_ids)}")
-
-        # Show a sample of what would be / was dropped that previously scored > 0
-        # (these are the meaningful changes to eyeball).
-        prev_sc = {
-            i for (i,) in session.query(Item.id).filter(Item.special_collection_score >= 0).all()
-        } if not args.apply else set()
-        sample = (
-            session.query(Item.title)
-            .filter(Item.id.in_(drop_ids[: args.samples]))
-            .all()
-        )
-        print(f"\n--- sample of dropped titles (first {args.samples}) ---")
-        for (t,) in sample:
-            safe = (t or "").encode("ascii", "replace").decode()
-            print("  DROP:", safe[:90])
-
-        if not args.apply:
-            print("\n[DRY RUN] No changes written. Re-run with --apply to prune.")
-            return 0
-
-        prune(session, drop_ids)
-        analytics_cache.invalidate_all()
-
-        remaining = session.query(Item).count()
-        sc_remaining = (
-            session.query(Item).filter(Item.special_collection_score > 0).count()
-        )
-        orphan_left = (
-            session.query(Author)
-            .filter(~Author.id.in_(session.query(item_authors.c.author_id)))
-            .count()
-        )
-        print("\n" + "=" * 64)
-        print(f"DONE. Items remaining: {remaining}  (score>0: {sc_remaining})")
-        print(f"Orphan authors remaining: {orphan_left}")
-        assert remaining == sc_remaining, "Mismatch: non-SC rows survived!"
-        assert orphan_left == 0, "Orphan authors survived!"
-        print("Invariants OK. Restart the dashboard to serve fresh data.")
-        print("=" * 64)
-        return 0
-    finally:
-        session.close()
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+"""
+Re-classify every Item with the Special Collections decision engine and prune
+everything that is not a genuine special collection.
+
+The platform is Special-Collections-only: papers that the engine scores 0 are
+research noise (STEM/medical/jargon) and must be removed.
+
+Usage:
+    python scripts/reclassify_and_prune_sc.py            # DRY RUN (default) — no writes
+    python scripts/reclassify_and_prune_sc.py --apply    # re-score + delete non-SC
+
+The --apply pass:
+  1. Backs up uraas.db -> uraas.db.bak (SQLite only).
+  2. Re-scores all items, writing special_collection_score / _categories.
+  3. Deletes items with score == 0 (ORM delete so association/file rows cascade),
+     then removes orphan authors / empty collections / empty communities.
+  4. Flushes the analytics cache.
+
+Run with the dashboard and any crawler STOPPED to avoid SQLite write locks.
+"""
+
+import argparse
+import os
+import shutil
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from sqlalchemy import text
+
+from uraas.config import config
+from uraas.database import (
+    Author,
+    Collection,
+    Community,
+    Item,
+    SessionLocal,
+    engine,
+    item_authors,
+)
+from uraas.services.sc_engine import is_special_collection
+from uraas.utils.analytics_cache import analytics_cache
+
+
+def backup_sqlite():
+    url = (config.DATABASE_URL or "").lower()
+    if not url.startswith("sqlite"):
+        print(f"[backup] Non-SQLite DB ({url[:30]}...) — skipping file backup.")
+        return
+    db_path = config.DATABASE_URL.split("///")[-1]
+    if not os.path.exists(db_path):
+        print(f"[backup] DB file not found at {db_path}; nothing to back up.")
+        return
+    bak = db_path + ".bak"
+    shutil.copy2(db_path, bak)
+    print(f"[backup] {db_path} -> {bak}")
+
+
+def rescore(session, apply: bool):
+    """Re-score every item. Returns (keep_ids, drop_ids)."""
+    items = session.query(Item).all()
+    keep_ids, drop_ids = [], []
+    for it in items:
+        is_sc, score, cats = is_special_collection(
+            it.title or "", it.abstract or "", it.dc_subject or ""
+        )
+        if apply:
+            it.special_collection_score = float(score)
+            it.special_collection_categories = ",".join(cats) if is_sc else ""
+        (keep_ids if is_sc else drop_ids).append(it.id)
+    if apply:
+        session.commit()
+    return keep_ids, drop_ids
+
+
+def prune(session, drop_ids):
+    """Delete non-SC items + orphan authors/collections/communities."""
+    # Enforce FK cascade for this SQLite connection (default is OFF).
+    session.execute(text("PRAGMA foreign_keys=ON"))
+
+    deleted = 0
+    for chunk_start in range(0, len(drop_ids), 500):
+        chunk = drop_ids[chunk_start : chunk_start + 500]
+        for it in session.query(Item).filter(Item.id.in_(chunk)).all():
+            session.delete(it)  # ORM delete -> association + file rows cascade
+            deleted += 1
+        session.commit()
+    print(f"[prune] deleted {deleted} non-SC items")
+
+    # Sweep stray association rows that referenced deleted items (SQLite FK
+    # cascade is unreliable for raw association tables across chunked deletes).
+    session.execute(
+        text(
+            "DELETE FROM item_authors WHERE item_id NOT IN (SELECT id FROM items) "
+            "OR author_id NOT IN (SELECT id FROM authors)"
+        )
+    )
+    session.execute(
+        text(
+            "DELETE FROM item_collections WHERE item_id NOT IN (SELECT id FROM items) "
+            "OR collection_id NOT IN (SELECT id FROM collections)"
+        )
+    )
+    session.commit()
+
+    # Orphan authors: no remaining item associations.
+    orphan_authors = (
+        session.query(Author)
+        .filter(~Author.id.in_(session.query(item_authors.c.author_id)))
+        .all()
+    )
+    for a in orphan_authors:
+        session.delete(a)
+    print(f"[prune] deleted {len(orphan_authors)} orphan authors")
+
+    # Empty collections (no items) and then empty communities (no collections).
+    empty_colls = [c for c in session.query(Collection).all() if not c.items]
+    for c in empty_colls:
+        session.delete(c)
+    session.commit()
+    print(f"[prune] deleted {len(empty_colls)} empty collections")
+
+    empty_comms = [c for c in session.query(Community).all() if not c.collections]
+    for c in empty_comms:
+        session.delete(c)
+    session.commit()
+    print(f"[prune] deleted {len(empty_comms)} empty communities")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Re-classify & prune non-SC papers")
+    parser.add_argument(
+        "--apply", action="store_true", help="Actually re-score and delete (default: dry run)"
+    )
+    parser.add_argument(
+        "--samples", type=int, default=20, help="How many borderline drops to print"
+    )
+    args = parser.parse_args()
+
+    session = SessionLocal()
+    try:
+        total = session.query(Item).count()
+        old_sc = session.query(Item).filter(Item.special_collection_score > 0).count()
+        print("=" * 64)
+        print(f"Total items: {total}   (old score>0: {old_sc})")
+        print("=" * 64)
+
+        if args.apply:
+            backup_sqlite()
+
+        keep_ids, drop_ids = rescore(session, apply=args.apply)
+        print(f"\nKEEP (special collections): {len(keep_ids)}")
+        print(f"DROP (not special collections): {len(drop_ids)}")
+
+        # Show a sample of what would be / was dropped that previously scored > 0
+        # (these are the meaningful changes to eyeball).
+        prev_sc = {
+            i for (i,) in session.query(Item.id).filter(Item.special_collection_score >= 0).all()
+        } if not args.apply else set()
+        sample = (
+            session.query(Item.title)
+            .filter(Item.id.in_(drop_ids[: args.samples]))
+            .all()
+        )
+        print(f"\n--- sample of dropped titles (first {args.samples}) ---")
+        for (t,) in sample:
+            safe = (t or "").encode("ascii", "replace").decode()
+            print("  DROP:", safe[:90])
+
+        if not args.apply:
+            print("\n[DRY RUN] No changes written. Re-run with --apply to prune.")
+            return 0
+
+        prune(session, drop_ids)
+        analytics_cache.invalidate_all()
+
+        remaining = session.query(Item).count()
+        sc_remaining = (
+            session.query(Item).filter(Item.special_collection_score > 0).count()
+        )
+        orphan_left = (
+            session.query(Author)
+            .filter(~Author.id.in_(session.query(item_authors.c.author_id)))
+            .count()
+        )
+        print("\n" + "=" * 64)
+        print(f"DONE. Items remaining: {remaining}  (score>0: {sc_remaining})")
+        print(f"Orphan authors remaining: {orphan_left}")
+        assert remaining == sc_remaining, "Mismatch: non-SC rows survived!"
+        assert orphan_left == 0, "Orphan authors survived!"
+        print("Invariants OK. Restart the dashboard to serve fresh data.")
+        print("=" * 64)
+        return 0
+    finally:
+        session.close()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/scrape_nigerian_universities.py

@@ -1,618 +1,618 @@
-"""
-Comprehensive scraper for Nigerian university faculty directories
-Collects full staff names with high accuracy
-"""
-
-import json
-import re
-import time
-from typing import Dict, List, Set
-from urllib.parse import urljoin, urlparse
-
-import requests
-from bs4 import BeautifulSoup
-
-
-class UniversityStaffScraper:
-    """Base class for university staff scraping"""
-
-    def __init__(self, institution_name: str, base_url: str):
-        self.institution_name = institution_name
-        self.base_url = base_url
-        self.session = requests.Session()
-        self.session.headers.update(
-            {
-                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
-            }
-        )
-        self.staff_data = []
-        self.staff_names = set()
-
-    def clean_name(self, name: str) -> str:
-        """Clean and standardize name"""
-        if not name:
-            return ""
-
-        # Remove extra whitespace
-        name = re.sub(r"\s+", " ", name).strip()
-
-        # Remove common artifacts
-        name = re.sub(r"\s*\([^)]*\)\s*", " ", name)  # Remove parentheses content
-        name = re.sub(r"\s*\[[^\]]*\]\s*", " ", name)  # Remove brackets content
-        name = re.sub(r"\s+", " ", name).strip()
-
-        # Ensure proper capitalization
-        if name.isupper() or name.islower():
-            name = name.title()
-
-        return name
-
-    def is_valid_name(self, name: str) -> bool:
-        """Validate if string is a proper name"""
-        if not name or len(name) < 5:
-            return False
-
-        # Must have at least 2 words
-        words = name.split()
-        if len(words) < 2:
-            return False
-
-        # Must contain letters
-        if not re.search(r"[a-zA-Z]", name):
-            return False
-
-        # Reject if too many numbers
-        if len(re.findall(r"\d", name)) > 3:
-            return False
-
-        # Reject common non-name patterns
-        reject_patterns = [
-            r"^(page|home|about|contact|staff|faculty|department)",
-            r"(\.pdf|\.doc|\.jpg|\.png)$",
-            r"^(dr|prof|mr|mrs|ms)\.?$",
-            r"^\d+$",
-        ]
-
-        for pattern in reject_patterns:
-            if re.search(pattern, name.lower()):
-                return False
-
-        return True
-
-    def save_to_json(self, filename: str):
-        """Save collected staff data to JSON"""
-        output = {
-            "institution": self.institution_name,
-            "total_staff": len(self.staff_names),
-            "collection_date": time.strftime("%Y-%m-%d"),
-            "staff": sorted(list(self.staff_names)),
-            "detailed_records": self.staff_data,
-        }
-
-        with open(filename, "w", encoding="utf-8") as f:
-            json.dump(output, f, indent=2, ensure_ascii=False)
-
-        print(f"\n✓ Saved {len(self.staff_names)} staff members to {filename}")
-
-    def scrape(self):
-        """Override in subclass"""
-        raise NotImplementedError
-
-
-class UIStaffScraper(UniversityStaffScraper):
-    """University of Ibadan staff scraper"""
-
-    def __init__(self):
-        super().__init__("University of Ibadan", "https://www.ui.edu.ng")
-
-    def scrape(self):
-        """Scrape UI faculty directory"""
-        print(f"\n{'='*60}")
-        print(f"Scraping: {self.institution_name}")
-        print(f"{'='*60}")
-
-        # UI faculty pages
-        faculty_urls = [
-            "/faculties/arts",
-            "/faculties/science",
-            "/faculties/technology",
-            "/faculties/agriculture-and-forestry",
-            "/faculties/veterinary-medicine",
-            "/faculties/medicine",
-            "/faculties/dentistry",
-            "/faculties/pharmacy",
-            "/faculties/public-health",
-            "/faculties/social-sciences",
-            "/faculties/law",
-            "/faculties/education",
-            "/faculties/environmental-design-and-management",
-        ]
-
-        # Try to scrape from staff directory if available
-        try:
-            response = self.session.get(f"{self.base_url}/staff-directory", timeout=10)
-            if response.status_code == 200:
-                self._parse_staff_page(response.text, "Staff Directory")
-        except Exception as e:
-            print(f"  Note: Staff directory not accessible: {e}")
-
-        # Try faculty pages
-        for faculty_url in faculty_urls:
-            try:
-                url = urljoin(self.base_url, faculty_url)
-                print(f"  Checking: {url}")
-                response = self.session.get(url, timeout=10)
-
-                if response.status_code == 200:
-                    self._parse_staff_page(response.text, faculty_url)
-                    time.sleep(1)  # Be polite
-
-            except Exception as e:
-                print(f"  Error accessing {faculty_url}: {e}")
-
-        print(f"\n  Total staff collected: {len(self.staff_names)}")
-
-    def _parse_staff_page(self, html: str, source: str):
-        """Parse HTML page for staff names"""
-        soup = BeautifulSoup(html, "html.parser")
-
-        # Look for common patterns
-        patterns = [
-            ("div", {"class": re.compile(r"staff|faculty|member|person")}),
-            ("li", {"class": re.compile(r"staff|faculty|member")}),
-            ("h3", {}),
-            ("h4", {}),
-            ("p", {"class": re.compile(r"name|staff")}),
-        ]
-
-        for tag, attrs in patterns:
-            elements = soup.find_all(tag, attrs)
-            for elem in elements:
-                text = elem.get_text(strip=True)
-                name = self.clean_name(text)
-
-                if self.is_valid_name(name) and name not in self.staff_names:
-                    self.staff_names.add(name)
-                    self.staff_data.append(
-                        {"name": name, "source": source, "faculty": "Unknown"}
-                    )
-
-
-class OAUStaffScraper(UniversityStaffScraper):
-    """Obafemi Awolowo University staff scraper"""
-
-    def __init__(self):
-        super().__init__("Obafemi Awolowo University", "https://oauife.edu.ng")
-
-    def scrape(self):
-        """Scrape OAU faculty directory"""
-        print(f"\n{'='*60}")
-        print(f"Scraping: {self.institution_name}")
-        print(f"{'='*60}")
-
-        # OAU faculty pages
-        faculty_urls = [
-            "/faculties/arts",
-            "/faculties/science",
-            "/faculties/technology",
-            "/faculties/agriculture",
-            "/faculties/basic-medical-sciences",
-            "/faculties/clinical-sciences",
-            "/faculties/dentistry",
-            "/faculties/pharmacy",
-            "/faculties/social-sciences",
-            "/faculties/law",
-            "/faculties/education",
-            "/faculties/environmental-design",
-        ]
-
-        # Try staff directory
-        try:
-            response = self.session.get(f"{self.base_url}/staff", timeout=10)
-            if response.status_code == 200:
-                self._parse_staff_page(response.text, "Staff Directory")
-        except Exception as e:
-            print(f"  Note: Staff directory not accessible: {e}")
-
-        # Try faculty pages
-        for faculty_url in faculty_urls:
-            try:
-                url = urljoin(self.base_url, faculty_url)
-                print(f"  Checking: {url}")
-                response = self.session.get(url, timeout=10)
-
-                if response.status_code == 200:
-                    self._parse_staff_page(response.text, faculty_url)
-                    time.sleep(1)
-
-            except Exception as e:
-                print(f"  Error accessing {faculty_url}: {e}")
-
-        print(f"\n  Total staff collected: {len(self.staff_names)}")
-
-    def _parse_staff_page(self, html: str, source: str):
-        """Parse HTML page for staff names"""
-        soup = BeautifulSoup(html, "html.parser")
-
-        # Look for staff names
-        patterns = [
-            ("div", {"class": re.compile(r"staff|faculty|member|person")}),
-            ("li", {"class": re.compile(r"staff|faculty|member")}),
-            ("h3", {}),
-            ("h4", {}),
-            ("span", {"class": re.compile(r"name")}),
-        ]
-
-        for tag, attrs in patterns:
-            elements = soup.find_all(tag, attrs)
-            for elem in elements:
-                text = elem.get_text(strip=True)
-                name = self.clean_name(text)
-
-                if self.is_valid_name(name) and name not in self.staff_names:
-                    self.staff_names.add(name)
-                    self.staff_data.append(
-                        {"name": name, "source": source, "faculty": "Unknown"}
-                    )
-
-
-class UNNStaffScraper(UniversityStaffScraper):
-    """University of Nigeria, Nsukka staff scraper"""
-
-    def __init__(self):
-        super().__init__("University of Nigeria, Nsukka", "https://www.unn.edu.ng")
-
-    def scrape(self):
-        """Scrape UNN faculty directory"""
-        print(f"\n{'='*60}")
-        print(f"Scraping: {self.institution_name}")
-        print(f"{'='*60}")
-
-        # UNN faculty pages
-        faculty_urls = [
-            "/faculties/arts",
-            "/faculties/biological-sciences",
-            "/faculties/physical-sciences",
-            "/faculties/engineering",
-            "/faculties/agriculture",
-            "/faculties/veterinary-medicine",
-            "/faculties/medical-sciences",
-            "/faculties/dentistry",
-            "/faculties/pharmaceutical-sciences",
-            "/faculties/health-sciences",
-            "/faculties/social-sciences",
-            "/faculties/law",
-            "/faculties/education",
-            "/faculties/environmental-studies",
-            "/faculties/business-administration",
-        ]
-
-        # Try staff directory
-        try:
-            response = self.session.get(f"{self.base_url}/staff-directory", timeout=10)
-            if response.status_code == 200:
-                self._parse_staff_page(response.text, "Staff Directory")
-        except Exception as e:
-            print(f"  Note: Staff directory not accessible: {e}")
-
-        # Try faculty pages
-        for faculty_url in faculty_urls:
-            try:
-                url = urljoin(self.base_url, faculty_url)
-                print(f"  Checking: {url}")
-                response = self.session.get(url, timeout=10)
-
-                if response.status_code == 200:
-                    self._parse_staff_page(response.text, faculty_url)
-                    time.sleep(1)
-
-            except Exception as e:
-                print(f"  Error accessing {faculty_url}: {e}")
-
-        print(f"\n  Total staff collected: {len(self.staff_names)}")
-
-    def _parse_staff_page(self, html: str, source: str):
-        """Parse HTML page for staff names"""
-        soup = BeautifulSoup(html, "html.parser")
-
-        # Look for staff names
-        patterns = [
-            ("div", {"class": re.compile(r"staff|faculty|member|person")}),
-            ("li", {"class": re.compile(r"staff|faculty|member")}),
-            ("h3", {}),
-            ("h4", {}),
-            ("td", {}),
-        ]
-
-        for tag, attrs in patterns:
-            elements = soup.find_all(tag, attrs)
-            for elem in elements:
-                text = elem.get_text(strip=True)
-                name = self.clean_name(text)
-
-                if self.is_valid_name(name) and name not in self.staff_names:
-                    self.staff_names.add(name)
-                    self.staff_data.append(
-                        {"name": name, "source": source, "faculty": "Unknown"}
-                    )
-
-
-class ABUStaffScraper(UniversityStaffScraper):
-    """Ahmadu Bello University staff scraper"""
-
-    def __init__(self):
-        super().__init__("Ahmadu Bello University", "https://www.abu.edu.ng")
-
-    def scrape(self):
-        """Scrape ABU faculty directory"""
-        print(f"\n{'='*60}")
-        print(f"Scraping: {self.institution_name}")
-        print(f"{'='*60}")
-
-        # ABU faculty pages
-        faculty_urls = [
-            "/faculties/arts-and-islamic-studies",
-            "/faculties/science",
-            "/faculties/engineering",
-            "/faculties/agriculture",
-            "/faculties/veterinary-medicine",
-            "/faculties/medicine",
-            "/faculties/dentistry",
-            "/faculties/pharmaceutical-sciences",
-            "/faculties/allied-health-sciences",
-            "/faculties/social-sciences",
-            "/faculties/law",
-            "/faculties/education",
-            "/faculties/environmental-design",
-            "/faculties/administration",
-        ]
-
-        # Try staff directory
-        try:
-            response = self.session.get(f"{self.base_url}/staff", timeout=10)
-            if response.status_code == 200:
-                self._parse_staff_page(response.text, "Staff Directory")
-        except Exception as e:
-            print(f"  Note: Staff directory not accessible: {e}")
-
-        # Try faculty pages
-        for faculty_url in faculty_urls:
-            try:
-                url = urljoin(self.base_url, faculty_url)
-                print(f"  Checking: {url}")
-                response = self.session.get(url, timeout=10)
-
-                if response.status_code == 200:
-                    self._parse_staff_page(response.text, faculty_url)
-                    time.sleep(1)
-
-            except Exception as e:
-                print(f"  Error accessing {faculty_url}: {e}")
-
-        print(f"\n  Total staff collected: {len(self.staff_names)}")
-
-    def _parse_staff_page(self, html: str, source: str):
-        """Parse HTML page for staff names"""
-        soup = BeautifulSoup(html, "html.parser")
-
-        # Look for staff names
-        patterns = [
-            ("div", {"class": re.compile(r"staff|faculty|member|person")}),
-            ("li", {"class": re.compile(r"staff|faculty|member")}),
-            ("h3", {}),
-            ("h4", {}),
-            ("span", {"class": re.compile(r"name")}),
-        ]
-
-        for tag, attrs in patterns:
-            elements = soup.find_all(tag, attrs)
-            for elem in elements:
-                text = elem.get_text(strip=True)
-                name = self.clean_name(text)
-
-                if self.is_valid_name(name) and name not in self.staff_names:
-                    self.staff_names.add(name)
-                    self.staff_data.append(
-                        {"name": name, "source": source, "faculty": "Unknown"}
-                    )
-
-
-def generate_sample_names(institution: str, count: int) -> List[str]:
-    """
-    Generate realistic Nigerian academic staff names as fallback
-    Uses common Nigerian naming patterns
-    """
-
-    # Common Nigerian surnames by region
-    yoruba_surnames = [
-        "Adeyemi",
-        "Ogunlana",
-        "Oluwaseun",
-        "Babatunde",
-        "Adebayo",
-        "Oladipo",
-        "Adekunle",
-        "Olatunji",
-        "Adewale",
-        "Olaniyan",
-        "Afolabi",
-        "Ogunbiyi",
-        "Adeyinka",
-        "Oladele",
-        "Adebisi",
-        "Ogunleye",
-        "Adeola",
-        "Olayinka",
-    ]
-
-    igbo_surnames = [
-        "Okonkwo",
-        "Nwosu",
-        "Okeke",
-        "Eze",
-        "Okafor",
-        "Nwankwo",
-        "Chukwu",
-        "Onyeka",
-        "Ikechukwu",
-        "Obiora",
-        "Emeka",
-        "Chinedu",
-        "Ugochukwu",
-        "Nnamdi",
-        "Chibueze",
-        "Obinna",
-        "Kelechi",
-        "Chukwuemeka",
-    ]
-
-    hausa_surnames = [
-        "Ibrahim",
-        "Mohammed",
-        "Abdullahi",
-        "Usman",
-        "Ahmad",
-        "Hassan",
-        "Aliyu",
-        "Musa",
-        "Abubakar",
-        "Suleiman",
-        "Yusuf",
-        "Ismail",
-        "Bello",
-        "Garba",
-        "Sani",
-        "Umar",
-        "Tijjani",
-        "Kabir",
-    ]
-
-    # Common first names
-    first_names = [
-        "Oluwaseun",
-        "Chinedu",
-        "Abubakar",
-        "Ngozi",
-        "Fatima",
-        "Chiamaka",
-        "Tunde",
-        "Emeka",
-        "Musa",
-        "Adaeze",
-        "Zainab",
-        "Chioma",
-        "Segun",
-        "Obinna",
-        "Aliyu",
-        "Amaka",
-        "Aisha",
-        "Ifeoma",
-    ]
-
-    # Academic titles
-    titles = ["Prof.", "Dr.", "Mr.", "Mrs.", "Ms."]
-
-    # Middle initials
-    initials = list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
-
-    import random
-
-    random.seed(42)  # For reproducibility
-
-    all_surnames = yoruba_surnames + igbo_surnames + hausa_surnames
-    names = set()
-
-    while len(names) < count:
-        title = random.choice(titles)
-        first = random.choice(first_names)
-        middle = random.choice(initials)
-        surname = random.choice(all_surnames)
-
-        # Various name formats
-        formats = [
-            f"{title} {first} {middle}. {surname}",
-            f"{title} {first} {surname}",
-            f"{first} {middle}. {surname}",
-            f"{surname}, {first} {middle}.",
-        ]
-
-        name = random.choice(formats)
-        names.add(name)
-
-    return sorted(list(names))
-
-
-def main():
-    """Main scraping function"""
-    print("\n" + "=" * 60)
-    print("NIGERIAN UNIVERSITIES STAFF DATA COLLECTION")
-    print("=" * 60)
-    print("\nTarget: Collect full staff names from 4 universities")
-    print("Quality: Full names only, no abbreviations, no mistakes")
-    print("=" * 60)
-
-    results = {}
-
-    # Scrape each university
-    scrapers = [
-        (UIStaffScraper(), "data/ui_staff.json"),
-        (OAUStaffScraper(), "data/oau_staff.json"),
-        (UNNStaffScraper(), "data/unn_staff.json"),
-        (ABUStaffScraper(), "data/abu_staff.json"),
-    ]
-
-    for scraper, filename in scrapers:
-        try:
-            scraper.scrape()
-
-            # If scraping didn't yield enough results, generate sample data
-            if len(scraper.staff_names) < 50:
-                print(f"\n  ⚠ Warning: Only {len(scraper.staff_names)} names collected")
-                print(f"  Generating sample Nigerian academic names for testing...")
-
-                sample_names = generate_sample_names(scraper.institution_name, 300)
-                scraper.staff_names.update(sample_names)
-
-                for name in sample_names:
-                    scraper.staff_data.append(
-                        {
-                            "name": name,
-                            "source": "Generated Sample",
-                            "faculty": "Unknown",
-                        }
-                    )
-
-                print(f"  ✓ Added {len(sample_names)} sample names")
-
-            scraper.save_to_json(filename)
-            results[scraper.institution_name] = len(scraper.staff_names)
-
-        except Exception as e:
-            print(f"\n✗ Error scraping {scraper.institution_name}: {e}")
-            import traceback
-
-            traceback.print_exc()
-
-    # Summary
-    print("\n" + "=" * 60)
-    print("COLLECTION SUMMARY")
-    print("=" * 60)
-
-    total = 0
-    for institution, count in results.items():
-        print(f"  {institution}: {count} staff members")
-        total += count
-
-    print(f"\n  TOTAL: {total} staff members across 4 universities")
-    print("=" * 60)
-
-    print("\n✓ Data collection complete!")
-    print("\nNext steps:")
-    print("  1. Review generated JSON files in data/ directory")
-    print("  2. Manually verify sample of names")
-    print("  3. Run test_multi_institution.py to verify")
-    print("  4. Proceed to spider integration (Day 5-7)")
-
-
-if __name__ == "__main__":
-    main()
+"""
+Comprehensive scraper for Nigerian university faculty directories
+Collects full staff names with high accuracy
+"""
+
+import json
+import re
+import time
+from typing import Dict, List, Set
+from urllib.parse import urljoin, urlparse
+
+import requests
+from bs4 import BeautifulSoup
+
+
+class UniversityStaffScraper:
+    """Base class for university staff scraping"""
+
+    def __init__(self, institution_name: str, base_url: str):
+        self.institution_name = institution_name
+        self.base_url = base_url
+        self.session = requests.Session()
+        self.session.headers.update(
+            {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+            }
+        )
+        self.staff_data = []
+        self.staff_names = set()
+
+    def clean_name(self, name: str) -> str:
+        """Clean and standardize name"""
+        if not name:
+            return ""
+
+        # Remove extra whitespace
+        name = re.sub(r"\s+", " ", name).strip()
+
+        # Remove common artifacts
+        name = re.sub(r"\s*\([^)]*\)\s*", " ", name)  # Remove parentheses content
+        name = re.sub(r"\s*\[[^\]]*\]\s*", " ", name)  # Remove brackets content
+        name = re.sub(r"\s+", " ", name).strip()
+
+        # Ensure proper capitalization
+        if name.isupper() or name.islower():
+            name = name.title()
+
+        return name
+
+    def is_valid_name(self, name: str) -> bool:
+        """Validate if string is a proper name"""
+        if not name or len(name) < 5:
+            return False
+
+        # Must have at least 2 words
+        words = name.split()
+        if len(words) < 2:
+            return False
+
+        # Must contain letters
+        if not re.search(r"[a-zA-Z]", name):
+            return False
+
+        # Reject if too many numbers
+        if len(re.findall(r"\d", name)) > 3:
+            return False
+
+        # Reject common non-name patterns
+        reject_patterns = [
+            r"^(page|home|about|contact|staff|faculty|department)",
+            r"(\.pdf|\.doc|\.jpg|\.png)$",
+            r"^(dr|prof|mr|mrs|ms)\.?$",
+            r"^\d+$",
+        ]
+
+        for pattern in reject_patterns:
+            if re.search(pattern, name.lower()):
+                return False
+
+        return True
+
+    def save_to_json(self, filename: str):
+        """Save collected staff data to JSON"""
+        output = {
+            "institution": self.institution_name,
+            "total_staff": len(self.staff_names),
+            "collection_date": time.strftime("%Y-%m-%d"),
+            "staff": sorted(list(self.staff_names)),
+            "detailed_records": self.staff_data,
+        }
+
+        with open(filename, "w", encoding="utf-8") as f:
+            json.dump(output, f, indent=2, ensure_ascii=False)
+
+        print(f"\n✓ Saved {len(self.staff_names)} staff members to {filename}")
+
+    def scrape(self):
+        """Override in subclass"""
+        raise NotImplementedError
+
+
+class UIStaffScraper(UniversityStaffScraper):
+    """University of Ibadan staff scraper"""
+
+    def __init__(self):
+        super().__init__("University of Ibadan", "https://www.ui.edu.ng")
+
+    def scrape(self):
+        """Scrape UI faculty directory"""
+        print(f"\n{'='*60}")
+        print(f"Scraping: {self.institution_name}")
+        print(f"{'='*60}")
+
+        # UI faculty pages
+        faculty_urls = [
+            "/faculties/arts",
+            "/faculties/science",
+            "/faculties/technology",
+            "/faculties/agriculture-and-forestry",
+            "/faculties/veterinary-medicine",
+            "/faculties/medicine",
+            "/faculties/dentistry",
+            "/faculties/pharmacy",
+            "/faculties/public-health",
+            "/faculties/social-sciences",
+            "/faculties/law",
+            "/faculties/education",
+            "/faculties/environmental-design-and-management",
+        ]
+
+        # Try to scrape from staff directory if available
+        try:
+            response = self.session.get(f"{self.base_url}/staff-directory", timeout=10)
+            if response.status_code == 200:
+                self._parse_staff_page(response.text, "Staff Directory")
+        except Exception as e:
+            print(f"  Note: Staff directory not accessible: {e}")
+
+        # Try faculty pages
+        for faculty_url in faculty_urls:
+            try:
+                url = urljoin(self.base_url, faculty_url)
+                print(f"  Checking: {url}")
+                response = self.session.get(url, timeout=10)
+
+                if response.status_code == 200:
+                    self._parse_staff_page(response.text, faculty_url)
+                    time.sleep(1)  # Be polite
+
+            except Exception as e:
+                print(f"  Error accessing {faculty_url}: {e}")
+
+        print(f"\n  Total staff collected: {len(self.staff_names)}")
+
+    def _parse_staff_page(self, html: str, source: str):
+        """Parse HTML page for staff names"""
+        soup = BeautifulSoup(html, "html.parser")
+
+        # Look for common patterns
+        patterns = [
+            ("div", {"class": re.compile(r"staff|faculty|member|person")}),
+            ("li", {"class": re.compile(r"staff|faculty|member")}),
+            ("h3", {}),
+            ("h4", {}),
+            ("p", {"class": re.compile(r"name|staff")}),
+        ]
+
+        for tag, attrs in patterns:
+            elements = soup.find_all(tag, attrs)
+            for elem in elements:
+                text = elem.get_text(strip=True)
+                name = self.clean_name(text)
+
+                if self.is_valid_name(name) and name not in self.staff_names:
+                    self.staff_names.add(name)
+                    self.staff_data.append(
+                        {"name": name, "source": source, "faculty": "Unknown"}
+                    )
+
+
+class OAUStaffScraper(UniversityStaffScraper):
+    """Obafemi Awolowo University staff scraper"""
+
+    def __init__(self):
+        super().__init__("Obafemi Awolowo University", "https://oauife.edu.ng")
+
+    def scrape(self):
+        """Scrape OAU faculty directory"""
+        print(f"\n{'='*60}")
+        print(f"Scraping: {self.institution_name}")
+        print(f"{'='*60}")
+
+        # OAU faculty pages
+        faculty_urls = [
+            "/faculties/arts",
+            "/faculties/science",
+            "/faculties/technology",
+            "/faculties/agriculture",
+            "/faculties/basic-medical-sciences",
+            "/faculties/clinical-sciences",
+            "/faculties/dentistry",
+            "/faculties/pharmacy",
+            "/faculties/social-sciences",
+            "/faculties/law",
+            "/faculties/education",
+            "/faculties/environmental-design",
+        ]
+
+        # Try staff directory
+        try:
+            response = self.session.get(f"{self.base_url}/staff", timeout=10)
+            if response.status_code == 200:
+                self._parse_staff_page(response.text, "Staff Directory")
+        except Exception as e:
+            print(f"  Note: Staff directory not accessible: {e}")
+
+        # Try faculty pages
+        for faculty_url in faculty_urls:
+            try:
+                url = urljoin(self.base_url, faculty_url)
+                print(f"  Checking: {url}")
+                response = self.session.get(url, timeout=10)
+
+                if response.status_code == 200:
+                    self._parse_staff_page(response.text, faculty_url)
+                    time.sleep(1)
+
+            except Exception as e:
+                print(f"  Error accessing {faculty_url}: {e}")
+
+        print(f"\n  Total staff collected: {len(self.staff_names)}")
+
+    def _parse_staff_page(self, html: str, source: str):
+        """Parse HTML page for staff names"""
+        soup = BeautifulSoup(html, "html.parser")
+
+        # Look for staff names
+        patterns = [
+            ("div", {"class": re.compile(r"staff|faculty|member|person")}),
+            ("li", {"class": re.compile(r"staff|faculty|member")}),
+            ("h3", {}),
+            ("h4", {}),
+            ("span", {"class": re.compile(r"name")}),
+        ]
+
+        for tag, attrs in patterns:
+            elements = soup.find_all(tag, attrs)
+            for elem in elements:
+                text = elem.get_text(strip=True)
+                name = self.clean_name(text)
+
+                if self.is_valid_name(name) and name not in self.staff_names:
+                    self.staff_names.add(name)
+                    self.staff_data.append(
+                        {"name": name, "source": source, "faculty": "Unknown"}
+                    )
+
+
+class UNNStaffScraper(UniversityStaffScraper):
+    """University of Nigeria, Nsukka staff scraper"""
+
+    def __init__(self):
+        super().__init__("University of Nigeria, Nsukka", "https://www.unn.edu.ng")
+
+    def scrape(self):
+        """Scrape UNN faculty directory"""
+        print(f"\n{'='*60}")
+        print(f"Scraping: {self.institution_name}")
+        print(f"{'='*60}")
+
+        # UNN faculty pages
+        faculty_urls = [
+            "/faculties/arts",
+            "/faculties/biological-sciences",
+            "/faculties/physical-sciences",
+            "/faculties/engineering",
+            "/faculties/agriculture",
+            "/faculties/veterinary-medicine",
+            "/faculties/medical-sciences",
+            "/faculties/dentistry",
+            "/faculties/pharmaceutical-sciences",
+            "/faculties/health-sciences",
+            "/faculties/social-sciences",
+            "/faculties/law",
+            "/faculties/education",
+            "/faculties/environmental-studies",
+            "/faculties/business-administration",
+        ]
+
+        # Try staff directory
+        try:
+            response = self.session.get(f"{self.base_url}/staff-directory", timeout=10)
+            if response.status_code == 200:
+                self._parse_staff_page(response.text, "Staff Directory")
+        except Exception as e:
+            print(f"  Note: Staff directory not accessible: {e}")
+
+        # Try faculty pages
+        for faculty_url in faculty_urls:
+            try:
+                url = urljoin(self.base_url, faculty_url)
+                print(f"  Checking: {url}")
+                response = self.session.get(url, timeout=10)
+
+                if response.status_code == 200:
+                    self._parse_staff_page(response.text, faculty_url)
+                    time.sleep(1)
+
+            except Exception as e:
+                print(f"  Error accessing {faculty_url}: {e}")
+
+        print(f"\n  Total staff collected: {len(self.staff_names)}")
+
+    def _parse_staff_page(self, html: str, source: str):
+        """Parse HTML page for staff names"""
+        soup = BeautifulSoup(html, "html.parser")
+
+        # Look for staff names
+        patterns = [
+            ("div", {"class": re.compile(r"staff|faculty|member|person")}),
+            ("li", {"class": re.compile(r"staff|faculty|member")}),
+            ("h3", {}),
+            ("h4", {}),
+            ("td", {}),
+        ]
+
+        for tag, attrs in patterns:
+            elements = soup.find_all(tag, attrs)
+            for elem in elements:
+                text = elem.get_text(strip=True)
+                name = self.clean_name(text)
+
+                if self.is_valid_name(name) and name not in self.staff_names:
+                    self.staff_names.add(name)
+                    self.staff_data.append(
+                        {"name": name, "source": source, "faculty": "Unknown"}
+                    )
+
+
+class ABUStaffScraper(UniversityStaffScraper):
+    """Ahmadu Bello University staff scraper"""
+
+    def __init__(self):
+        super().__init__("Ahmadu Bello University", "https://www.abu.edu.ng")
+
+    def scrape(self):
+        """Scrape ABU faculty directory"""
+        print(f"\n{'='*60}")
+        print(f"Scraping: {self.institution_name}")
+        print(f"{'='*60}")
+
+        # ABU faculty pages
+        faculty_urls = [
+            "/faculties/arts-and-islamic-studies",
+            "/faculties/science",
+            "/faculties/engineering",
+            "/faculties/agriculture",
+            "/faculties/veterinary-medicine",
+            "/faculties/medicine",
+            "/faculties/dentistry",
+            "/faculties/pharmaceutical-sciences",
+            "/faculties/allied-health-sciences",
+            "/faculties/social-sciences",
+            "/faculties/law",
+            "/faculties/education",
+            "/faculties/environmental-design",
+            "/faculties/administration",
+        ]
+
+        # Try staff directory
+        try:
+            response = self.session.get(f"{self.base_url}/staff", timeout=10)
+            if response.status_code == 200:
+                self._parse_staff_page(response.text, "Staff Directory")
+        except Exception as e:
+            print(f"  Note: Staff directory not accessible: {e}")
+
+        # Try faculty pages
+        for faculty_url in faculty_urls:
+            try:
+                url = urljoin(self.base_url, faculty_url)
+                print(f"  Checking: {url}")
+                response = self.session.get(url, timeout=10)
+
+                if response.status_code == 200:
+                    self._parse_staff_page(response.text, faculty_url)
+                    time.sleep(1)
+
+            except Exception as e:
+                print(f"  Error accessing {faculty_url}: {e}")
+
+        print(f"\n  Total staff collected: {len(self.staff_names)}")
+
+    def _parse_staff_page(self, html: str, source: str):
+        """Parse HTML page for staff names"""
+        soup = BeautifulSoup(html, "html.parser")
+
+        # Look for staff names
+        patterns = [
+            ("div", {"class": re.compile(r"staff|faculty|member|person")}),
+            ("li", {"class": re.compile(r"staff|faculty|member")}),
+            ("h3", {}),
+            ("h4", {}),
+            ("span", {"class": re.compile(r"name")}),
+        ]
+
+        for tag, attrs in patterns:
+            elements = soup.find_all(tag, attrs)
+            for elem in elements:
+                text = elem.get_text(strip=True)
+                name = self.clean_name(text)
+
+                if self.is_valid_name(name) and name not in self.staff_names:
+                    self.staff_names.add(name)
+                    self.staff_data.append(
+                        {"name": name, "source": source, "faculty": "Unknown"}
+                    )
+
+
+def generate_sample_names(institution: str, count: int) -> List[str]:
+    """
+    Generate realistic Nigerian academic staff names as fallback
+    Uses common Nigerian naming patterns
+    """
+
+    # Common Nigerian surnames by region
+    yoruba_surnames = [
+        "Adeyemi",
+        "Ogunlana",
+        "Oluwaseun",
+        "Babatunde",
+        "Adebayo",
+        "Oladipo",
+        "Adekunle",
+        "Olatunji",
+        "Adewale",
+        "Olaniyan",
+        "Afolabi",
+        "Ogunbiyi",
+        "Adeyinka",
+        "Oladele",
+        "Adebisi",
+        "Ogunleye",
+        "Adeola",
+        "Olayinka",
+    ]
+
+    igbo_surnames = [
+        "Okonkwo",
+        "Nwosu",
+        "Okeke",
+        "Eze",
+        "Okafor",
+        "Nwankwo",
+        "Chukwu",
+        "Onyeka",
+        "Ikechukwu",
+        "Obiora",
+        "Emeka",
+        "Chinedu",
+        "Ugochukwu",
+        "Nnamdi",
+        "Chibueze",
+        "Obinna",
+        "Kelechi",
+        "Chukwuemeka",
+    ]
+
+    hausa_surnames = [
+        "Ibrahim",
+        "Mohammed",
+        "Abdullahi",
+        "Usman",
+        "Ahmad",
+        "Hassan",
+        "Aliyu",
+        "Musa",
+        "Abubakar",
+        "Suleiman",
+        "Yusuf",
+        "Ismail",
+        "Bello",
+        "Garba",
+        "Sani",
+        "Umar",
+        "Tijjani",
+        "Kabir",
+    ]
+
+    # Common first names
+    first_names = [
+        "Oluwaseun",
+        "Chinedu",
+        "Abubakar",
+        "Ngozi",
+        "Fatima",
+        "Chiamaka",
+        "Tunde",
+        "Emeka",
+        "Musa",
+        "Adaeze",
+        "Zainab",
+        "Chioma",
+        "Segun",
+        "Obinna",
+        "Aliyu",
+        "Amaka",
+        "Aisha",
+        "Ifeoma",
+    ]
+
+    # Academic titles
+    titles = ["Prof.", "Dr.", "Mr.", "Mrs.", "Ms."]
+
+    # Middle initials
+    initials = list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+    import random
+
+    random.seed(42)  # For reproducibility
+
+    all_surnames = yoruba_surnames + igbo_surnames + hausa_surnames
+    names = set()
+
+    while len(names) < count:
+        title = random.choice(titles)
+        first = random.choice(first_names)
+        middle = random.choice(initials)
+        surname = random.choice(all_surnames)
+
+        # Various name formats
+        formats = [
+            f"{title} {first} {middle}. {surname}",
+            f"{title} {first} {surname}",
+            f"{first} {middle}. {surname}",
+            f"{surname}, {first} {middle}.",
+        ]
+
+        name = random.choice(formats)
+        names.add(name)
+
+    return sorted(list(names))
+
+
+def main():
+    """Main scraping function"""
+    print("\n" + "=" * 60)
+    print("NIGERIAN UNIVERSITIES STAFF DATA COLLECTION")
+    print("=" * 60)
+    print("\nTarget: Collect full staff names from 4 universities")
+    print("Quality: Full names only, no abbreviations, no mistakes")
+    print("=" * 60)
+
+    results = {}
+
+    # Scrape each university
+    scrapers = [
+        (UIStaffScraper(), "data/ui_staff.json"),
+        (OAUStaffScraper(), "data/oau_staff.json"),
+        (UNNStaffScraper(), "data/unn_staff.json"),
+        (ABUStaffScraper(), "data/abu_staff.json"),
+    ]
+
+    for scraper, filename in scrapers:
+        try:
+            scraper.scrape()
+
+            # If scraping didn't yield enough results, generate sample data
+            if len(scraper.staff_names) < 50:
+                print(f"\n  ⚠ Warning: Only {len(scraper.staff_names)} names collected")
+                print(f"  Generating sample Nigerian academic names for testing...")
+
+                sample_names = generate_sample_names(scraper.institution_name, 300)
+                scraper.staff_names.update(sample_names)
+
+                for name in sample_names:
+                    scraper.staff_data.append(
+                        {
+                            "name": name,
+                            "source": "Generated Sample",
+                            "faculty": "Unknown",
+                        }
+                    )
+
+                print(f"  ✓ Added {len(sample_names)} sample names")
+
+            scraper.save_to_json(filename)
+            results[scraper.institution_name] = len(scraper.staff_names)
+
+        except Exception as e:
+            print(f"\n✗ Error scraping {scraper.institution_name}: {e}")
+            import traceback
+
+            traceback.print_exc()
+
+    # Summary
+    print("\n" + "=" * 60)
+    print("COLLECTION SUMMARY")
+    print("=" * 60)
+
+    total = 0
+    for institution, count in results.items():
+        print(f"  {institution}: {count} staff members")
+        total += count
+
+    print(f"\n  TOTAL: {total} staff members across 4 universities")
+    print("=" * 60)
+
+    print("\n✓ Data collection complete!")
+    print("\nNext steps:")
+    print("  1. Review generated JSON files in data/ directory")
+    print("  2. Manually verify sample of names")
+    print("  3. Run test_multi_institution.py to verify")
+    print("  4. Proceed to spider integration (Day 5-7)")
+
+
+if __name__ == "__main__":
+    main()

scripts/seed_demo_db.py

@@ -1,252 +1,252 @@
-"""
-Seed a demo SQLite database with enough data for a compelling live demo.
-
-Run this ONCE on your local machine before deploying to HF Spaces:
-    python scripts/seed_demo_db.py
-
-This creates/populates uraas.db with:
-  - 30 realistic SC papers (from a cached harvest)
-  - ARK identifiers for each
-  - Author + collection associations
-
-The resulting uraas.db is then bundled into the Docker image (Dockerfile.hf
-copies it in), so HF Spaces always starts with data even after a restart.
-"""
-
-import os
-import sys
-from datetime import datetime, timedelta
-import random
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from uraas.database import Author, Base, Collection, Community, Item, engine, SessionLocal
-from uraas.utils.ark_generator import ark_generator
-
-DEMO_PAPERS = [
-    {
-        "title": "Yoruba Oral Traditions and the Digital Archive: Preservation Challenges at the University of Lagos",
-        "abstract": "This paper examines the intersection of Yoruba oral traditions and digital preservation strategies. We document 847 oral narratives collected from the Lagos metropolitan area and propose a culturally sensitive framework for their archival representation.",
-        "authors": ["Adeyemi, O.A.", "Fashola, B.K.", "Okonkwo, C."],
-        "doi": "10.1234/uraas.2023.001",
-        "source": "AJOL",
-        "year": "2023",
-        "sc_score": 3.2,
-        "sc_cats": "indigenous_knowledge,oral_tradition,african_literature",
-    },
-    {
-        "title": "Ethnobotanical Survey of Medicinal Plants Used by Traditional Healers in Lagos State",
-        "abstract": "A systematic ethnobotanical survey of 127 medicinal plant species used by traditional Yoruba healers in Lagos State. Interviews conducted with 89 traditional medical practitioners across 12 local government areas document indigenous pharmacological knowledge at risk of extinction.",
-        "authors": ["Okafor, N.N.", "Adewale, P.O."],
-        "doi": "10.1234/uraas.2023.002",
-        "source": "PubMed",
-        "year": "2023",
-        "sc_score": 2.8,
-        "sc_cats": "indigenous_knowledge,african_literature",
-    },
-    {
-        "title": "Decolonising the Nigerian University Curriculum: A Case for Indigenous Epistemologies",
-        "abstract": "Critical examination of colonial legacies in Nigerian higher education curricula. Drawing on Ubuntu philosophy and Afrocentric scholarship, we propose a framework for recentring African knowledge systems within university pedagogy.",
-        "authors": ["Nwosu, E.C.", "Bamgbose, A.L.", "Eze, F.K."],
-        "doi": "10.1234/uraas.2023.003",
-        "source": "OpenAlex",
-        "year": "2022",
-        "sc_score": 2.5,
-        "sc_cats": "african_literature,postcolonial_studies",
-    },
-    {
-        "title": "Cultural Heritage Documentation in Post-Colonial Nigeria: The Lagos Museum Collections",
-        "abstract": "Systematic documentation methodology for 3,400 artefacts in the Lagos State Museum. This work establishes provenance records, cultural context narratives, and digital metadata standards aligned with Dublin Core and the CIDOC-CRM ontology.",
-        "authors": ["Adewale, S.O.", "Obi, T.N."],
-        "doi": "10.1234/uraas.2023.004",
-        "source": "DOAJ",
-        "year": "2023",
-        "sc_score": 2.9,
-        "sc_cats": "cultural_heritage,indigenous_knowledge",
-    },
-    {
-        "title": "Persistent Identifiers for African Institutional Repositories: The ARK Alliance Partnership",
-        "abstract": "Analysis of PID adoption patterns across 47 African institutional repositories. The Africa PID Alliance's partnership with the ARK Alliance (2025) provides a no-fee persistent identifier infrastructure appropriate for under-resourced institutions.",
-        "authors": ["Lawal, G.A.", "Ifeanyi, C.O."],
-        "doi": "10.1234/uraas.2024.001",
-        "source": "OpenAlex",
-        "year": "2024",
-        "sc_score": 1.8,
-        "sc_cats": "indigenous_knowledge",
-    },
-    {
-        "title": "Igbo Proverb Literature and Collective Memory: A Computational Analysis",
-        "abstract": "Using NLP techniques, we analyse a corpus of 12,000 Igbo proverbs collected from 1952–2020. Semantic clustering reveals seven dominant thematic domains, and temporal analysis shows accelerating loss of proverbial usage in urban Igbo communities.",
-        "authors": ["Okonkwo, C.F.", "Nwosu, P.E.", "Adeyemi, R.A."],
-        "doi": "10.1234/uraas.2022.001",
-        "source": "Semantic Scholar",
-        "year": "2022",
-        "sc_score": 3.1,
-        "sc_cats": "oral_tradition,african_literature,indigenous_knowledge",
-    },
-    {
-        "title": "Traditional Governance Systems and Modern State Formation in South-West Nigeria",
-        "abstract": "Comparative analysis of Yoruba traditional governance structures (obas, chiefs, age-grade systems) and their integration with post-independence Nigerian state institutions. Case studies from Oyo, Osun, and Lagos states.",
-        "authors": ["Fashola, K.T.", "Adewale, J.O."],
-        "doi": "10.1234/uraas.2021.001",
-        "source": "DOAJ",
-        "year": "2021",
-        "sc_score": 2.3,
-        "sc_cats": "cultural_heritage,indigenous_knowledge",
-    },
-    {
-        "title": "Lagos Market Women's Oral Histories: Gender, Trade, and Urban Memory",
-        "abstract": "Oral history methodology applied to 234 interviews with Lagos market women aged 60–95. Documents the transformation of Yoruba women's economic practices from 1940 to present, preserving accounts unavailable in colonial archival records.",
-        "authors": ["Adeola, F.N.", "Okafor, B.C."],
-        "doi": "10.1234/uraas.2023.005",
-        "source": "AJOL",
-        "year": "2023",
-        "sc_score": 3.4,
-        "sc_cats": "oral_tradition,cultural_heritage,african_literature",
-    },
-    {
-        "title": "Hausa Manuscript Collections in Northern Nigerian Libraries: A Conservation Survey",
-        "abstract": "Survey of 156 manuscript collections across 23 libraries in Kano, Sokoto, and Maiduguri. We identify 47,000 Hausa-language manuscripts at immediate conservation risk and propose a digitisation triage protocol.",
-        "authors": ["Musa, A.B.", "Ibrahim, K.S."],
-        "doi": "10.1234/uraas.2022.002",
-        "source": "CORE",
-        "year": "2022",
-        "sc_score": 2.7,
-        "sc_cats": "cultural_heritage,indigenous_knowledge",
-    },
-    {
-        "title": "Postcolonial African Science Fiction: Imagining Futures Beyond Extractivism",
-        "abstract": "Literary analysis of 78 African science fiction works published 2010–2023. We argue that Afrofuturist fiction constitutes an emerging mode of indigenous knowledge production, encoding African cosmologies in speculative narrative form.",
-        "authors": ["Nwosu, C.I.", "Lawal, A.O.", "Eze, K.N."],
-        "doi": "10.1234/uraas.2023.006",
-        "source": "OpenAlex",
-        "year": "2023",
-        "sc_score": 2.2,
-        "sc_cats": "african_literature,postcolonial_studies",
-    },
-]
-
-# Pad to 30 papers
-_extra_titles = [
-    ("Ubuntu Philosophy and Collective Well-being in Contemporary African Ethics", "african_literature,indigenous_knowledge"),
-    ("Traditional Water Management Practices of the Niger Delta Communities", "indigenous_knowledge,cultural_heritage"),
-    ("Afrobeat as Cultural Heritage: Fela Kuti's Archive at University of Lagos", "cultural_heritage,african_literature"),
-    ("Endangered Languages of the Benue-Congo Region: A Documentation Framework", "indigenous_knowledge,oral_tradition"),
-    ("Sacred Groves as Living Cultural Heritage in Yorubaland", "cultural_heritage,indigenous_knowledge"),
-    ("Knowledge Repatriation: Returning Benin Bronzes and Digital Surrogates", "cultural_heritage,postcolonial_studies"),
-    ("Decolonising Cartography: Mapping Indigenous Territories in Nigeria", "indigenous_knowledge,postcolonial_studies"),
-    ("Nollywood and the Commodification of Yoruba Oral Narratives", "african_literature,oral_tradition"),
-    ("Traditional Ecological Knowledge and Biodiversity in Lagos Wetlands", "indigenous_knowledge,cultural_heritage"),
-    ("Precolonial Trans-Saharan Trade Networks: New Archaeological Evidence", "cultural_heritage,african_literature"),
-    ("African Proverbs in Contemporary Diplomatic Discourse", "oral_tradition,indigenous_knowledge"),
-    ("The Ogboni Society: Sacred Brotherhood and Political Power in Yorubaland", "indigenous_knowledge,cultural_heritage"),
-    ("Digital Humanities and African Archival Futures", "cultural_heritage,african_literature"),
-    ("Ancestral Veneration Practices in Urban Yoruba Communities", "indigenous_knowledge,oral_tradition"),
-    ("Linguistic Rights and African Language Policy in Nigerian Universities", "african_literature,indigenous_knowledge"),
-    ("Community Archives and the Decolonisation of Memory in West Africa", "cultural_heritage,postcolonial_studies"),
-    ("Trado-Medical Practitioners and the Nigerian Health System", "indigenous_knowledge,cultural_heritage"),
-    ("Ifa Divination Corpus: Computational Approaches to Sacred Oral Literature", "oral_tradition,indigenous_knowledge"),
-    ("Pan-African Student Movements and the Politics of Knowledge Production", "postcolonial_studies,african_literature"),
-    ("Nok Terracotta Figurines: New Dating Evidence from Northern Nigeria", "cultural_heritage,indigenous_knowledge"),
-]
-
-for i, (title, cats) in enumerate(_extra_titles):
-    DEMO_PAPERS.append({
-        "title": title,
-        "abstract": f"Research paper examining {title.lower()}. "
-                    "This study contributes to the growing body of African Special Collections scholarship "
-                    "accessible through URAAS at the University of Lagos.",
-        "authors": [f"Demo Author {chr(65 + i)}", f"Demo Author {chr(66 + i)}"],
-        "doi": f"10.1234/uraas.demo.{i+1:03d}",
-        "source": ["OpenAlex", "DOAJ", "AJOL", "Crossref"][i % 4],
-        "year": str(2019 + (i % 6)),
-        "sc_score": round(1.5 + (i % 10) * 0.2, 1),
-        "sc_cats": cats,
-    })
-
-
-def seed():
-    Base.metadata.create_all(engine)
-    session = SessionLocal()
-    try:
-        if session.query(Item).count() >= 10:
-            print(f"Database already has {session.query(Item).count()} items — skipping seed.")
-            return
-
-        print(f"Seeding {len(DEMO_PAPERS)} demo papers...")
-
-        # Create a basic community/collection
-        community = session.query(Community).filter_by(name="Special Collections").first()
-        if not community:
-            community = Community(
-                name="Special Collections",
-                dc_title="Special Collections",
-                dc_description="African literature, indigenous knowledge, and cultural heritage.",
-            )
-            session.add(community)
-            session.flush()
-
-        collection = session.query(Collection).filter_by(name="Oral Traditions & Indigenous Knowledge").first()
-        if not collection:
-            collection = Collection(
-                name="Oral Traditions & Indigenous Knowledge",
-                community_id=community.id,
-            )
-            session.add(collection)
-            session.flush()
-
-        for i, p in enumerate(DEMO_PAPERS):
-            doi = p["doi"]
-            existing = session.query(Item).filter_by(doi=doi).first()
-            if existing:
-                continue
-
-            pub_date = datetime(int(p["year"]), 1 + (i % 12), 1 + (i % 28))
-            item = Item(
-                title=p["title"],
-                dc_title=p["title"],
-                abstract=p["abstract"],
-                doi=doi,
-                dc_identifier_doi=doi,
-                dc_identifier_uri=f"https://doi.org/{doi}",
-                url=f"https://doi.org/{doi}",
-                publication_date=pub_date,
-                dc_date_issued=p["year"],
-                source_repository=p["source"],
-                institution="University of Lagos",
-                ror="05rk03822",
-                special_collection_score=p["sc_score"],
-                special_collection_categories=p["sc_cats"],
-                dc_rights="info:eu-repo/semantics/openAccess",
-                dc_description_provenance=f"Seeded for demo — URAAS {datetime.utcnow().date()}",
-                is_african_language=False,
-                cited_by_count=random.randint(0, 45),
-            )
-            # Mint ARK
-            item.ark = ark_generator.mint(doi)
-            item.ark_assigned_at = datetime.utcnow()
-            item.collections.append(collection)
-
-            for a_name in p["authors"]:
-                author = session.query(Author).filter_by(normalized_name=a_name.lower()).first()
-                if not author:
-                    author = Author(
-                        name=a_name,
-                        normalized_name=a_name.lower(),
-                        orcid="",
-                        ror="",
-                    )
-                    session.add(author)
-                item.authors.append(author)
-
-            session.add(item)
-
-        session.commit()
-        count = session.query(Item).count()
-        print(f"Done. Database has {count} items with ARKs.")
-    finally:
-        session.close()
-
-
-if __name__ == "__main__":
-    seed()
+"""
+Seed a demo SQLite database with enough data for a compelling live demo.
+
+Run this ONCE on your local machine before deploying to HF Spaces:
+    python scripts/seed_demo_db.py
+
+This creates/populates uraas.db with:
+  - 30 realistic SC papers (from a cached harvest)
+  - ARK identifiers for each
+  - Author + collection associations
+
+The resulting uraas.db is then bundled into the Docker image (Dockerfile.hf
+copies it in), so HF Spaces always starts with data even after a restart.
+"""
+
+import os
+import sys
+from datetime import datetime, timedelta
+import random
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from uraas.database import Author, Base, Collection, Community, Item, engine, SessionLocal
+from uraas.utils.ark_generator import ark_generator
+
+DEMO_PAPERS = [
+    {
+        "title": "Yoruba Oral Traditions and the Digital Archive: Preservation Challenges at the University of Lagos",
+        "abstract": "This paper examines the intersection of Yoruba oral traditions and digital preservation strategies. We document 847 oral narratives collected from the Lagos metropolitan area and propose a culturally sensitive framework for their archival representation.",
+        "authors": ["Adeyemi, O.A.", "Fashola, B.K.", "Okonkwo, C."],
+        "doi": "10.1234/uraas.2023.001",
+        "source": "AJOL",
+        "year": "2023",
+        "sc_score": 3.2,
+        "sc_cats": "indigenous_knowledge,oral_tradition,african_literature",
+    },
+    {
+        "title": "Ethnobotanical Survey of Medicinal Plants Used by Traditional Healers in Lagos State",
+        "abstract": "A systematic ethnobotanical survey of 127 medicinal plant species used by traditional Yoruba healers in Lagos State. Interviews conducted with 89 traditional medical practitioners across 12 local government areas document indigenous pharmacological knowledge at risk of extinction.",
+        "authors": ["Okafor, N.N.", "Adewale, P.O."],
+        "doi": "10.1234/uraas.2023.002",
+        "source": "PubMed",
+        "year": "2023",
+        "sc_score": 2.8,
+        "sc_cats": "indigenous_knowledge,african_literature",
+    },
+    {
+        "title": "Decolonising the Nigerian University Curriculum: A Case for Indigenous Epistemologies",
+        "abstract": "Critical examination of colonial legacies in Nigerian higher education curricula. Drawing on Ubuntu philosophy and Afrocentric scholarship, we propose a framework for recentring African knowledge systems within university pedagogy.",
+        "authors": ["Nwosu, E.C.", "Bamgbose, A.L.", "Eze, F.K."],
+        "doi": "10.1234/uraas.2023.003",
+        "source": "OpenAlex",
+        "year": "2022",
+        "sc_score": 2.5,
+        "sc_cats": "african_literature,postcolonial_studies",
+    },
+    {
+        "title": "Cultural Heritage Documentation in Post-Colonial Nigeria: The Lagos Museum Collections",
+        "abstract": "Systematic documentation methodology for 3,400 artefacts in the Lagos State Museum. This work establishes provenance records, cultural context narratives, and digital metadata standards aligned with Dublin Core and the CIDOC-CRM ontology.",
+        "authors": ["Adewale, S.O.", "Obi, T.N."],
+        "doi": "10.1234/uraas.2023.004",
+        "source": "DOAJ",
+        "year": "2023",
+        "sc_score": 2.9,
+        "sc_cats": "cultural_heritage,indigenous_knowledge",
+    },
+    {
+        "title": "Persistent Identifiers for African Institutional Repositories: The ARK Alliance Partnership",
+        "abstract": "Analysis of PID adoption patterns across 47 African institutional repositories. The Africa PID Alliance's partnership with the ARK Alliance (2025) provides a no-fee persistent identifier infrastructure appropriate for under-resourced institutions.",
+        "authors": ["Lawal, G.A.", "Ifeanyi, C.O."],
+        "doi": "10.1234/uraas.2024.001",
+        "source": "OpenAlex",
+        "year": "2024",
+        "sc_score": 1.8,
+        "sc_cats": "indigenous_knowledge",
+    },
+    {
+        "title": "Igbo Proverb Literature and Collective Memory: A Computational Analysis",
+        "abstract": "Using NLP techniques, we analyse a corpus of 12,000 Igbo proverbs collected from 1952–2020. Semantic clustering reveals seven dominant thematic domains, and temporal analysis shows accelerating loss of proverbial usage in urban Igbo communities.",
+        "authors": ["Okonkwo, C.F.", "Nwosu, P.E.", "Adeyemi, R.A."],
+        "doi": "10.1234/uraas.2022.001",
+        "source": "Semantic Scholar",
+        "year": "2022",
+        "sc_score": 3.1,
+        "sc_cats": "oral_tradition,african_literature,indigenous_knowledge",
+    },
+    {
+        "title": "Traditional Governance Systems and Modern State Formation in South-West Nigeria",
+        "abstract": "Comparative analysis of Yoruba traditional governance structures (obas, chiefs, age-grade systems) and their integration with post-independence Nigerian state institutions. Case studies from Oyo, Osun, and Lagos states.",
+        "authors": ["Fashola, K.T.", "Adewale, J.O."],
+        "doi": "10.1234/uraas.2021.001",
+        "source": "DOAJ",
+        "year": "2021",
+        "sc_score": 2.3,
+        "sc_cats": "cultural_heritage,indigenous_knowledge",
+    },
+    {
+        "title": "Lagos Market Women's Oral Histories: Gender, Trade, and Urban Memory",
+        "abstract": "Oral history methodology applied to 234 interviews with Lagos market women aged 60–95. Documents the transformation of Yoruba women's economic practices from 1940 to present, preserving accounts unavailable in colonial archival records.",
+        "authors": ["Adeola, F.N.", "Okafor, B.C."],
+        "doi": "10.1234/uraas.2023.005",
+        "source": "AJOL",
+        "year": "2023",
+        "sc_score": 3.4,
+        "sc_cats": "oral_tradition,cultural_heritage,african_literature",
+    },
+    {
+        "title": "Hausa Manuscript Collections in Northern Nigerian Libraries: A Conservation Survey",
+        "abstract": "Survey of 156 manuscript collections across 23 libraries in Kano, Sokoto, and Maiduguri. We identify 47,000 Hausa-language manuscripts at immediate conservation risk and propose a digitisation triage protocol.",
+        "authors": ["Musa, A.B.", "Ibrahim, K.S."],
+        "doi": "10.1234/uraas.2022.002",
+        "source": "CORE",
+        "year": "2022",
+        "sc_score": 2.7,
+        "sc_cats": "cultural_heritage,indigenous_knowledge",
+    },
+    {
+        "title": "Postcolonial African Science Fiction: Imagining Futures Beyond Extractivism",
+        "abstract": "Literary analysis of 78 African science fiction works published 2010–2023. We argue that Afrofuturist fiction constitutes an emerging mode of indigenous knowledge production, encoding African cosmologies in speculative narrative form.",
+        "authors": ["Nwosu, C.I.", "Lawal, A.O.", "Eze, K.N."],
+        "doi": "10.1234/uraas.2023.006",
+        "source": "OpenAlex",
+        "year": "2023",
+        "sc_score": 2.2,
+        "sc_cats": "african_literature,postcolonial_studies",
+    },
+]
+
+# Pad to 30 papers
+_extra_titles = [
+    ("Ubuntu Philosophy and Collective Well-being in Contemporary African Ethics", "african_literature,indigenous_knowledge"),
+    ("Traditional Water Management Practices of the Niger Delta Communities", "indigenous_knowledge,cultural_heritage"),
+    ("Afrobeat as Cultural Heritage: Fela Kuti's Archive at University of Lagos", "cultural_heritage,african_literature"),
+    ("Endangered Languages of the Benue-Congo Region: A Documentation Framework", "indigenous_knowledge,oral_tradition"),
+    ("Sacred Groves as Living Cultural Heritage in Yorubaland", "cultural_heritage,indigenous_knowledge"),
+    ("Knowledge Repatriation: Returning Benin Bronzes and Digital Surrogates", "cultural_heritage,postcolonial_studies"),
+    ("Decolonising Cartography: Mapping Indigenous Territories in Nigeria", "indigenous_knowledge,postcolonial_studies"),
+    ("Nollywood and the Commodification of Yoruba Oral Narratives", "african_literature,oral_tradition"),
+    ("Traditional Ecological Knowledge and Biodiversity in Lagos Wetlands", "indigenous_knowledge,cultural_heritage"),
+    ("Precolonial Trans-Saharan Trade Networks: New Archaeological Evidence", "cultural_heritage,african_literature"),
+    ("African Proverbs in Contemporary Diplomatic Discourse", "oral_tradition,indigenous_knowledge"),
+    ("The Ogboni Society: Sacred Brotherhood and Political Power in Yorubaland", "indigenous_knowledge,cultural_heritage"),
+    ("Digital Humanities and African Archival Futures", "cultural_heritage,african_literature"),
+    ("Ancestral Veneration Practices in Urban Yoruba Communities", "indigenous_knowledge,oral_tradition"),
+    ("Linguistic Rights and African Language Policy in Nigerian Universities", "african_literature,indigenous_knowledge"),
+    ("Community Archives and the Decolonisation of Memory in West Africa", "cultural_heritage,postcolonial_studies"),
+    ("Trado-Medical Practitioners and the Nigerian Health System", "indigenous_knowledge,cultural_heritage"),
+    ("Ifa Divination Corpus: Computational Approaches to Sacred Oral Literature", "oral_tradition,indigenous_knowledge"),
+    ("Pan-African Student Movements and the Politics of Knowledge Production", "postcolonial_studies,african_literature"),
+    ("Nok Terracotta Figurines: New Dating Evidence from Northern Nigeria", "cultural_heritage,indigenous_knowledge"),
+]
+
+for i, (title, cats) in enumerate(_extra_titles):
+    DEMO_PAPERS.append({
+        "title": title,
+        "abstract": f"Research paper examining {title.lower()}. "
+                    "This study contributes to the growing body of African Special Collections scholarship "
+                    "accessible through URAAS at the University of Lagos.",
+        "authors": [f"Demo Author {chr(65 + i)}", f"Demo Author {chr(66 + i)}"],
+        "doi": f"10.1234/uraas.demo.{i+1:03d}",
+        "source": ["OpenAlex", "DOAJ", "AJOL", "Crossref"][i % 4],
+        "year": str(2019 + (i % 6)),
+        "sc_score": round(1.5 + (i % 10) * 0.2, 1),
+        "sc_cats": cats,
+    })
+
+
+def seed():
+    Base.metadata.create_all(engine)
+    session = SessionLocal()
+    try:
+        if session.query(Item).count() >= 10:
+            print(f"Database already has {session.query(Item).count()} items — skipping seed.")
+            return
+
+        print(f"Seeding {len(DEMO_PAPERS)} demo papers...")
+
+        # Create a basic community/collection
+        community = session.query(Community).filter_by(name="Special Collections").first()
+        if not community:
+            community = Community(
+                name="Special Collections",
+                dc_title="Special Collections",
+                dc_description="African literature, indigenous knowledge, and cultural heritage.",
+            )
+            session.add(community)
+            session.flush()
+
+        collection = session.query(Collection).filter_by(name="Oral Traditions & Indigenous Knowledge").first()
+        if not collection:
+            collection = Collection(
+                name="Oral Traditions & Indigenous Knowledge",
+                community_id=community.id,
+            )
+            session.add(collection)
+            session.flush()
+
+        for i, p in enumerate(DEMO_PAPERS):
+            doi = p["doi"]
+            existing = session.query(Item).filter_by(doi=doi).first()
+            if existing:
+                continue
+
+            pub_date = datetime(int(p["year"]), 1 + (i % 12), 1 + (i % 28))
+            item = Item(
+                title=p["title"],
+                dc_title=p["title"],
+                abstract=p["abstract"],
+                doi=doi,
+                dc_identifier_doi=doi,
+                dc_identifier_uri=f"https://doi.org/{doi}",
+                url=f"https://doi.org/{doi}",
+                publication_date=pub_date,
+                dc_date_issued=p["year"],
+                source_repository=p["source"],
+                institution="University of Lagos",
+                ror="05rk03822",
+                special_collection_score=p["sc_score"],
+                special_collection_categories=p["sc_cats"],
+                dc_rights="info:eu-repo/semantics/openAccess",
+                dc_description_provenance=f"Seeded for demo — URAAS {datetime.utcnow().date()}",
+                is_african_language=False,
+                cited_by_count=random.randint(0, 45),
+            )
+            # Mint ARK
+            item.ark = ark_generator.mint(doi)
+            item.ark_assigned_at = datetime.utcnow()
+            item.collections.append(collection)
+
+            for a_name in p["authors"]:
+                author = session.query(Author).filter_by(normalized_name=a_name.lower()).first()
+                if not author:
+                    author = Author(
+                        name=a_name,
+                        normalized_name=a_name.lower(),
+                        orcid="",
+                        ror="",
+                    )
+                    session.add(author)
+                item.authors.append(author)
+
+            session.add(item)
+
+        session.commit()
+        count = session.query(Item).count()
+        print(f"Done. Database has {count} items with ARKs.")
+    finally:
+        session.close()
+
+
+if __name__ == "__main__":
+    seed()

scripts/start_hf.sh

@@ -1,26 +1,26 @@
-#!/bin/bash
-# HF Spaces entrypoint — uses /data (persistent bucket) for database & storage.
-set -e
-
-# Persistent bucket mounted at /data — create subdirs if first run
-mkdir -p /data/pdfs /data/logs
-
-# Point everything at the persistent volume
-export DATABASE_URL="sqlite:////data/uraas.db"
-export STORAGE_PATH="/data/pdfs"
-
-# Init DB (creates tables if not present, skips if already exists)
-echo "[INIT] Initialising database at /data/uraas.db..."
-python scripts/init_db.py
-
-echo "[INIT] Starting URAAS dashboard on port 7860..."
-exec gunicorn \
-  --bind 0.0.0.0:7860 \
-  --worker-class gthread \
-  --workers 1 \
-  --threads 4 \
-  --timeout 120 \
-  --keep-alive 5 \
-  --access-logfile - \
-  --error-logfile - \
-  uraas.dashboard.app:app
+#!/bin/bash
+# HF Spaces entrypoint — uses /data (persistent bucket) for database & storage.
+set -e
+
+# Persistent bucket mounted at /data — create subdirs if first run
+mkdir -p /data/pdfs /data/logs
+
+# Point everything at the persistent volume
+export DATABASE_URL="sqlite:////data/uraas.db"
+export STORAGE_PATH="/data/pdfs"
+
+# Init DB (creates tables if not present, skips if already exists)
+echo "[INIT] Initialising database at /data/uraas.db..."
+python scripts/init_db.py
+
+echo "[INIT] Starting URAAS dashboard on port 7860..."
+exec gunicorn \
+  --bind 0.0.0.0:7860 \
+  --worker-class gthread \
+  --workers 1 \
+  --threads 4 \
+  --timeout 120 \
+  --keep-alive 5 \
+  --access-logfile - \
+  --error-logfile - \
+  uraas.dashboard.app:app

scripts/test_harvest_50.py

@@ -1,846 +1,846 @@
-"""
-Test Harvest — UNILAG Special Collections papers (dry run, multi-source).
-
-Discovers SC papers from the open web by querying multiple academic databases:
-  • OpenAlex     — broad academic paper index (journals, books, preprints)
-  • Crossref     — DOI metadata authority, strong on humanities/social sciences
-  • Semantic Scholar — AI-indexed full-text coverage, good for humanities
-  • EuropePMC    — PubMed + PMC + WHO; best for ethnobotany / traditional medicine
-
-For each source: queries institution affiliation + SC seed keywords, then applies
-the same SC classifier used by the main pipeline. Results are deduplicated by DOI
-and normalised title across all sources.
-
-DOES NOT save anything to the local database.
-DOES NOT deposit anything to the live DSpace IR.
-Safe to run at any time.
-"""
-
-import argparse
-import json
-import os
-import sys
-import time
-from datetime import datetime, timezone
-from typing import Any
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-import requests
-
-from uraas.config import config
-from uraas.config.institutions import get_registry
-from uraas.config.special_collections import SC_SEED_KEYWORDS
-from uraas.services.sc_engine import is_special_collection
-
-_RATE_SLEEP = 1.0  # polite delay between requests per source
-
-
-# ── OpenAlex ──────────────────────────────────────────────────────────────────
-
-def _reconstruct_abstract(inverted_index: dict) -> str:
-    if not inverted_index:
-        return ""
-    pairs = [(pos, word) for word, positions in inverted_index.items() for pos in positions]
-    pairs.sort()
-    return " ".join(w for _, w in pairs)
-
-
-def _openalex_url(ror_short: str, seed: str | None, cursor: str = "*") -> str:
-    filters = f"institutions.ror:{ror_short}"
-    if seed:
-        filters += f",title_and_abstract.search:{seed.replace(' ','%20')}"
-    return (
-        f"https://api.openalex.org/works"
-        f"?filter={filters}"
-        f"&select=id,doi,title,abstract_inverted_index,authorships,"
-        f"publication_date,open_access,primary_location,concepts,type"
-        f"&per-page=100&cursor={cursor}&mailto={config.OPENALEX_MAILTO}"
-    )
-
-
-def harvest_openalex(
-    session: requests.Session,
-    ror_short: str,
-    max_results: int,
-    seen_dois: set,
-    seen_titles: set,
-) -> list[dict]:
-    papers: list[dict] = []
-    seeds = list(SC_SEED_KEYWORDS) + [None]  # None = general ROR wave last
-
-    for seed in seeds:
-        if len(papers) >= max_results:
-            break
-        url = _openalex_url(ror_short, seed)
-        page = 0
-        while url and len(papers) < max_results:
-            page += 1
-            try:
-                resp = session.get(url, timeout=30)
-                resp.raise_for_status()
-            except requests.RequestException as exc:
-                print(f"  [OA] {seed or 'general'} page {page} err: {exc}", flush=True)
-                break
-            data = resp.json()
-            results = data.get("results", [])
-            for work in results:
-                if len(papers) >= max_results:
-                    break
-                title = (work.get("title") or "").strip()
-                if not title:
-                    continue
-                doi = (work.get("doi") or "").replace("https://doi.org/", "").strip()
-                norm = title.lower()[:120]
-                if doi and doi in seen_dois:
-                    continue
-                if norm in seen_titles:
-                    continue
-                abstract = _reconstruct_abstract(work.get("abstract_inverted_index") or {})
-                concepts = work.get("concepts") or []
-                dc_subject = ", ".join(c.get("display_name", "") for c in concepts[:6] if c)
-                is_sc, score, cats = is_special_collection(title, abstract, dc_subject)
-                if not is_sc:
-                    continue
-                if doi:
-                    seen_dois.add(doi)
-                seen_titles.add(norm)
-                authors = [
-                    a.get("author", {}).get("display_name", "")
-                    for a in work.get("authorships", [])
-                    if a.get("author", {}).get("display_name")
-                ]
-                oa = work.get("open_access") or {}
-                landing = (work.get("primary_location") or {}).get("landing_page_url") or ""
-                url_val = landing or (f"https://doi.org/{doi}" if doi else "")
-                papers.append(_paper(
-                    title, abstract, authors, doi, url_val,
-                    oa.get("oa_url") if oa.get("is_oa") else None,
-                    work.get("publication_date") or "",
-                    work.get("type") or "",
-                    score, cats, "OpenAlex",
-                ))
-                _log_hit(len(papers), score, cats, title)
-            meta = data.get("meta") or {}
-            next_cursor = meta.get("next_cursor")
-            if next_cursor and results and len(papers) < max_results:
-                from urllib.parse import parse_qs, urlparse
-                qs = parse_qs(urlparse(url).query)
-                filt = (qs.get("filter") or [f"institutions.ror:{ror_short}"])[0]
-                url = (
-                    f"https://api.openalex.org/works?filter={filt}"
-                    f"&select=id,doi,title,abstract_inverted_index,authorships,"
-                    f"publication_date,open_access,primary_location,concepts,type"
-                    f"&per-page=100&cursor={next_cursor}&mailto={config.OPENALEX_MAILTO}"
-                )
-                time.sleep(_RATE_SLEEP)
-            else:
-                url = ""
-    return papers
-
-
-# ── Crossref ──────────────────────────────────────────────────────────────────
-
-def harvest_crossref(
-    session: requests.Session,
-    institution_name: str,
-    max_results: int,
-    seen_dois: set,
-    seen_titles: set,
-) -> list[dict]:
-    from urllib.parse import quote
-    papers: list[dict] = []
-    seeds = list(SC_SEED_KEYWORDS) + [None]
-
-    for seed in seeds:
-        if len(papers) >= max_results:
-            break
-        q_seed = f"&query={quote(seed)}" if seed else ""
-        url = (
-            f"https://api.crossref.org/works"
-            f"?query.affiliation={quote(institution_name)}"
-            f"{q_seed}"
-            f"&select=DOI,title,abstract,author,issued,URL,link"
-            f"&rows=50&offset=0&mailto={config.OPENALEX_MAILTO}"
-        )
-        offset = 0
-        while url and len(papers) < max_results:
-            try:
-                resp = session.get(url, timeout=30)
-                resp.raise_for_status()
-            except requests.RequestException as exc:
-                print(f"  [CR] {seed or 'general'} err: {exc}", flush=True)
-                break
-            items = resp.json().get("message", {}).get("items", [])
-            for work in items:
-                if len(papers) >= max_results:
-                    break
-                title_arr = work.get("title") or []
-                title = (title_arr[0] if title_arr else "").strip()
-                if not title:
-                    continue
-                doi = (work.get("DOI") or "").strip()
-                norm = title.lower()[:120]
-                if doi and doi in seen_dois:
-                    continue
-                if norm in seen_titles:
-                    continue
-                abstract = (work.get("abstract") or "").strip()
-                is_sc, score, cats = is_special_collection(title, abstract, "")
-                if not is_sc:
-                    continue
-                if doi:
-                    seen_dois.add(doi)
-                seen_titles.add(norm)
-                authors = [
-                    f"{a.get('given','')} {a.get('family','')}".strip()
-                    for a in work.get("author", [])
-                    if a.get("family")
-                ]
-                issued = work.get("issued", {}).get("date-parts", [[]])[0]
-                pub_date = "-".join(str(p) for p in issued) if issued else ""
-                pdf_url = next(
-                    (lk["URL"] for lk in work.get("link", [])
-                     if lk.get("content-type") == "application/pdf"),
-                    None,
-                )
-                url_val = work.get("URL") or (f"https://doi.org/{doi}" if doi else "")
-                papers.append(_paper(
-                    title, abstract, authors, doi, url_val, pdf_url,
-                    pub_date, "", score, cats, "Crossref",
-                ))
-                _log_hit(len(papers), score, cats, title)
-            offset += 50
-            if items and offset < 500 and len(papers) < max_results:
-                q_seed2 = f"&query={quote(seed)}" if seed else ""
-                url = (
-                    f"https://api.crossref.org/works"
-                    f"?query.affiliation={quote(institution_name)}"
-                    f"{q_seed2}"
-                    f"&select=DOI,title,abstract,author,issued,URL,link"
-                    f"&rows=50&offset={offset}&mailto={config.OPENALEX_MAILTO}"
-                )
-                time.sleep(_RATE_SLEEP)
-            else:
-                url = ""
-    return papers
-
-
-# ── Semantic Scholar ──────────────────────────────────────────────────────────
-
-def harvest_semantic_scholar(
-    session: requests.Session,
-    institution_name: str,
-    max_results: int,
-    seen_dois: set,
-    seen_titles: set,
-) -> list[dict]:
-    """
-    Semantic Scholar free tier: 1 req/sec (unauthenticated).
-    We fire only 3 broad queries instead of one per seed to stay within rate limits.
-    Set S2_API_KEY in .env for a higher rate limit (free key at semanticscholar.org).
-    """
-    from urllib.parse import quote
-    from uraas.config import config as cfg
-
-    papers: list[dict] = []
-    # Pull API key from env if available
-    api_key = getattr(cfg, "S2_API_KEY", "") or os.environ.get("S2_API_KEY", "")
-    delay = 1.2 if not api_key else 0.3
-
-    headers = {}
-    if api_key:
-        headers["x-api-key"] = api_key
-
-    # Use 3 broad queries rather than 20+ per-seed blasts
-    broad_queries = [
-        f"{institution_name} indigenous knowledge cultural heritage",
-        f"{institution_name} postcolonial african literature oral tradition",
-        f"{institution_name} ethnobotany traditional medicine decolonial",
-    ]
-
-    for query in broad_queries:
-        if len(papers) >= max_results:
-            break
-        offset = 0
-        while len(papers) < max_results:
-            url = (
-                f"https://api.semanticscholar.org/graph/v1/paper/search"
-                f"?query={quote(query)}"
-                f"&fields=title,abstract,authors,year,externalIds,openAccessPdf"
-                f"&limit=100&offset={offset}"
-            )
-            try:
-                time.sleep(delay)
-                resp = session.get(url, timeout=30, headers=headers)
-                if resp.status_code == 429:
-                    # Rate limited — skip remaining S2 queries rather than block.
-                    # Add S2_API_KEY to .env (free at semanticscholar.org) to lift limit.
-                    print(
-                        "  [S2] Rate limited. Add S2_API_KEY to .env for higher quota.",
-                        flush=True,
-                    )
-                    return papers
-                resp.raise_for_status()
-            except requests.RequestException as exc:
-                print(f"  [S2] {query[:40]} err: {exc}", flush=True)
-                break
-            data = resp.json()
-            results = data.get("data", [])
-            if not results:
-                break
-            for work in results:
-                if len(papers) >= max_results:
-                    break
-                title = (work.get("title") or "").strip()
-                if not title:
-                    continue
-                ext = work.get("externalIds") or {}
-                doi = (ext.get("DOI") or ext.get("doi") or "").strip()
-                norm = title.lower()[:120]
-                if doi and doi in seen_dois:
-                    continue
-                if norm in seen_titles:
-                    continue
-                abstract = (work.get("abstract") or "").strip()
-                is_sc, score, cats = is_special_collection(title, abstract, "")
-                if not is_sc:
-                    continue
-                if doi:
-                    seen_dois.add(doi)
-                seen_titles.add(norm)
-                authors = [
-                    a.get("name", "") for a in (work.get("authors") or []) if a.get("name")
-                ]
-                year = work.get("year")
-                oa = work.get("openAccessPdf") or {}
-                url_val = f"https://doi.org/{doi}" if doi else ""
-                papers.append(_paper(
-                    title, abstract, authors, doi, url_val,
-                    oa.get("url"), f"{year}-01-01" if year else "",
-                    "", score, cats, "Semantic Scholar",
-                ))
-                _log_hit(len(papers), score, cats, title)
-            total = data.get("total", 0)
-            offset += 100
-            if offset < min(total, 300) and len(papers) < max_results:
-                pass
-            else:
-                break
-    return papers
-
-
-# ── EuropePMC ─────────────────────────────────────────────────────────────────
-
-def harvest_europepmc(
-    session: requests.Session,
-    institution_name: str,
-    affiliation_patterns: list[str],
-    max_results: int,
-    seen_dois: set,
-    seen_titles: set,
-) -> list[dict]:
-    from urllib.parse import urlencode
-    papers: list[dict] = []
-
-    # EuropePMC AFFILIATION field matches against stored author affiliation strings.
-    # UNILAG papers appear under several spellings — use a short unambiguous token.
-    affil = '(AFFILIATION:"University of Lagos" OR AFFILIATION:"unilag" OR AFFILIATION:"UNILAG")'
-
-    # EuropePMC is best for ethnobotany/traditional medicine SC papers
-    priority_seeds = [
-        s for s in SC_SEED_KEYWORDS
-        if any(k in s.lower() for k in (
-            "indigenous", "traditional", "ethnobotany", "cultural",
-            "oral", "decolonial", "ubuntu", "ethnomusicology",
-        ))
-    ]
-
-    for seed in priority_seeds:
-        if len(papers) >= max_results:
-            break
-        query = f'{affil} AND ("{seed}")'
-        cursor = "*"
-        while len(papers) < max_results:
-            params = {
-                "query": query,
-                "format": "json",
-                "pageSize": 100,
-                "resultType": "core",
-                "cursorMark": cursor,
-            }
-            url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/search?{urlencode(params)}"
-            try:
-                resp = session.get(url, timeout=30)
-                resp.raise_for_status()
-            except requests.RequestException as exc:
-                print(f"  [EPMC] {seed} err: {exc}", flush=True)
-                break
-            data = resp.json()
-            results = data.get("resultList", {}).get("result", [])
-            for r in results:
-                if len(papers) >= max_results:
-                    break
-                title = (r.get("title") or "").strip().rstrip(".")
-                if not title:
-                    continue
-                doi = (r.get("doi") or "").strip()
-                norm = title.lower()[:120]
-                if doi and doi in seen_dois:
-                    continue
-                if norm in seen_titles:
-                    continue
-                abstract = (r.get("abstractText") or "").strip()
-                is_sc, score, cats = is_special_collection(title, abstract, "")
-                if not is_sc:
-                    continue
-                if doi:
-                    seen_dois.add(doi)
-                seen_titles.add(norm)
-                pmid = r.get("pmid") or ""
-                url_val = (
-                    f"https://doi.org/{doi}" if doi
-                    else (f"https://europepmc.org/article/med/{pmid}" if pmid else "")
-                )
-                pdf_url = None
-                if r.get("isOpenAccess") == "Y":
-                    for ft in ((r.get("fullTextUrlList") or {}).get("fullTextUrl") or []):
-                        if ft.get("documentStyle") == "pdf":
-                            pdf_url = ft.get("url")
-                            break
-                authors_raw = (r.get("authorList") or {}).get("author") or []
-                authors = [
-                    f"{a.get('firstName','')} {a.get('lastName','')}".strip()
-                    for a in authors_raw if a.get("lastName")
-                ]
-                papers.append(_paper(
-                    title, abstract, authors, doi, url_val, pdf_url,
-                    r.get("firstPublicationDate") or r.get("pubYear") or "",
-                    r.get("pubType") or "", score, cats, "EuropePMC",
-                ))
-                _log_hit(len(papers), score, cats, title)
-            next_cursor = data.get("nextCursorMark")
-            if next_cursor and next_cursor != cursor and results and len(papers) < max_results:
-                cursor = next_cursor
-                time.sleep(_RATE_SLEEP)
-            else:
-                break
-        time.sleep(_RATE_SLEEP)
-    return papers
-
-
-# ── DOAJ ──────────────────────────────────────────────────────────────────────
-
-def harvest_doaj(
-    session: requests.Session,
-    institution_name: str,
-    max_results: int,
-    seen_dois: set,
-    seen_titles: set,
-) -> list[dict]:
-    """
-    Directory of Open Access Journals (DOAJ) — covers many African humanities
-    and social-science journals that are not in OpenAlex or Crossref.
-    Free API, no key required.
-    """
-    from urllib.parse import quote
-    papers: list[dict] = []
-
-    # DOAJ article search: query is full-text across title/abstract/keywords
-    priority_seeds = [
-        s for s in SC_SEED_KEYWORDS
-        if any(k in s.lower() for k in (
-            "indigenous", "traditional", "cultural", "oral",
-            "postcolonial", "decolonial", "african", "ubuntu",
-        ))
-    ]
-
-    for seed in priority_seeds:
-        if len(papers) >= max_results:
-            break
-        query = f'"{institution_name}" "{seed}"'
-        page = 1
-        while len(papers) < max_results:
-            url = (
-                f"https://doaj.org/api/search/articles/{quote(query)}"
-                f"?page={page}&pageSize=100"
-            )
-            try:
-                resp = session.get(url, timeout=30)
-                resp.raise_for_status()
-            except requests.RequestException as exc:
-                print(f"  [DOAJ] {seed} err: {exc}", flush=True)
-                break
-            data = resp.json()
-            results = data.get("results", [])
-            if not results:
-                break
-            for article in results:
-                if len(papers) >= max_results:
-                    break
-                bib = article.get("bibjson") or {}
-                title_arr = bib.get("title") or ""
-                title = (title_arr if isinstance(title_arr, str) else "").strip()
-                if not title:
-                    continue
-                # DOI from identifiers list
-                doi = ""
-                for ident in bib.get("identifier") or []:
-                    if ident.get("type") == "doi":
-                        doi = ident.get("id") or ""
-                        break
-                norm = title.lower()[:120]
-                if doi and doi in seen_dois:
-                    continue
-                if norm in seen_titles:
-                    continue
-                abstract = (bib.get("abstract") or "").strip()
-                is_sc, score, cats = is_special_collection(title, abstract, "")
-                if not is_sc:
-                    continue
-                if doi:
-                    seen_dois.add(doi)
-                seen_titles.add(norm)
-                authors = [
-                    a.get("name", "") for a in (bib.get("author") or []) if a.get("name")
-                ]
-                pub_date = bib.get("year") or ""
-                url_val = f"https://doi.org/{doi}" if doi else ""
-                for lnk in bib.get("link") or []:
-                    if lnk.get("type") in ("fulltext", "homepage"):
-                        url_val = url_val or lnk.get("url", "")
-                papers.append(_paper(
-                    title, abstract, authors, doi, url_val, None,
-                    pub_date, bib.get("journal", {}).get("title", "") or "",
-                    score, cats, "DOAJ",
-                ))
-                _log_hit(len(papers), score, cats, title)
-            total = data.get("total", 0)
-            if page * 100 < min(total, 500) and len(papers) < max_results:
-                page += 1
-                time.sleep(_RATE_SLEEP)
-            else:
-                break
-        time.sleep(_RATE_SLEEP)
-    return papers
-
-
-# ── Helpers ───────────────────────────────────────────────────────────────────
-
-def _paper(
-    title, abstract, authors, doi, url, pdf_url,
-    pub_date, doc_type, score, cats, source,
-) -> dict:
-    return {
-        "title": title,
-        "abstract": abstract[:500] + ("…" if len(abstract) > 500 else ""),
-        "authors": authors[:5],
-        "doi": doi,
-        "url": url,
-        "pdf_url": pdf_url,
-        "publication_date": pub_date,
-        "dc_type": doc_type,
-        "sc_score": round(score, 1),
-        "sc_categories": cats,
-        "source": source,
-    }
-
-
-def _log_hit(n: int, score: float, cats: list, title: str):
-    safe = title.encode("ascii", errors="replace").decode("ascii")
-    print(
-        f"  [SC] #{n:>3}  score={score:.1f}  cats={','.join(cats)[:45]}  {safe[:65]}",
-        flush=True,
-    )
-
-
-# ── Main harvest orchestrator ─────────────────────────────────────────────────
-
-def harvest_all(
-    ror_short: str,
-    institution_name: str,
-    affiliation_patterns: list[str],
-    max_results: int,
-) -> list[dict]:
-    """
-    Fan out across all sources in parallel-quota mode.
-
-    Each source gets an equal quota (max_results // 4, minimum 10). After all
-    four sources run, results are merged (deduplicated), sorted by SC score, and
-    capped at max_results. This ensures the email reflects genuine multi-source
-    coverage rather than being filled by whichever source responds fastest.
-    """
-    session = requests.Session()
-    session.headers.update({
-        "User-Agent": f"URAAS-TestHarvest/1.0 (dry-run; mailto:{config.OPENALEX_MAILTO})",
-        "Accept": "application/json",
-    })
-
-    # Per-source quota: each source gets at least 10, up to max_results
-    per_source = max(10, max_results // 4)
-
-    # Each source uses its OWN seen sets so they don't clobber each other;
-    # dedup across sources happens in the merge step below.
-    source_results: dict[str, list[dict]] = {}
-
-    source_fns = [
-        ("OpenAlex", lambda: harvest_openalex(
-            session, ror_short, per_source, set(), set())),
-        ("Crossref", lambda: harvest_crossref(
-            session, institution_name, per_source, set(), set())),
-        ("Semantic Scholar", lambda: harvest_semantic_scholar(
-            session, institution_name, per_source, set(), set())),
-        ("DOAJ", lambda: harvest_doaj(
-            session, institution_name, per_source, set(), set())),
-    ]
-
-    for source_name, harvest_fn in source_fns:
-        print(f"\n[SOURCE] {source_name} (quota: {per_source})", flush=True)
-        print("-" * 40, flush=True)
-        papers = harvest_fn()
-        source_results[source_name] = papers
-        print(f"[{source_name}] found {len(papers)} SC papers", flush=True)
-
-    # Merge and deduplicate across sources
-    seen_dois: set[str] = set()
-    seen_titles: set[str] = set()
-    merged: list[dict] = []
-
-    # Interleave sources (round-robin) so the final list is balanced
-    max_src_len = max(len(v) for v in source_results.values()) if source_results else 0
-    source_names = list(source_results.keys())
-    for i in range(max_src_len):
-        for sname in source_names:
-            papers = source_results[sname]
-            if i >= len(papers):
-                continue
-            p = papers[i]
-            doi = p.get("doi") or ""
-            norm = (p.get("title") or "").lower()[:120]
-            if doi and doi in seen_dois:
-                continue
-            if norm and norm in seen_titles:
-                continue
-            if doi:
-                seen_dois.add(doi)
-            if norm:
-                seen_titles.add(norm)
-            merged.append(p)
-
-    # Sort by SC score descending
-    merged.sort(key=lambda p: p.get("sc_score", 0), reverse=True)
-    return merged[:max_results]
-
-
-# ── Email ─────────────────────────────────────────────────────────────────────
-
-def send_preview_email(to_email: str, institution_name: str, papers: list[dict]) -> bool:
-    from uraas.config import config as cfg
-
-    if not cfg.SMTP_HOST or not cfg.SMTP_USER or not cfg.SMTP_PASSWORD:
-        print("[WARN] SMTP not configured — skipping email. Set SMTP_* in .env", flush=True)
-        preview = json.dumps(papers[:3], indent=2, ensure_ascii=True)
-        print(f"       First 3 papers: {preview}", flush=True)
-        return False
-
-    import smtplib
-    from email.mime.multipart import MIMEMultipart
-    from email.mime.text import MIMEText
-
-    n = len(papers)
-    subject = f"[URAAS] Test Harvest Preview — {n} SC papers from {institution_name}"
-
-    source_counts: dict[str, int] = {}
-    for p in papers:
-        source_counts[p.get("source", "?")] = source_counts.get(p.get("source", "?"), 0) + 1
-    source_summary = " · ".join(f"{s}: {c}" for s, c in sorted(source_counts.items()))
-
-    rows_html = ""
-    for i, p in enumerate(papers, 1):
-        cats = ", ".join(p["sc_categories"])
-        url_part = (
-            f'<a href="{p["url"]}" style="color:#3b82f6">{p["url"][:55]}</a>'
-            if p["url"] else "—"
-        )
-        pdf_part = (
-            f' <a href="{p["pdf_url"]}" style="color:#16a34a;font-size:10px">[PDF]</a>'
-            if p.get("pdf_url") else ""
-        )
-        rows_html += f"""
-        <tr style="border-bottom:1px solid #e5e7eb">
-          <td style="padding:6px 4px;color:#9ca3af;font-size:11px">{i}</td>
-          <td style="padding:6px 6px;font-size:12px">
-            <strong>{p['title'][:85]}</strong><br>
-            <span style="font-size:10px;color:#6b7280">{', '.join(p['authors'][:2])}</span>
-          </td>
-          <td style="padding:6px 4px;font-size:11px;color:#6b7280">{p.get('dc_type','—')[:20]}</td>
-          <td style="padding:6px 4px;font-size:11px;color:#374151">{(p.get('publication_date') or '—')[:4]}</td>
-          <td style="padding:6px 4px;font-size:11px;color:#7c3aed">{cats}</td>
-          <td style="padding:6px 4px;font-size:10px;color:#2563eb">{p.get('source','?')}</td>
-          <td style="padding:6px 4px;font-size:10px">{url_part}{pdf_part}</td>
-        </tr>"""
-
-    plain_rows = "\n".join(
-        f"{i:>3}. [{p.get('source','?')}] {p['title'][:75]}\n"
-        f"     By: {', '.join(p['authors'][:2]) or '—'}  |  {(p.get('publication_date') or '')[:4]}\n"
-        f"     SC: {', '.join(p['sc_categories'])}  |  Score: {p['sc_score']}\n"
-        f"     URL: {p['url'] or '—'}\n"
-        for i, p in enumerate(papers, 1)
-    )
-
-    html = f"""<!DOCTYPE html>
-<html lang="en"><head><meta charset="utf-8"></head>
-<body style="font-family:Arial,sans-serif;background:#f9fafb;margin:0;padding:24px">
-<div style="max-width:950px;margin:0 auto;background:#fff;border-radius:8px;overflow:hidden;box-shadow:0 1px 8px rgba(0,0,0,.1)">
-  <div style="background:#1a3a5c;padding:24px 28px">
-    <p style="margin:0;font-size:10px;color:#7eb3d4;text-transform:uppercase;letter-spacing:2px">University of Lagos · URAAS</p>
-    <h1 style="margin:6px 0 0;font-size:20px;color:#fff">Test Harvest — Special Collections Preview</h1>
-    <p style="margin:6px 0 0;font-size:12px;color:#a8c9e0">
-      OpenAlex · Crossref · Semantic Scholar · DOAJ · Dry run · No IR deposit
-    </p>
-  </div>
-  <div style="padding:24px 28px">
-    <p style="font-size:14px;color:#374151;margin:0 0 12px">
-      <strong>Dry-run preview</strong> — no papers were saved to the local database and nothing was
-      deposited to the live IR. These are Special Collections papers by <strong>{institution_name}</strong>
-      authors discovered from across the open web.
-    </p>
-    <table style="width:100%;border-collapse:collapse;font-size:13px;margin:0 0 16px">
-      <tr style="background:#f3f4f6">
-        <td style="padding:8px 6px;font-weight:600">Institution</td>
-        <td style="padding:8px 6px">{institution_name}</td>
-        <td style="padding:8px 6px;font-weight:600">Total SC papers</td>
-        <td style="padding:8px 6px"><strong style="color:#1a7a4a">{n}</strong></td>
-      </tr>
-      <tr>
-        <td style="padding:8px 6px;font-weight:600">Sources</td>
-        <td colspan="3" style="padding:8px 6px">{source_summary}</td>
-      </tr>
-    </table>
-    <table style="width:100%;border-collapse:collapse;font-size:12px">
-      <thead>
-        <tr style="background:#f3f4f6;text-align:left">
-          <th style="padding:6px 4px">#</th>
-          <th style="padding:6px 6px">Title / Authors</th>
-          <th style="padding:6px 4px">Type</th>
-          <th style="padding:6px 4px">Year</th>
-          <th style="padding:6px 4px">SC Categories</th>
-          <th style="padding:6px 4px">Source</th>
-          <th style="padding:6px 4px">URL / PDF</th>
-        </tr>
-      </thead>
-      <tbody>{rows_html}</tbody>
-    </table>
-    <p style="font-size:12px;color:#6b7280;margin:20px 0 0">
-      Papers above were found on the open web and are NOT yet confirmed to be in the UNILAG IR.
-      When ready to queue them for IR deposit, use the <strong>IR Deposit</strong> panel in the dashboard.
-    </p>
-  </div>
-  <div style="background:#f0f4f8;padding:14px 28px;font-size:11px;color:#9ca3af;text-align:center">
-    URAAS · APA Intelligence &amp; Analytics Platform · University of Lagos · Dry-run — nothing was changed
-  </div>
-</div>
-</body></html>"""
-
-    plain = f"""URAAS Test Harvest — {institution_name}
-Sources: {source_summary}
-DRY RUN — nothing saved to DB, nothing deposited to IR.
-
-Special Collections papers found: {n}
-
-{plain_rows}
----
-URAAS — APA Intelligence & Analytics Platform
-"""
-
-    msg = MIMEMultipart("alternative")
-    msg["Subject"] = subject
-    msg["From"] = cfg.SMTP_FROM
-    msg["To"] = to_email
-    msg.attach(MIMEText(plain, "plain", "utf-8"))
-    msg.attach(MIMEText(html, "html", "utf-8"))
-
-    try:
-        if cfg.SMTP_USE_TLS:
-            srv = smtplib.SMTP(cfg.SMTP_HOST, cfg.SMTP_PORT, timeout=30)
-            srv.ehlo()
-            srv.starttls()
-            srv.ehlo()
-        else:
-            srv = smtplib.SMTP_SSL(cfg.SMTP_HOST, cfg.SMTP_PORT, timeout=30)
-        srv.login(cfg.SMTP_USER, cfg.SMTP_PASSWORD)
-        srv.sendmail(cfg.SMTP_FROM, [to_email], msg.as_bytes())
-        srv.quit()
-        print(f"[OK] Email sent to {to_email}", flush=True)
-        return True
-    except Exception as exc:
-        print(f"[ERR] Email failed: {exc}", flush=True)
-        return False
-
-
-def save_json_preview(papers: list[dict], institution: str) -> str:
-    out_path = os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        "storage",
-        f"test_harvest_{institution}_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}.json",
-    )
-    os.makedirs(os.path.dirname(out_path), exist_ok=True)
-    with open(out_path, "w", encoding="utf-8") as f:
-        json.dump(
-            {"institution": institution, "count": len(papers), "papers": papers},
-            f, indent=2, ensure_ascii=False,
-        )
-    print(f"[OK] Preview saved to: {out_path}", flush=True)
-    return out_path
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Multi-source test harvest (dry run — no DB, no IR deposit)"
-    )
-    parser.add_argument("--institution", default="unilag")
-    parser.add_argument("--count", type=int, default=50, help="Max SC papers to collect")
-    parser.add_argument("--email", default="lawalgiyath200716@gmail.com")
-    args = parser.parse_args()
-
-    registry = get_registry()
-    inst_cfg = registry.get(args.institution)
-    if not inst_cfg:
-        print(f"[ERR] Institution '{args.institution}' not found", flush=True)
-        sys.exit(1)
-
-    ror_short = inst_cfg.ror.split("/")[-1]
-
-    print(f"\n{'='*60}", flush=True)
-    print(f"URAAS DRY-RUN HARVEST — {inst_cfg.name}", flush=True)
-    print(f"Sources: OpenAlex · Crossref · Semantic Scholar · DOAJ", flush=True)
-    print(f"{'='*60}", flush=True)
-
-    papers = harvest_all(
-        ror_short,
-        inst_cfg.name,
-        inst_cfg.affiliation_patterns,
-        args.count,
-    )
-
-    save_json_preview(papers, args.institution)
-    send_preview_email(args.email, inst_cfg.name, papers)
-
-    # Source breakdown
-    source_counts: dict[str, int] = {}
-    for p in papers:
-        source_counts[p.get("source", "?")] = source_counts.get(p.get("source", "?"), 0) + 1
-
-    print(f"\n{'='*60}", flush=True)
-    print("HARVEST SUMMARY", flush=True)
-    print(f"  Institution : {inst_cfg.name}", flush=True)
-    print(f"  Total SC    : {len(papers)}", flush=True)
-    for src, cnt in sorted(source_counts.items()):
-        print(f"    {src:<22}: {cnt}", flush=True)
-    print(f"  Email       : {args.email}", flush=True)
-    print("  IR deposit  : NOT performed (dry run)", flush=True)
-    print(f"{'='*60}\n", flush=True)
-
-
-if __name__ == "__main__":
-    main()
+"""
+Test Harvest — UNILAG Special Collections papers (dry run, multi-source).
+
+Discovers SC papers from the open web by querying multiple academic databases:
+  • OpenAlex     — broad academic paper index (journals, books, preprints)
+  • Crossref     — DOI metadata authority, strong on humanities/social sciences
+  • Semantic Scholar — AI-indexed full-text coverage, good for humanities
+  • EuropePMC    — PubMed + PMC + WHO; best for ethnobotany / traditional medicine
+
+For each source: queries institution affiliation + SC seed keywords, then applies
+the same SC classifier used by the main pipeline. Results are deduplicated by DOI
+and normalised title across all sources.
+
+DOES NOT save anything to the local database.
+DOES NOT deposit anything to the live DSpace IR.
+Safe to run at any time.
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+from datetime import datetime, timezone
+from typing import Any
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import requests
+
+from uraas.config import config
+from uraas.config.institutions import get_registry
+from uraas.config.special_collections import SC_SEED_KEYWORDS
+from uraas.services.sc_engine import is_special_collection
+
+_RATE_SLEEP = 1.0  # polite delay between requests per source
+
+
+# ── OpenAlex ──────────────────────────────────────────────────────────────────
+
+def _reconstruct_abstract(inverted_index: dict) -> str:
+    if not inverted_index:
+        return ""
+    pairs = [(pos, word) for word, positions in inverted_index.items() for pos in positions]
+    pairs.sort()
+    return " ".join(w for _, w in pairs)
+
+
+def _openalex_url(ror_short: str, seed: str | None, cursor: str = "*") -> str:
+    filters = f"institutions.ror:{ror_short}"
+    if seed:
+        filters += f",title_and_abstract.search:{seed.replace(' ','%20')}"
+    return (
+        f"https://api.openalex.org/works"
+        f"?filter={filters}"
+        f"&select=id,doi,title,abstract_inverted_index,authorships,"
+        f"publication_date,open_access,primary_location,concepts,type"
+        f"&per-page=100&cursor={cursor}&mailto={config.OPENALEX_MAILTO}"
+    )
+
+
+def harvest_openalex(
+    session: requests.Session,
+    ror_short: str,
+    max_results: int,
+    seen_dois: set,
+    seen_titles: set,
+) -> list[dict]:
+    papers: list[dict] = []
+    seeds = list(SC_SEED_KEYWORDS) + [None]  # None = general ROR wave last
+
+    for seed in seeds:
+        if len(papers) >= max_results:
+            break
+        url = _openalex_url(ror_short, seed)
+        page = 0
+        while url and len(papers) < max_results:
+            page += 1
+            try:
+                resp = session.get(url, timeout=30)
+                resp.raise_for_status()
+            except requests.RequestException as exc:
+                print(f"  [OA] {seed or 'general'} page {page} err: {exc}", flush=True)
+                break
+            data = resp.json()
+            results = data.get("results", [])
+            for work in results:
+                if len(papers) >= max_results:
+                    break
+                title = (work.get("title") or "").strip()
+                if not title:
+                    continue
+                doi = (work.get("doi") or "").replace("https://doi.org/", "").strip()
+                norm = title.lower()[:120]
+                if doi and doi in seen_dois:
+                    continue
+                if norm in seen_titles:
+                    continue
+                abstract = _reconstruct_abstract(work.get("abstract_inverted_index") or {})
+                concepts = work.get("concepts") or []
+                dc_subject = ", ".join(c.get("display_name", "") for c in concepts[:6] if c)
+                is_sc, score, cats = is_special_collection(title, abstract, dc_subject)
+                if not is_sc:
+                    continue
+                if doi:
+                    seen_dois.add(doi)
+                seen_titles.add(norm)
+                authors = [
+                    a.get("author", {}).get("display_name", "")
+                    for a in work.get("authorships", [])
+                    if a.get("author", {}).get("display_name")
+                ]
+                oa = work.get("open_access") or {}
+                landing = (work.get("primary_location") or {}).get("landing_page_url") or ""
+                url_val = landing or (f"https://doi.org/{doi}" if doi else "")
+                papers.append(_paper(
+                    title, abstract, authors, doi, url_val,
+                    oa.get("oa_url") if oa.get("is_oa") else None,
+                    work.get("publication_date") or "",
+                    work.get("type") or "",
+                    score, cats, "OpenAlex",
+                ))
+                _log_hit(len(papers), score, cats, title)
+            meta = data.get("meta") or {}
+            next_cursor = meta.get("next_cursor")
+            if next_cursor and results and len(papers) < max_results:
+                from urllib.parse import parse_qs, urlparse
+                qs = parse_qs(urlparse(url).query)
+                filt = (qs.get("filter") or [f"institutions.ror:{ror_short}"])[0]
+                url = (
+                    f"https://api.openalex.org/works?filter={filt}"
+                    f"&select=id,doi,title,abstract_inverted_index,authorships,"
+                    f"publication_date,open_access,primary_location,concepts,type"
+                    f"&per-page=100&cursor={next_cursor}&mailto={config.OPENALEX_MAILTO}"
+                )
+                time.sleep(_RATE_SLEEP)
+            else:
+                url = ""
+    return papers
+
+
+# ── Crossref ──────────────────────────────────────────────────────────────────
+
+def harvest_crossref(
+    session: requests.Session,
+    institution_name: str,
+    max_results: int,
+    seen_dois: set,
+    seen_titles: set,
+) -> list[dict]:
+    from urllib.parse import quote
+    papers: list[dict] = []
+    seeds = list(SC_SEED_KEYWORDS) + [None]
+
+    for seed in seeds:
+        if len(papers) >= max_results:
+            break
+        q_seed = f"&query={quote(seed)}" if seed else ""
+        url = (
+            f"https://api.crossref.org/works"
+            f"?query.affiliation={quote(institution_name)}"
+            f"{q_seed}"
+            f"&select=DOI,title,abstract,author,issued,URL,link"
+            f"&rows=50&offset=0&mailto={config.OPENALEX_MAILTO}"
+        )
+        offset = 0
+        while url and len(papers) < max_results:
+            try:
+                resp = session.get(url, timeout=30)
+                resp.raise_for_status()
+            except requests.RequestException as exc:
+                print(f"  [CR] {seed or 'general'} err: {exc}", flush=True)
+                break
+            items = resp.json().get("message", {}).get("items", [])
+            for work in items:
+                if len(papers) >= max_results:
+                    break
+                title_arr = work.get("title") or []
+                title = (title_arr[0] if title_arr else "").strip()
+                if not title:
+                    continue
+                doi = (work.get("DOI") or "").strip()
+                norm = title.lower()[:120]
+                if doi and doi in seen_dois:
+                    continue
+                if norm in seen_titles:
+                    continue
+                abstract = (work.get("abstract") or "").strip()
+                is_sc, score, cats = is_special_collection(title, abstract, "")
+                if not is_sc:
+                    continue
+                if doi:
+                    seen_dois.add(doi)
+                seen_titles.add(norm)
+                authors = [
+                    f"{a.get('given','')} {a.get('family','')}".strip()
+                    for a in work.get("author", [])
+                    if a.get("family")
+                ]
+                issued = work.get("issued", {}).get("date-parts", [[]])[0]
+                pub_date = "-".join(str(p) for p in issued) if issued else ""
+                pdf_url = next(
+                    (lk["URL"] for lk in work.get("link", [])
+                     if lk.get("content-type") == "application/pdf"),
+                    None,
+                )
+                url_val = work.get("URL") or (f"https://doi.org/{doi}" if doi else "")
+                papers.append(_paper(
+                    title, abstract, authors, doi, url_val, pdf_url,
+                    pub_date, "", score, cats, "Crossref",
+                ))
+                _log_hit(len(papers), score, cats, title)
+            offset += 50
+            if items and offset < 500 and len(papers) < max_results:
+                q_seed2 = f"&query={quote(seed)}" if seed else ""
+                url = (
+                    f"https://api.crossref.org/works"
+                    f"?query.affiliation={quote(institution_name)}"
+                    f"{q_seed2}"
+                    f"&select=DOI,title,abstract,author,issued,URL,link"
+                    f"&rows=50&offset={offset}&mailto={config.OPENALEX_MAILTO}"
+                )
+                time.sleep(_RATE_SLEEP)
+            else:
+                url = ""
+    return papers
+
+
+# ── Semantic Scholar ─────────────────���────────────────────────────────────────
+
+def harvest_semantic_scholar(
+    session: requests.Session,
+    institution_name: str,
+    max_results: int,
+    seen_dois: set,
+    seen_titles: set,
+) -> list[dict]:
+    """
+    Semantic Scholar free tier: 1 req/sec (unauthenticated).
+    We fire only 3 broad queries instead of one per seed to stay within rate limits.
+    Set S2_API_KEY in .env for a higher rate limit (free key at semanticscholar.org).
+    """
+    from urllib.parse import quote
+    from uraas.config import config as cfg
+
+    papers: list[dict] = []
+    # Pull API key from env if available
+    api_key = getattr(cfg, "S2_API_KEY", "") or os.environ.get("S2_API_KEY", "")
+    delay = 1.2 if not api_key else 0.3
+
+    headers = {}
+    if api_key:
+        headers["x-api-key"] = api_key
+
+    # Use 3 broad queries rather than 20+ per-seed blasts
+    broad_queries = [
+        f"{institution_name} indigenous knowledge cultural heritage",
+        f"{institution_name} postcolonial african literature oral tradition",
+        f"{institution_name} ethnobotany traditional medicine decolonial",
+    ]
+
+    for query in broad_queries:
+        if len(papers) >= max_results:
+            break
+        offset = 0
+        while len(papers) < max_results:
+            url = (
+                f"https://api.semanticscholar.org/graph/v1/paper/search"
+                f"?query={quote(query)}"
+                f"&fields=title,abstract,authors,year,externalIds,openAccessPdf"
+                f"&limit=100&offset={offset}"
+            )
+            try:
+                time.sleep(delay)
+                resp = session.get(url, timeout=30, headers=headers)
+                if resp.status_code == 429:
+                    # Rate limited — skip remaining S2 queries rather than block.
+                    # Add S2_API_KEY to .env (free at semanticscholar.org) to lift limit.
+                    print(
+                        "  [S2] Rate limited. Add S2_API_KEY to .env for higher quota.",
+                        flush=True,
+                    )
+                    return papers
+                resp.raise_for_status()
+            except requests.RequestException as exc:
+                print(f"  [S2] {query[:40]} err: {exc}", flush=True)
+                break
+            data = resp.json()
+            results = data.get("data", [])
+            if not results:
+                break
+            for work in results:
+                if len(papers) >= max_results:
+                    break
+                title = (work.get("title") or "").strip()
+                if not title:
+                    continue
+                ext = work.get("externalIds") or {}
+                doi = (ext.get("DOI") or ext.get("doi") or "").strip()
+                norm = title.lower()[:120]
+                if doi and doi in seen_dois:
+                    continue
+                if norm in seen_titles:
+                    continue
+                abstract = (work.get("abstract") or "").strip()
+                is_sc, score, cats = is_special_collection(title, abstract, "")
+                if not is_sc:
+                    continue
+                if doi:
+                    seen_dois.add(doi)
+                seen_titles.add(norm)
+                authors = [
+                    a.get("name", "") for a in (work.get("authors") or []) if a.get("name")
+                ]
+                year = work.get("year")
+                oa = work.get("openAccessPdf") or {}
+                url_val = f"https://doi.org/{doi}" if doi else ""
+                papers.append(_paper(
+                    title, abstract, authors, doi, url_val,
+                    oa.get("url"), f"{year}-01-01" if year else "",
+                    "", score, cats, "Semantic Scholar",
+                ))
+                _log_hit(len(papers), score, cats, title)
+            total = data.get("total", 0)
+            offset += 100
+            if offset < min(total, 300) and len(papers) < max_results:
+                pass
+            else:
+                break
+    return papers
+
+
+# ── EuropePMC ─────────────────────────────────────────────────────────────────
+
+def harvest_europepmc(
+    session: requests.Session,
+    institution_name: str,
+    affiliation_patterns: list[str],
+    max_results: int,
+    seen_dois: set,
+    seen_titles: set,
+) -> list[dict]:
+    from urllib.parse import urlencode
+    papers: list[dict] = []
+
+    # EuropePMC AFFILIATION field matches against stored author affiliation strings.
+    # UNILAG papers appear under several spellings — use a short unambiguous token.
+    affil = '(AFFILIATION:"University of Lagos" OR AFFILIATION:"unilag" OR AFFILIATION:"UNILAG")'
+
+    # EuropePMC is best for ethnobotany/traditional medicine SC papers
+    priority_seeds = [
+        s for s in SC_SEED_KEYWORDS
+        if any(k in s.lower() for k in (
+            "indigenous", "traditional", "ethnobotany", "cultural",
+            "oral", "decolonial", "ubuntu", "ethnomusicology",
+        ))
+    ]
+
+    for seed in priority_seeds:
+        if len(papers) >= max_results:
+            break
+        query = f'{affil} AND ("{seed}")'
+        cursor = "*"
+        while len(papers) < max_results:
+            params = {
+                "query": query,
+                "format": "json",
+                "pageSize": 100,
+                "resultType": "core",
+                "cursorMark": cursor,
+            }
+            url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/search?{urlencode(params)}"
+            try:
+                resp = session.get(url, timeout=30)
+                resp.raise_for_status()
+            except requests.RequestException as exc:
+                print(f"  [EPMC] {seed} err: {exc}", flush=True)
+                break
+            data = resp.json()
+            results = data.get("resultList", {}).get("result", [])
+            for r in results:
+                if len(papers) >= max_results:
+                    break
+                title = (r.get("title") or "").strip().rstrip(".")
+                if not title:
+                    continue
+                doi = (r.get("doi") or "").strip()
+                norm = title.lower()[:120]
+                if doi and doi in seen_dois:
+                    continue
+                if norm in seen_titles:
+                    continue
+                abstract = (r.get("abstractText") or "").strip()
+                is_sc, score, cats = is_special_collection(title, abstract, "")
+                if not is_sc:
+                    continue
+                if doi:
+                    seen_dois.add(doi)
+                seen_titles.add(norm)
+                pmid = r.get("pmid") or ""
+                url_val = (
+                    f"https://doi.org/{doi}" if doi
+                    else (f"https://europepmc.org/article/med/{pmid}" if pmid else "")
+                )
+                pdf_url = None
+                if r.get("isOpenAccess") == "Y":
+                    for ft in ((r.get("fullTextUrlList") or {}).get("fullTextUrl") or []):
+                        if ft.get("documentStyle") == "pdf":
+                            pdf_url = ft.get("url")
+                            break
+                authors_raw = (r.get("authorList") or {}).get("author") or []
+                authors = [
+                    f"{a.get('firstName','')} {a.get('lastName','')}".strip()
+                    for a in authors_raw if a.get("lastName")
+                ]
+                papers.append(_paper(
+                    title, abstract, authors, doi, url_val, pdf_url,
+                    r.get("firstPublicationDate") or r.get("pubYear") or "",
+                    r.get("pubType") or "", score, cats, "EuropePMC",
+                ))
+                _log_hit(len(papers), score, cats, title)
+            next_cursor = data.get("nextCursorMark")
+            if next_cursor and next_cursor != cursor and results and len(papers) < max_results:
+                cursor = next_cursor
+                time.sleep(_RATE_SLEEP)
+            else:
+                break
+        time.sleep(_RATE_SLEEP)
+    return papers
+
+
+# ── DOAJ ──────────────────────────────────────────────────────────────────────
+
+def harvest_doaj(
+    session: requests.Session,
+    institution_name: str,
+    max_results: int,
+    seen_dois: set,
+    seen_titles: set,
+) -> list[dict]:
+    """
+    Directory of Open Access Journals (DOAJ) — covers many African humanities
+    and social-science journals that are not in OpenAlex or Crossref.
+    Free API, no key required.
+    """
+    from urllib.parse import quote
+    papers: list[dict] = []
+
+    # DOAJ article search: query is full-text across title/abstract/keywords
+    priority_seeds = [
+        s for s in SC_SEED_KEYWORDS
+        if any(k in s.lower() for k in (
+            "indigenous", "traditional", "cultural", "oral",
+            "postcolonial", "decolonial", "african", "ubuntu",
+        ))
+    ]
+
+    for seed in priority_seeds:
+        if len(papers) >= max_results:
+            break
+        query = f'"{institution_name}" "{seed}"'
+        page = 1
+        while len(papers) < max_results:
+            url = (
+                f"https://doaj.org/api/search/articles/{quote(query)}"
+                f"?page={page}&pageSize=100"
+            )
+            try:
+                resp = session.get(url, timeout=30)
+                resp.raise_for_status()
+            except requests.RequestException as exc:
+                print(f"  [DOAJ] {seed} err: {exc}", flush=True)
+                break
+            data = resp.json()
+            results = data.get("results", [])
+            if not results:
+                break
+            for article in results:
+                if len(papers) >= max_results:
+                    break
+                bib = article.get("bibjson") or {}
+                title_arr = bib.get("title") or ""
+                title = (title_arr if isinstance(title_arr, str) else "").strip()
+                if not title:
+                    continue
+                # DOI from identifiers list
+                doi = ""
+                for ident in bib.get("identifier") or []:
+                    if ident.get("type") == "doi":
+                        doi = ident.get("id") or ""
+                        break
+                norm = title.lower()[:120]
+                if doi and doi in seen_dois:
+                    continue
+                if norm in seen_titles:
+                    continue
+                abstract = (bib.get("abstract") or "").strip()
+                is_sc, score, cats = is_special_collection(title, abstract, "")
+                if not is_sc:
+                    continue
+                if doi:
+                    seen_dois.add(doi)
+                seen_titles.add(norm)
+                authors = [
+                    a.get("name", "") for a in (bib.get("author") or []) if a.get("name")
+                ]
+                pub_date = bib.get("year") or ""
+                url_val = f"https://doi.org/{doi}" if doi else ""
+                for lnk in bib.get("link") or []:
+                    if lnk.get("type") in ("fulltext", "homepage"):
+                        url_val = url_val or lnk.get("url", "")
+                papers.append(_paper(
+                    title, abstract, authors, doi, url_val, None,
+                    pub_date, bib.get("journal", {}).get("title", "") or "",
+                    score, cats, "DOAJ",
+                ))
+                _log_hit(len(papers), score, cats, title)
+            total = data.get("total", 0)
+            if page * 100 < min(total, 500) and len(papers) < max_results:
+                page += 1
+                time.sleep(_RATE_SLEEP)
+            else:
+                break
+        time.sleep(_RATE_SLEEP)
+    return papers
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+def _paper(
+    title, abstract, authors, doi, url, pdf_url,
+    pub_date, doc_type, score, cats, source,
+) -> dict:
+    return {
+        "title": title,
+        "abstract": abstract[:500] + ("…" if len(abstract) > 500 else ""),
+        "authors": authors[:5],
+        "doi": doi,
+        "url": url,
+        "pdf_url": pdf_url,
+        "publication_date": pub_date,
+        "dc_type": doc_type,
+        "sc_score": round(score, 1),
+        "sc_categories": cats,
+        "source": source,
+    }
+
+
+def _log_hit(n: int, score: float, cats: list, title: str):
+    safe = title.encode("ascii", errors="replace").decode("ascii")
+    print(
+        f"  [SC] #{n:>3}  score={score:.1f}  cats={','.join(cats)[:45]}  {safe[:65]}",
+        flush=True,
+    )
+
+
+# ── Main harvest orchestrator ─────────────────────────────────────────────────
+
+def harvest_all(
+    ror_short: str,
+    institution_name: str,
+    affiliation_patterns: list[str],
+    max_results: int,
+) -> list[dict]:
+    """
+    Fan out across all sources in parallel-quota mode.
+
+    Each source gets an equal quota (max_results // 4, minimum 10). After all
+    four sources run, results are merged (deduplicated), sorted by SC score, and
+    capped at max_results. This ensures the email reflects genuine multi-source
+    coverage rather than being filled by whichever source responds fastest.
+    """
+    session = requests.Session()
+    session.headers.update({
+        "User-Agent": f"URAAS-TestHarvest/1.0 (dry-run; mailto:{config.OPENALEX_MAILTO})",
+        "Accept": "application/json",
+    })
+
+    # Per-source quota: each source gets at least 10, up to max_results
+    per_source = max(10, max_results // 4)
+
+    # Each source uses its OWN seen sets so they don't clobber each other;
+    # dedup across sources happens in the merge step below.
+    source_results: dict[str, list[dict]] = {}
+
+    source_fns = [
+        ("OpenAlex", lambda: harvest_openalex(
+            session, ror_short, per_source, set(), set())),
+        ("Crossref", lambda: harvest_crossref(
+            session, institution_name, per_source, set(), set())),
+        ("Semantic Scholar", lambda: harvest_semantic_scholar(
+            session, institution_name, per_source, set(), set())),
+        ("DOAJ", lambda: harvest_doaj(
+            session, institution_name, per_source, set(), set())),
+    ]
+
+    for source_name, harvest_fn in source_fns:
+        print(f"\n[SOURCE] {source_name} (quota: {per_source})", flush=True)
+        print("-" * 40, flush=True)
+        papers = harvest_fn()
+        source_results[source_name] = papers
+        print(f"[{source_name}] found {len(papers)} SC papers", flush=True)
+
+    # Merge and deduplicate across sources
+    seen_dois: set[str] = set()
+    seen_titles: set[str] = set()
+    merged: list[dict] = []
+
+    # Interleave sources (round-robin) so the final list is balanced
+    max_src_len = max(len(v) for v in source_results.values()) if source_results else 0
+    source_names = list(source_results.keys())
+    for i in range(max_src_len):
+        for sname in source_names:
+            papers = source_results[sname]
+            if i >= len(papers):
+                continue
+            p = papers[i]
+            doi = p.get("doi") or ""
+            norm = (p.get("title") or "").lower()[:120]
+            if doi and doi in seen_dois:
+                continue
+            if norm and norm in seen_titles:
+                continue
+            if doi:
+                seen_dois.add(doi)
+            if norm:
+                seen_titles.add(norm)
+            merged.append(p)
+
+    # Sort by SC score descending
+    merged.sort(key=lambda p: p.get("sc_score", 0), reverse=True)
+    return merged[:max_results]
+
+
+# ── Email ─────────────────────────────────────────────────────────────────────
+
+def send_preview_email(to_email: str, institution_name: str, papers: list[dict]) -> bool:
+    from uraas.config import config as cfg
+
+    if not cfg.SMTP_HOST or not cfg.SMTP_USER or not cfg.SMTP_PASSWORD:
+        print("[WARN] SMTP not configured — skipping email. Set SMTP_* in .env", flush=True)
+        preview = json.dumps(papers[:3], indent=2, ensure_ascii=True)
+        print(f"       First 3 papers: {preview}", flush=True)
+        return False
+
+    import smtplib
+    from email.mime.multipart import MIMEMultipart
+    from email.mime.text import MIMEText
+
+    n = len(papers)
+    subject = f"[URAAS] Test Harvest Preview — {n} SC papers from {institution_name}"
+
+    source_counts: dict[str, int] = {}
+    for p in papers:
+        source_counts[p.get("source", "?")] = source_counts.get(p.get("source", "?"), 0) + 1
+    source_summary = " · ".join(f"{s}: {c}" for s, c in sorted(source_counts.items()))
+
+    rows_html = ""
+    for i, p in enumerate(papers, 1):
+        cats = ", ".join(p["sc_categories"])
+        url_part = (
+            f'<a href="{p["url"]}" style="color:#3b82f6">{p["url"][:55]}</a>'
+            if p["url"] else "—"
+        )
+        pdf_part = (
+            f' <a href="{p["pdf_url"]}" style="color:#16a34a;font-size:10px">[PDF]</a>'
+            if p.get("pdf_url") else ""
+        )
+        rows_html += f"""
+        <tr style="border-bottom:1px solid #e5e7eb">
+          <td style="padding:6px 4px;color:#9ca3af;font-size:11px">{i}</td>
+          <td style="padding:6px 6px;font-size:12px">
+            <strong>{p['title'][:85]}</strong><br>
+            <span style="font-size:10px;color:#6b7280">{', '.join(p['authors'][:2])}</span>
+          </td>
+          <td style="padding:6px 4px;font-size:11px;color:#6b7280">{p.get('dc_type','—')[:20]}</td>
+          <td style="padding:6px 4px;font-size:11px;color:#374151">{(p.get('publication_date') or '—')[:4]}</td>
+          <td style="padding:6px 4px;font-size:11px;color:#7c3aed">{cats}</td>
+          <td style="padding:6px 4px;font-size:10px;color:#2563eb">{p.get('source','?')}</td>
+          <td style="padding:6px 4px;font-size:10px">{url_part}{pdf_part}</td>
+        </tr>"""
+
+    plain_rows = "\n".join(
+        f"{i:>3}. [{p.get('source','?')}] {p['title'][:75]}\n"
+        f"     By: {', '.join(p['authors'][:2]) or '—'}  |  {(p.get('publication_date') or '')[:4]}\n"
+        f"     SC: {', '.join(p['sc_categories'])}  |  Score: {p['sc_score']}\n"
+        f"     URL: {p['url'] or '—'}\n"
+        for i, p in enumerate(papers, 1)
+    )
+
+    html = f"""<!DOCTYPE html>
+<html lang="en"><head><meta charset="utf-8"></head>
+<body style="font-family:Arial,sans-serif;background:#f9fafb;margin:0;padding:24px">
+<div style="max-width:950px;margin:0 auto;background:#fff;border-radius:8px;overflow:hidden;box-shadow:0 1px 8px rgba(0,0,0,.1)">
+  <div style="background:#1a3a5c;padding:24px 28px">
+    <p style="margin:0;font-size:10px;color:#7eb3d4;text-transform:uppercase;letter-spacing:2px">University of Lagos · URAAS</p>
+    <h1 style="margin:6px 0 0;font-size:20px;color:#fff">Test Harvest — Special Collections Preview</h1>
+    <p style="margin:6px 0 0;font-size:12px;color:#a8c9e0">
+      OpenAlex · Crossref · Semantic Scholar · DOAJ · Dry run · No IR deposit
+    </p>
+  </div>
+  <div style="padding:24px 28px">
+    <p style="font-size:14px;color:#374151;margin:0 0 12px">
+      <strong>Dry-run preview</strong> — no papers were saved to the local database and nothing was
+      deposited to the live IR. These are Special Collections papers by <strong>{institution_name}</strong>
+      authors discovered from across the open web.
+    </p>
+    <table style="width:100%;border-collapse:collapse;font-size:13px;margin:0 0 16px">
+      <tr style="background:#f3f4f6">
+        <td style="padding:8px 6px;font-weight:600">Institution</td>
+        <td style="padding:8px 6px">{institution_name}</td>
+        <td style="padding:8px 6px;font-weight:600">Total SC papers</td>
+        <td style="padding:8px 6px"><strong style="color:#1a7a4a">{n}</strong></td>
+      </tr>
+      <tr>
+        <td style="padding:8px 6px;font-weight:600">Sources</td>
+        <td colspan="3" style="padding:8px 6px">{source_summary}</td>
+      </tr>
+    </table>
+    <table style="width:100%;border-collapse:collapse;font-size:12px">
+      <thead>
+        <tr style="background:#f3f4f6;text-align:left">
+          <th style="padding:6px 4px">#</th>
+          <th style="padding:6px 6px">Title / Authors</th>
+          <th style="padding:6px 4px">Type</th>
+          <th style="padding:6px 4px">Year</th>
+          <th style="padding:6px 4px">SC Categories</th>
+          <th style="padding:6px 4px">Source</th>
+          <th style="padding:6px 4px">URL / PDF</th>
+        </tr>
+      </thead>
+      <tbody>{rows_html}</tbody>
+    </table>
+    <p style="font-size:12px;color:#6b7280;margin:20px 0 0">
+      Papers above were found on the open web and are NOT yet confirmed to be in the UNILAG IR.
+      When ready to queue them for IR deposit, use the <strong>IR Deposit</strong> panel in the dashboard.
+    </p>
+  </div>
+  <div style="background:#f0f4f8;padding:14px 28px;font-size:11px;color:#9ca3af;text-align:center">
+    URAAS · APA Intelligence &amp; Analytics Platform · University of Lagos · Dry-run — nothing was changed
+  </div>
+</div>
+</body></html>"""
+
+    plain = f"""URAAS Test Harvest — {institution_name}
+Sources: {source_summary}
+DRY RUN — nothing saved to DB, nothing deposited to IR.
+
+Special Collections papers found: {n}
+
+{plain_rows}
+---
+URAAS — APA Intelligence & Analytics Platform
+"""
+
+    msg = MIMEMultipart("alternative")
+    msg["Subject"] = subject
+    msg["From"] = cfg.SMTP_FROM
+    msg["To"] = to_email
+    msg.attach(MIMEText(plain, "plain", "utf-8"))
+    msg.attach(MIMEText(html, "html", "utf-8"))
+
+    try:
+        if cfg.SMTP_USE_TLS:
+            srv = smtplib.SMTP(cfg.SMTP_HOST, cfg.SMTP_PORT, timeout=30)
+            srv.ehlo()
+            srv.starttls()
+            srv.ehlo()
+        else:
+            srv = smtplib.SMTP_SSL(cfg.SMTP_HOST, cfg.SMTP_PORT, timeout=30)
+        srv.login(cfg.SMTP_USER, cfg.SMTP_PASSWORD)
+        srv.sendmail(cfg.SMTP_FROM, [to_email], msg.as_bytes())
+        srv.quit()
+        print(f"[OK] Email sent to {to_email}", flush=True)
+        return True
+    except Exception as exc:
+        print(f"[ERR] Email failed: {exc}", flush=True)
+        return False
+
+
+def save_json_preview(papers: list[dict], institution: str) -> str:
+    out_path = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+        "storage",
+        f"test_harvest_{institution}_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}.json",
+    )
+    os.makedirs(os.path.dirname(out_path), exist_ok=True)
+    with open(out_path, "w", encoding="utf-8") as f:
+        json.dump(
+            {"institution": institution, "count": len(papers), "papers": papers},
+            f, indent=2, ensure_ascii=False,
+        )
+    print(f"[OK] Preview saved to: {out_path}", flush=True)
+    return out_path
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Multi-source test harvest (dry run — no DB, no IR deposit)"
+    )
+    parser.add_argument("--institution", default="unilag")
+    parser.add_argument("--count", type=int, default=50, help="Max SC papers to collect")
+    parser.add_argument("--email", default="lawalgiyath200716@gmail.com")
+    args = parser.parse_args()
+
+    registry = get_registry()
+    inst_cfg = registry.get(args.institution)
+    if not inst_cfg:
+        print(f"[ERR] Institution '{args.institution}' not found", flush=True)
+        sys.exit(1)
+
+    ror_short = inst_cfg.ror.split("/")[-1]
+
+    print(f"\n{'='*60}", flush=True)
+    print(f"URAAS DRY-RUN HARVEST — {inst_cfg.name}", flush=True)
+    print(f"Sources: OpenAlex · Crossref · Semantic Scholar · DOAJ", flush=True)
+    print(f"{'='*60}", flush=True)
+
+    papers = harvest_all(
+        ror_short,
+        inst_cfg.name,
+        inst_cfg.affiliation_patterns,
+        args.count,
+    )
+
+    save_json_preview(papers, args.institution)
+    send_preview_email(args.email, inst_cfg.name, papers)
+
+    # Source breakdown
+    source_counts: dict[str, int] = {}
+    for p in papers:
+        source_counts[p.get("source", "?")] = source_counts.get(p.get("source", "?"), 0) + 1
+
+    print(f"\n{'='*60}", flush=True)
+    print("HARVEST SUMMARY", flush=True)
+    print(f"  Institution : {inst_cfg.name}", flush=True)
+    print(f"  Total SC    : {len(papers)}", flush=True)
+    for src, cnt in sorted(source_counts.items()):
+        print(f"    {src:<22}: {cnt}", flush=True)
+    print(f"  Email       : {args.email}", flush=True)
+    print("  IR deposit  : NOT performed (dry run)", flush=True)
+    print(f"{'='*60}\n", flush=True)
+
+
+if __name__ == "__main__":
+    main()

start_dashboard.py

@@ -1,32 +1,32 @@
-"""
-Simple script to start the URAAS dashboard.
-Handles Python path setup automatically.
-"""
-
-import os
-import sys
-
-# Add project root to Python path
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
-# Now import and run the dashboard
-import uraas
-
-print(f"DEBUG: uraas path: {uraas.__path__}", flush=True)
-from uraas.config import config
-from uraas.dashboard.app import app, socketio
-
-if __name__ == "__main__":
-    print("=" * 70, flush=True)
-    print("URAAS Dashboard Starting...", flush=True)
-    print("=" * 70, flush=True)
-    print(f"Dashboard URL: http://localhost:{config.DASHBOARD_PORT}", flush=True)
-    print("Press Ctrl+C to stop", flush=True)
-    print("=" * 70, flush=True)
-    socketio.run(
-        app,
-        host="0.0.0.0",
-        port=config.DASHBOARD_PORT,
-        debug=False,
-        allow_unsafe_werkzeug=True,
-    )
+"""
+Simple script to start the URAAS dashboard.
+Handles Python path setup automatically.
+"""
+
+import os
+import sys
+
+# Add project root to Python path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+# Now import and run the dashboard
+import uraas
+
+print(f"DEBUG: uraas path: {uraas.__path__}", flush=True)
+from uraas.config import config
+from uraas.dashboard.app import app, socketio
+
+if __name__ == "__main__":
+    print("=" * 70, flush=True)
+    print("URAAS Dashboard Starting...", flush=True)
+    print("=" * 70, flush=True)
+    print(f"Dashboard URL: http://localhost:{config.DASHBOARD_PORT}", flush=True)
+    print("Press Ctrl+C to stop", flush=True)
+    print("=" * 70, flush=True)
+    socketio.run(
+        app,
+        host="0.0.0.0",
+        port=config.DASHBOARD_PORT,
+        debug=False,
+        allow_unsafe_werkzeug=True,
+    )

tests/test_all_spiders.py

@@ -1,202 +1,202 @@
-"""
-Comprehensive test for all multi-institution spiders
-Tests initialization and configuration for all spider types
-"""
-
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
-from uraas.config.institutions import get_registry
-from uraas.spiders.sources.arxiv_spider import ArxivSpider
-from uraas.spiders.sources.crossref_spider import CrossrefSpider
-from uraas.spiders.sources.openalex_spider import OpenAlexSpider
-from uraas.spiders.sources.orcid_spider import ORCIDSpider
-from uraas.spiders.sources.scholar_spider import ScholarSpider
-
-
-def test_all_spiders():
-    """Test that all spiders can be initialized with different institutions"""
-    print("\n" + "=" * 60)
-    print("COMPREHENSIVE SPIDER TEST")
-    print("=" * 60)
-
-    registry = get_registry()
-    institutions = ["unilag", "ui", "oau"]
-
-    spider_classes = {
-        "OpenAlex": OpenAlexSpider,
-        "Crossref": CrossrefSpider,
-        "ArXiv": ArxivSpider,
-        "Scholar": ScholarSpider,
-        "ORCID": ORCIDSpider,
-    }
-
-    results = {}
-
-    for spider_name, spider_class in spider_classes.items():
-        print(f"\n{'='*60}")
-        print(f"Testing {spider_name} Spider")
-        print("=" * 60)
-
-        spider_results = {}
-
-        for inst in institutions:
-            try:
-                config = registry.get(inst)
-                if not config:
-                    print(f"  ✗ {inst}: Configuration not found")
-                    spider_results[inst] = False
-                    continue
-
-                # Try to initialize spider
-                spider = spider_class(institution=inst)
-
-                print(f"  ✓ {inst}: {spider.institution_name}")
-                print(f"    ROR: {spider.ror_id}")
-                print(f"    Staff: {len(config.staff_names)}")
-
-                spider_results[inst] = True
-
-            except Exception as e:
-                print(f"  ✗ {inst}: Failed - {e}")
-                spider_results[inst] = False
-
-        results[spider_name] = spider_results
-
-    # Summary
-    print("\n" + "=" * 60)
-    print("COMPREHENSIVE SUMMARY")
-    print("=" * 60)
-
-    total_tests = len(spider_classes) * len(institutions)
-    passed_tests = sum(
-        1
-        for spider_results in results.values()
-        for success in spider_results.values()
-        if success
-    )
-
-    print(f"\nTotal Tests: {passed_tests}/{total_tests}")
-    print(f"\nResults by Spider:")
-
-    for spider_name, spider_results in results.items():
-        passed = sum(1 for v in spider_results.values() if v)
-        total = len(spider_results)
-        status = "✓" if passed == total else "✗"
-        print(f"  {status} {spider_name}: {passed}/{total}")
-
-        for inst, success in spider_results.items():
-            inst_status = "✓" if success else "✗"
-            print(f"      {inst_status} {inst}")
-
-    if passed_tests == total_tests:
-        print("\n" + "=" * 60)
-        print("✓ ALL SPIDERS READY FOR MULTI-INSTITUTION CRAWLING")
-        print("=" * 60)
-        print("\nNext Steps:")
-        print(
-            "  1. Test crawl: python crawl_multi_institution.py --institutions unilag,ui --target 10 --spider openalex"
-        )
-        print("  2. Verify database: Check for papers with institution_ror tags")
-        print("  3. Test dashboard: Verify multi-institution comparison works")
-        print("  4. Production crawl: Run with higher targets for all institutions")
-        return True
-    else:
-        print("\n✗ SOME TESTS FAILED")
-        return False
-
-
-def test_spider_metadata():
-    """Test that spiders have correct metadata"""
-    print("\n" + "=" * 60)
-    print("SPIDER METADATA TEST")
-    print("=" * 60)
-
-    spider_classes = {
-        "OpenAlex": OpenAlexSpider,
-        "Crossref": CrossrefSpider,
-        "ArXiv": ArxivSpider,
-        "Scholar": ScholarSpider,
-        "ORCID": ORCIDSpider,
-    }
-
-    for spider_name, spider_class in spider_classes.items():
-        spider = spider_class(institution="unilag")
-        print(f"\n{spider_name}:")
-        print(f"  Name: {spider.name}")
-        print(f"  Institution: {spider.institution_name}")
-        print(f"  ROR: {spider.ror_id}")
-
-        # Check for required attributes
-        required_attrs = ["institution_name", "ror_id", "institution_config"]
-        missing = [attr for attr in required_attrs if not hasattr(spider, attr)]
-
-        if missing:
-            print(f"  ✗ Missing attributes: {missing}")
-            return False
-        else:
-            print(f"  ✓ All required attributes present")
-
-    return True
-
-
-def main():
-    """Run all tests"""
-    print("\n" + "=" * 60)
-    print("MULTI-INSTITUTION SPIDER TEST SUITE")
-    print("=" * 60)
-
-    tests = [
-        ("Spider Metadata", test_spider_metadata),
-        ("All Spiders Initialization", test_all_spiders),
-    ]
-
-    results = {}
-
-    for test_name, test_func in tests:
-        try:
-            result = test_func()
-            results[test_name] = result
-        except Exception as e:
-            print(f"\n✗ {test_name} FAILED: {e}")
-            import traceback
-
-            traceback.print_exc()
-            results[test_name] = False
-
-    # Final summary
-    print("\n" + "=" * 60)
-    print("FINAL TEST SUMMARY")
-    print("=" * 60)
-
-    passed = sum(1 for v in results.values() if v)
-    total = len(results)
-
-    print(f"\nTests passed: {passed}/{total}\n")
-
-    for test_name, success in results.items():
-        status = "✓ PASS" if success else "✗ FAIL"
-        print(f"  {status}: {test_name}")
-
-    if passed == total:
-        print("\n" + "=" * 60)
-        print("✓ WEEK 1 DAY 5-7 COMPLETE")
-        print("=" * 60)
-        print("\nAll spiders updated for multi-institution support!")
-        print("\nImplementation Summary:")
-        print("  • 5 spiders updated: OpenAlex, Crossref, ArXiv, Scholar, ORCID")
-        print("  • 5 institutions configured: UNILAG, UI, OAU, UNN, ABU")
-        print("  • 2,146 total staff members loaded")
-        print("  • ROR-based identification implemented")
-        print("  • Backward compatibility maintained")
-        print("\nReady for production crawling!")
-        return 0
-    else:
-        print("\n✗ SOME TESTS FAILED")
-        return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+"""
+Comprehensive test for all multi-institution spiders
+Tests initialization and configuration for all spider types
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from uraas.config.institutions import get_registry
+from uraas.spiders.sources.arxiv_spider import ArxivSpider
+from uraas.spiders.sources.crossref_spider import CrossrefSpider
+from uraas.spiders.sources.openalex_spider import OpenAlexSpider
+from uraas.spiders.sources.orcid_spider import ORCIDSpider
+from uraas.spiders.sources.scholar_spider import ScholarSpider
+
+
+def test_all_spiders():
+    """Test that all spiders can be initialized with different institutions"""
+    print("\n" + "=" * 60)
+    print("COMPREHENSIVE SPIDER TEST")
+    print("=" * 60)
+
+    registry = get_registry()
+    institutions = ["unilag", "ui", "oau"]
+
+    spider_classes = {
+        "OpenAlex": OpenAlexSpider,
+        "Crossref": CrossrefSpider,
+        "ArXiv": ArxivSpider,
+        "Scholar": ScholarSpider,
+        "ORCID": ORCIDSpider,
+    }
+
+    results = {}
+
+    for spider_name, spider_class in spider_classes.items():
+        print(f"\n{'='*60}")
+        print(f"Testing {spider_name} Spider")
+        print("=" * 60)
+
+        spider_results = {}
+
+        for inst in institutions:
+            try:
+                config = registry.get(inst)
+                if not config:
+                    print(f"  ✗ {inst}: Configuration not found")
+                    spider_results[inst] = False
+                    continue
+
+                # Try to initialize spider
+                spider = spider_class(institution=inst)
+
+                print(f"  ✓ {inst}: {spider.institution_name}")
+                print(f"    ROR: {spider.ror_id}")
+                print(f"    Staff: {len(config.staff_names)}")
+
+                spider_results[inst] = True
+
+            except Exception as e:
+                print(f"  ✗ {inst}: Failed - {e}")
+                spider_results[inst] = False
+
+        results[spider_name] = spider_results
+
+    # Summary
+    print("\n" + "=" * 60)
+    print("COMPREHENSIVE SUMMARY")
+    print("=" * 60)
+
+    total_tests = len(spider_classes) * len(institutions)
+    passed_tests = sum(
+        1
+        for spider_results in results.values()
+        for success in spider_results.values()
+        if success
+    )
+
+    print(f"\nTotal Tests: {passed_tests}/{total_tests}")
+    print(f"\nResults by Spider:")
+
+    for spider_name, spider_results in results.items():
+        passed = sum(1 for v in spider_results.values() if v)
+        total = len(spider_results)
+        status = "✓" if passed == total else "✗"
+        print(f"  {status} {spider_name}: {passed}/{total}")
+
+        for inst, success in spider_results.items():
+            inst_status = "✓" if success else "✗"
+            print(f"      {inst_status} {inst}")
+
+    if passed_tests == total_tests:
+        print("\n" + "=" * 60)
+        print("✓ ALL SPIDERS READY FOR MULTI-INSTITUTION CRAWLING")
+        print("=" * 60)
+        print("\nNext Steps:")
+        print(
+            "  1. Test crawl: python crawl_multi_institution.py --institutions unilag,ui --target 10 --spider openalex"
+        )
+        print("  2. Verify database: Check for papers with institution_ror tags")
+        print("  3. Test dashboard: Verify multi-institution comparison works")
+        print("  4. Production crawl: Run with higher targets for all institutions")
+        return True
+    else:
+        print("\n✗ SOME TESTS FAILED")
+        return False
+
+
+def test_spider_metadata():
+    """Test that spiders have correct metadata"""
+    print("\n" + "=" * 60)
+    print("SPIDER METADATA TEST")
+    print("=" * 60)
+
+    spider_classes = {
+        "OpenAlex": OpenAlexSpider,
+        "Crossref": CrossrefSpider,
+        "ArXiv": ArxivSpider,
+        "Scholar": ScholarSpider,
+        "ORCID": ORCIDSpider,
+    }
+
+    for spider_name, spider_class in spider_classes.items():
+        spider = spider_class(institution="unilag")
+        print(f"\n{spider_name}:")
+        print(f"  Name: {spider.name}")
+        print(f"  Institution: {spider.institution_name}")
+        print(f"  ROR: {spider.ror_id}")
+
+        # Check for required attributes
+        required_attrs = ["institution_name", "ror_id", "institution_config"]
+        missing = [attr for attr in required_attrs if not hasattr(spider, attr)]
+
+        if missing:
+            print(f"  ✗ Missing attributes: {missing}")
+            return False
+        else:
+            print(f"  ✓ All required attributes present")
+
+    return True
+
+
+def main():
+    """Run all tests"""
+    print("\n" + "=" * 60)
+    print("MULTI-INSTITUTION SPIDER TEST SUITE")
+    print("=" * 60)
+
+    tests = [
+        ("Spider Metadata", test_spider_metadata),
+        ("All Spiders Initialization", test_all_spiders),
+    ]
+
+    results = {}
+
+    for test_name, test_func in tests:
+        try:
+            result = test_func()
+            results[test_name] = result
+        except Exception as e:
+            print(f"\n✗ {test_name} FAILED: {e}")
+            import traceback
+
+            traceback.print_exc()
+            results[test_name] = False
+
+    # Final summary
+    print("\n" + "=" * 60)
+    print("FINAL TEST SUMMARY")
+    print("=" * 60)
+
+    passed = sum(1 for v in results.values() if v)
+    total = len(results)
+
+    print(f"\nTests passed: {passed}/{total}\n")
+
+    for test_name, success in results.items():
+        status = "✓ PASS" if success else "✗ FAIL"
+        print(f"  {status}: {test_name}")
+
+    if passed == total:
+        print("\n" + "=" * 60)
+        print("✓ WEEK 1 DAY 5-7 COMPLETE")
+        print("=" * 60)
+        print("\nAll spiders updated for multi-institution support!")
+        print("\nImplementation Summary:")
+        print("  • 5 spiders updated: OpenAlex, Crossref, ArXiv, Scholar, ORCID")
+        print("  • 5 institutions configured: UNILAG, UI, OAU, UNN, ABU")
+        print("  • 2,146 total staff members loaded")
+        print("  • ROR-based identification implemented")
+        print("  • Backward compatibility maintained")
+        print("\nReady for production crawling!")
+        return 0
+    else:
+        print("\n✗ SOME TESTS FAILED")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_api.py

@@ -1,473 +1,473 @@
-"""
-URAAS Test Suite  covers every API endpoint and APA analytics metrics.
-Run: pytest tests/test_api.py -v
-"""
-
-import os
-import sys
-
-import pytest
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from uraas.analytics.engine import URAASAnalyticsEngine, analytics
-from uraas.dashboard.app import app as flask_app
-from uraas.database import Author, Collection, Community, Item, SessionLocal
-from uraas.utils.ai_keyword_extractor import ai_extractor
-from uraas.utils.docid_generator import docid_generator
-
-
-@pytest.fixture(scope="module")
-def client():
-    flask_app.config["TESTING"] = True
-    with flask_app.test_client() as c:
-        yield c
-
-
-#  Core page
-
-
-def test_index_loads(client):
-    r = client.get("/")
-    assert r.status_code == 200
-
-
-#  Analytics overview
-
-
-def test_analytics_overview(client):
-    r = client.get("/api/analytics/overview")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert "total_papers" in d
-    assert "total_authors" in d
-    assert "oa_percentage" in d
-    assert isinstance(d["total_papers"], int)
-    assert 0 <= d["oa_percentage"] <= 100
-
-
-def test_publications_by_year(client):
-    r = client.get("/api/analytics/publications-by-year")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert isinstance(d, list)
-    for item in d:
-        assert "year" in item and "count" in item
-        assert isinstance(item["year"], int)
-        assert item["count"] >= 0
-
-
-def test_papers_by_faculty(client):
-    r = client.get("/api/analytics/papers-by-faculty")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert isinstance(d, list)
-
-
-def test_top_authors(client):
-    r = client.get("/api/analytics/top-authors?limit=10")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert isinstance(d, list)
-    assert len(d) <= 10
-
-
-def test_oa_breakdown(client):
-    r = client.get("/api/analytics/open-access-breakdown")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert isinstance(d, list)
-    labels = [x["label"] for x in d]
-    assert "Open Access" in labels
-
-
-def test_recent_papers(client):
-    r = client.get("/api/analytics/recent-papers?limit=5")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert isinstance(d, list)
-    assert len(d) <= 5
-
-
-def test_impact_metrics(client):
-    r = client.get("/api/analytics/impact-metrics")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert "total_papers" in d
-    assert "oa_rate" in d
-    assert "doi_rate" in d
-
-
-def test_faculties_list(client):
-    r = client.get("/api/analytics/faculties")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert isinstance(d, list)
-
-
-# ── Search ────────────────────────────────────────────────────────────────────
-
-
-def test_search_empty(client):
-    r = client.get("/api/analytics/search?q=&limit=10")
-    assert r.status_code == 200
-    assert isinstance(r.get_json(), list)
-
-
-def test_search_with_query(client):
-    r = client.get("/api/analytics/search?q=health&limit=10")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert isinstance(d, list)
-    assert len(d) <= 10
-
-
-def test_search_oa_filter(client):
-    r = client.get("/api/analytics/search?oa_only=true&limit=20")
-    assert r.status_code == 200
-    d = r.get_json()
-    for item in d:
-        assert item["is_oa"] == True
-
-
-def test_search_sql_injection(client):
-    r = client.get("/api/analytics/search?q='; DROP TABLE items; --")
-    assert r.status_code == 200  # should not crash
-
-
-#  Papers tree
-
-
-def test_papers_tree(client):
-    r = client.get("/api/papers/tree")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert "status" in d
-    assert "data" in d
-
-
-#  Paper detail
-
-
-def test_paper_not_found(client):
-    r = client.get("/api/papers/999999")
-    assert r.status_code == 404
-
-
-def test_paper_detail_if_exists(client):
-    session = SessionLocal()
-    try:
-        item = session.query(Item).first()
-        if item:
-            r = client.get(f"/api/papers/{item.id}")
-            assert r.status_code == 200
-            d = r.get_json()
-            assert "title" in d
-            assert "authors" in d
-            assert "dc" in d
-    finally:
-        session.close()
-
-
-#  Keyword cloud
-
-
-def test_keyword_cloud(client):
-    r = client.get("/api/analytics/keyword-cloud")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert isinstance(d, list)
-    for item in d:
-        assert "word" in item
-        assert "count" in item
-        assert "score" in item
-
-
-#  Research trends
-
-
-def test_research_trends(client):
-    r = client.get("/api/analytics/research-trends")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert isinstance(d, list)
-    for item in d:
-        assert "topic" in item
-        assert "total" in item
-        assert "by_year" in item
-
-
-#  Language research
-
-
-def test_language_research(client):
-    r = client.get("/api/analytics/language-research")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert "total_language_papers" in d
-    assert "papers" in d
-    assert "top_keywords" in d
-    # Verify no false positives
-    bad_terms = [
-        "machine learning",
-        "concrete",
-        "cancer",
-        "covid",
-        "petroleum",
-        "galaxy",
-    ]
-    for paper in d["papers"]:
-        title_lower = (paper.get("title") or "").lower()
-        for bad in bad_terms:
-            assert bad not in title_lower, f"False positive: '{bad}' in '{title_lower}'"
-
-
-#  APA Novel Metrics
-
-
-def test_tk_vitality_score(client):
-    r = client.get("/api/analytics/tk-vitality-score")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert "score" in d
-    assert 0 <= d["score"] <= 100
-    assert "breakdown" in d
-    assert "total_items" in d
-
-
-def test_linguistic_diversity_index(client):
-    r = client.get("/api/analytics/linguistic-diversity-index")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert "index" in d
-    assert 0 <= d["index"] <= 100
-    assert "breakdown" in d
-
-
-def test_patent_velocity(client):
-    r = client.get("/api/analytics/patent-velocity")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert "total_patents" in d
-    assert "velocity_distribution" in d
-
-
-def test_docid_coverage(client):
-    r = client.get("/api/analytics/docid-coverage")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert "total_papers" in d
-    assert "docid_assigned" in d
-    assert "coverage_percent" in d
-    assert 0 <= d["coverage_percent"] <= 100
-
-
-def test_docid_stats(client):
-    r = client.get("/api/docid/stats")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert "total_docid_papers" in d
-    assert "docid_coverage" in d
-
-
-#  Author network
-
-
-def test_author_network_global(client):
-    r = client.get("/api/analytics/author-network")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert "nodes" in d
-    assert "edges" in d
-
-
-def test_authors_search(client):
-    r = client.get("/api/analytics/authors-search?q=a&limit=5")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert isinstance(d, list)
-    assert len(d) <= 5
-
-
-#  Faculty comparison
-
-
-def test_faculty_comparison_empty(client):
-    r = client.get("/api/analytics/faculty-comparison")
-    assert r.status_code == 200
-    assert isinstance(r.get_json(), dict)
-
-
-#  Exports
-
-
-def test_export_csv(client):
-    r = client.get("/api/export/papers.csv")
-    assert r.status_code == 200
-    assert "text/csv" in r.content_type
-    data = r.data.decode("utf-8")
-    assert "Title" in data or "ID" in data
-
-
-def test_export_bibtex(client):
-    r = client.get("/api/export/papers.bibtex")
-    assert r.status_code == 200
-
-
-#  Crawler status
-
-
-def test_crawler_status(client):
-    r = client.get("/api/crawler/status")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert d["status"] in ("running", "idle")
-
-
-def test_docid_crawler_status(client):
-    r = client.get("/api/docid-crawler/status")
-    assert r.status_code == 200
-    d = r.get_json()
-    assert d["status"] in ("running", "idle")
-
-
-#  Analytics engine unit tests
-
-
-def test_engine_top_authors():
-    result = analytics.get_top_authors(limit=5)
-    assert isinstance(result, list)
-    assert len(result) <= 5
-    for r in result:
-        assert "author" in r
-        assert "count" in r
-        assert r["count"] > 0
-
-
-def test_engine_sdg_alignment():
-    result = analytics.get_sdg_alignment()
-    assert isinstance(result, list)
-    sdg_names = [r["sdg"] for r in result]
-    # Should have at least some SDGs with papers
-    assert len(result) >= 0
-
-
-def test_engine_keyword_cloud():
-    result = analytics.get_keyword_cloud(top_n=20)
-    assert isinstance(result, list)
-    assert len(result) <= 20
-    for item in result:
-        assert "word" in item
-        assert "score" in item
-        assert item["score"] > 0
-
-
-def test_engine_tk_vitality():
-    result = analytics.get_tk_vitality_score()
-    assert "score" in result
-    assert 0 <= result["score"] <= 100
-
-
-def test_engine_linguistic_diversity():
-    result = analytics.get_linguistic_diversity_index()
-    assert "index" in result
-    assert 0 <= result["index"] <= 100
-
-
-def test_engine_patent_velocity():
-    result = analytics.get_patent_velocity()
-    assert "total_patents" in result
-    assert isinstance(result["total_patents"], int)
-
-
-def test_engine_docid_coverage():
-    result = analytics.get_docid_coverage()
-    assert "coverage_percent" in result
-    assert 0 <= result["coverage_percent"] <= 100
-
-
-#  DocID generator
-
-
-def test_docid_generation():
-    docid = docid_generator.generate_docid("Test Paper Title", doi="10.1234/test")
-    assert docid.startswith("20.500.14351/")
-    parts = docid.split("/")
-    assert len(parts) == 2
-    assert len(parts[1]) >= 10
-
-
-def test_docid_validation():
-    valid = docid_generator.generate_docid("Test")
-    assert docid_generator.validate_docid(valid) == True
-    assert docid_generator.validate_docid("") == False
-    assert docid_generator.validate_docid("invalid") == False
-    assert docid_generator.validate_docid("99.999.99999/abc") == False
-
-
-def test_docid_uniqueness():
-    ids = {docid_generator.generate_docid("Same Title") for _ in range(10)}
-    assert len(ids) == 10  # all unique due to uuid4
-
-
-#  AI keyword extractor
-
-
-def test_keyword_extraction():
-    text = (
-        "machine learning deep neural networks artificial intelligence computer vision"
-    )
-    kws = ai_extractor.extract_keywords(text, top_n=5)
-    assert len(kws) > 0
-    assert all(isinstance(k, tuple) and len(k) == 2 for k in kws)
-
-
-def test_keyword_extraction_empty():
-    assert ai_extractor.extract_keywords("", top_n=5) == []
-
-
-def test_domain_classification():
-    text = "algorithm data structure programming software engineering database"
-    domains = ai_extractor.classify_domain(text)
-    assert len(domains) > 0
-    assert domains[0][0] == "computer_science"
-
-
-def test_paper_scoring():
-    score = ai_extractor.score_paper(
-        "Machine Learning for Medical Diagnosis",
-        "This study investigates machine learning algorithms for medical diagnosis using deep neural networks to classify medical images with significant improvement over existing methods.",
-    )
-    assert "quality_score" in score
-    assert 0 <= score["quality_score"] <= 1
-    assert "keywords" in score
-
-
-#  Performance
-
-
-def test_overview_response_time(client):
-    import time
-
-    start = time.time()
-    client.get("/api/analytics/overview")
-    elapsed = time.time() - start
-    assert elapsed < 3.0, f"Overview took {elapsed:.2f}s, should be < 3s"
-
-
-def test_search_response_time(client):
-    import time
-
-    start = time.time()
-    client.get("/api/analytics/search?q=health&limit=20")
-    elapsed = time.time() - start
-    assert elapsed < 5.0, f"Search took {elapsed:.2f}s, should be < 5s"
-
-
-def test_keyword_cloud_response_time(client):
-    import time
-
-    start = time.time()
-    client.get("/api/analytics/keyword-cloud")
-    elapsed = time.time() - start
-    assert elapsed < 10.0, f"Keyword cloud took {elapsed:.2f}s, should be < 10s"
+"""
+URAAS Test Suite  covers every API endpoint and APA analytics metrics.
+Run: pytest tests/test_api.py -v
+"""
+
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from uraas.analytics.engine import URAASAnalyticsEngine, analytics
+from uraas.dashboard.app import app as flask_app
+from uraas.database import Author, Collection, Community, Item, SessionLocal
+from uraas.utils.ai_keyword_extractor import ai_extractor
+from uraas.utils.docid_generator import docid_generator
+
+
+@pytest.fixture(scope="module")
+def client():
+    flask_app.config["TESTING"] = True
+    with flask_app.test_client() as c:
+        yield c
+
+
+#  Core page
+
+
+def test_index_loads(client):
+    r = client.get("/")
+    assert r.status_code == 200
+
+
+#  Analytics overview
+
+
+def test_analytics_overview(client):
+    r = client.get("/api/analytics/overview")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert "total_papers" in d
+    assert "total_authors" in d
+    assert "oa_percentage" in d
+    assert isinstance(d["total_papers"], int)
+    assert 0 <= d["oa_percentage"] <= 100
+
+
+def test_publications_by_year(client):
+    r = client.get("/api/analytics/publications-by-year")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert isinstance(d, list)
+    for item in d:
+        assert "year" in item and "count" in item
+        assert isinstance(item["year"], int)
+        assert item["count"] >= 0
+
+
+def test_papers_by_faculty(client):
+    r = client.get("/api/analytics/papers-by-faculty")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert isinstance(d, list)
+
+
+def test_top_authors(client):
+    r = client.get("/api/analytics/top-authors?limit=10")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert isinstance(d, list)
+    assert len(d) <= 10
+
+
+def test_oa_breakdown(client):
+    r = client.get("/api/analytics/open-access-breakdown")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert isinstance(d, list)
+    labels = [x["label"] for x in d]
+    assert "Open Access" in labels
+
+
+def test_recent_papers(client):
+    r = client.get("/api/analytics/recent-papers?limit=5")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert isinstance(d, list)
+    assert len(d) <= 5
+
+
+def test_impact_metrics(client):
+    r = client.get("/api/analytics/impact-metrics")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert "total_papers" in d
+    assert "oa_rate" in d
+    assert "doi_rate" in d
+
+
+def test_faculties_list(client):
+    r = client.get("/api/analytics/faculties")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert isinstance(d, list)
+
+
+# ── Search ────────────────────────────────────────────────────────────────────
+
+
+def test_search_empty(client):
+    r = client.get("/api/analytics/search?q=&limit=10")
+    assert r.status_code == 200
+    assert isinstance(r.get_json(), list)
+
+
+def test_search_with_query(client):
+    r = client.get("/api/analytics/search?q=health&limit=10")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert isinstance(d, list)
+    assert len(d) <= 10
+
+
+def test_search_oa_filter(client):
+    r = client.get("/api/analytics/search?oa_only=true&limit=20")
+    assert r.status_code == 200
+    d = r.get_json()
+    for item in d:
+        assert item["is_oa"] == True
+
+
+def test_search_sql_injection(client):
+    r = client.get("/api/analytics/search?q='; DROP TABLE items; --")
+    assert r.status_code == 200  # should not crash
+
+
+#  Papers tree
+
+
+def test_papers_tree(client):
+    r = client.get("/api/papers/tree")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert "status" in d
+    assert "data" in d
+
+
+#  Paper detail
+
+
+def test_paper_not_found(client):
+    r = client.get("/api/papers/999999")
+    assert r.status_code == 404
+
+
+def test_paper_detail_if_exists(client):
+    session = SessionLocal()
+    try:
+        item = session.query(Item).first()
+        if item:
+            r = client.get(f"/api/papers/{item.id}")
+            assert r.status_code == 200
+            d = r.get_json()
+            assert "title" in d
+            assert "authors" in d
+            assert "dc" in d
+    finally:
+        session.close()
+
+
+#  Keyword cloud
+
+
+def test_keyword_cloud(client):
+    r = client.get("/api/analytics/keyword-cloud")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert isinstance(d, list)
+    for item in d:
+        assert "word" in item
+        assert "count" in item
+        assert "score" in item
+
+
+#  Research trends
+
+
+def test_research_trends(client):
+    r = client.get("/api/analytics/research-trends")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert isinstance(d, list)
+    for item in d:
+        assert "topic" in item
+        assert "total" in item
+        assert "by_year" in item
+
+
+#  Language research
+
+
+def test_language_research(client):
+    r = client.get("/api/analytics/language-research")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert "total_language_papers" in d
+    assert "papers" in d
+    assert "top_keywords" in d
+    # Verify no false positives
+    bad_terms = [
+        "machine learning",
+        "concrete",
+        "cancer",
+        "covid",
+        "petroleum",
+        "galaxy",
+    ]
+    for paper in d["papers"]:
+        title_lower = (paper.get("title") or "").lower()
+        for bad in bad_terms:
+            assert bad not in title_lower, f"False positive: '{bad}' in '{title_lower}'"
+
+
+#  APA Novel Metrics
+
+
+def test_tk_vitality_score(client):
+    r = client.get("/api/analytics/tk-vitality-score")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert "score" in d
+    assert 0 <= d["score"] <= 100
+    assert "breakdown" in d
+    assert "total_items" in d
+
+
+def test_linguistic_diversity_index(client):
+    r = client.get("/api/analytics/linguistic-diversity-index")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert "index" in d
+    assert 0 <= d["index"] <= 100
+    assert "breakdown" in d
+
+
+def test_patent_velocity(client):
+    r = client.get("/api/analytics/patent-velocity")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert "total_patents" in d
+    assert "velocity_distribution" in d
+
+
+def test_docid_coverage(client):
+    r = client.get("/api/analytics/docid-coverage")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert "total_papers" in d
+    assert "docid_assigned" in d
+    assert "coverage_percent" in d
+    assert 0 <= d["coverage_percent"] <= 100
+
+
+def test_docid_stats(client):
+    r = client.get("/api/docid/stats")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert "total_docid_papers" in d
+    assert "docid_coverage" in d
+
+
+#  Author network
+
+
+def test_author_network_global(client):
+    r = client.get("/api/analytics/author-network")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert "nodes" in d
+    assert "edges" in d
+
+
+def test_authors_search(client):
+    r = client.get("/api/analytics/authors-search?q=a&limit=5")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert isinstance(d, list)
+    assert len(d) <= 5
+
+
+#  Faculty comparison
+
+
+def test_faculty_comparison_empty(client):
+    r = client.get("/api/analytics/faculty-comparison")
+    assert r.status_code == 200
+    assert isinstance(r.get_json(), dict)
+
+
+#  Exports
+
+
+def test_export_csv(client):
+    r = client.get("/api/export/papers.csv")
+    assert r.status_code == 200
+    assert "text/csv" in r.content_type
+    data = r.data.decode("utf-8")
+    assert "Title" in data or "ID" in data
+
+
+def test_export_bibtex(client):
+    r = client.get("/api/export/papers.bibtex")
+    assert r.status_code == 200
+
+
+#  Crawler status
+
+
+def test_crawler_status(client):
+    r = client.get("/api/crawler/status")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert d["status"] in ("running", "idle")
+
+
+def test_docid_crawler_status(client):
+    r = client.get("/api/docid-crawler/status")
+    assert r.status_code == 200
+    d = r.get_json()
+    assert d["status"] in ("running", "idle")
+
+
+#  Analytics engine unit tests
+
+
+def test_engine_top_authors():
+    result = analytics.get_top_authors(limit=5)
+    assert isinstance(result, list)
+    assert len(result) <= 5
+    for r in result:
+        assert "author" in r
+        assert "count" in r
+        assert r["count"] > 0
+
+
+def test_engine_sdg_alignment():
+    result = analytics.get_sdg_alignment()
+    assert isinstance(result, list)
+    sdg_names = [r["sdg"] for r in result]
+    # Should have at least some SDGs with papers
+    assert len(result) >= 0
+
+
+def test_engine_keyword_cloud():
+    result = analytics.get_keyword_cloud(top_n=20)
+    assert isinstance(result, list)
+    assert len(result) <= 20
+    for item in result:
+        assert "word" in item
+        assert "score" in item
+        assert item["score"] > 0
+
+
+def test_engine_tk_vitality():
+    result = analytics.get_tk_vitality_score()
+    assert "score" in result
+    assert 0 <= result["score"] <= 100
+
+
+def test_engine_linguistic_diversity():
+    result = analytics.get_linguistic_diversity_index()
+    assert "index" in result
+    assert 0 <= result["index"] <= 100
+
+
+def test_engine_patent_velocity():
+    result = analytics.get_patent_velocity()
+    assert "total_patents" in result
+    assert isinstance(result["total_patents"], int)
+
+
+def test_engine_docid_coverage():
+    result = analytics.get_docid_coverage()
+    assert "coverage_percent" in result
+    assert 0 <= result["coverage_percent"] <= 100
+
+
+#  DocID generator
+
+
+def test_docid_generation():
+    docid = docid_generator.generate_docid("Test Paper Title", doi="10.1234/test")
+    assert docid.startswith("20.500.14351/")
+    parts = docid.split("/")
+    assert len(parts) == 2
+    assert len(parts[1]) >= 10
+
+
+def test_docid_validation():
+    valid = docid_generator.generate_docid("Test")
+    assert docid_generator.validate_docid(valid) == True
+    assert docid_generator.validate_docid("") == False
+    assert docid_generator.validate_docid("invalid") == False
+    assert docid_generator.validate_docid("99.999.99999/abc") == False
+
+
+def test_docid_uniqueness():
+    ids = {docid_generator.generate_docid("Same Title") for _ in range(10)}
+    assert len(ids) == 10  # all unique due to uuid4
+
+
+#  AI keyword extractor
+
+
+def test_keyword_extraction():
+    text = (
+        "machine learning deep neural networks artificial intelligence computer vision"
+    )
+    kws = ai_extractor.extract_keywords(text, top_n=5)
+    assert len(kws) > 0
+    assert all(isinstance(k, tuple) and len(k) == 2 for k in kws)
+
+
+def test_keyword_extraction_empty():
+    assert ai_extractor.extract_keywords("", top_n=5) == []
+
+
+def test_domain_classification():
+    text = "algorithm data structure programming software engineering database"
+    domains = ai_extractor.classify_domain(text)
+    assert len(domains) > 0
+    assert domains[0][0] == "computer_science"
+
+
+def test_paper_scoring():
+    score = ai_extractor.score_paper(
+        "Machine Learning for Medical Diagnosis",
+        "This study investigates machine learning algorithms for medical diagnosis using deep neural networks to classify medical images with significant improvement over existing methods.",
+    )
+    assert "quality_score" in score
+    assert 0 <= score["quality_score"] <= 1
+    assert "keywords" in score
+
+
+#  Performance
+
+
+def test_overview_response_time(client):
+    import time
+
+    start = time.time()
+    client.get("/api/analytics/overview")
+    elapsed = time.time() - start
+    assert elapsed < 3.0, f"Overview took {elapsed:.2f}s, should be < 3s"
+
+
+def test_search_response_time(client):
+    import time
+
+    start = time.time()
+    client.get("/api/analytics/search?q=health&limit=20")
+    elapsed = time.time() - start
+    assert elapsed < 5.0, f"Search took {elapsed:.2f}s, should be < 5s"
+
+
+def test_keyword_cloud_response_time(client):
+    import time
+
+    start = time.time()
+    client.get("/api/analytics/keyword-cloud")
+    elapsed = time.time() - start
+    assert elapsed < 10.0, f"Keyword cloud took {elapsed:.2f}s, should be < 10s"

tests/test_multi_institution.py

@@ -1,175 +1,175 @@
-"""
-Test script for multi-institution support
-Tests institution configuration and staff validation
-"""
-
-import os
-import sys
-
-import pytest
-
-# Add project root to path
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
-from uraas.config.institutions import InstitutionRegistry, get_registry
-from uraas.utils.staff_validator import StaffValidator
-
-
-@pytest.fixture
-def registry():
-    """Provide the institution registry as a pytest fixture."""
-    return get_registry()
-
-
-def test_institution_registry():
-    """Test institution registry loading"""
-    print("=" * 60)
-    print("TEST 1: Institution Registry")
-    print("=" * 60)
-
-    registry = get_registry()
-
-    print(f"\nLoaded {len(registry.institutions)} institutions:")
-    for config in registry.list_all():
-        print(f"  - {config.name} ({config.short_name})")
-        print(f"    ROR: {config.ror}")
-        print(f"    Country: {config.country}")
-        print(f"    Staff count: {len(config.staff_names)}")
-        print(f"    Affiliation patterns: {len(config.affiliation_patterns)}")
-        print()
-
-    # Test retrieval by short name
-    print("\nTest retrieval by short name:")
-    unilag = registry.get("unilag")
-    if unilag:
-        print(f"  ✓ Found UNILAG: {unilag.name}")
-    else:
-        print(f"  ✗ UNILAG not found")
-
-    # Test retrieval by ROR
-    print("\nTest retrieval by ROR:")
-    ui = registry.get_by_ror("https://ror.org/01js2sh04")
-    if ui:
-        print(f"  ✓ Found UI: {ui.name}")
-    else:
-        print(f"  ✗ UI not found")
-
-    # Test affiliation matching
-    print("\nTest affiliation matching:")
-    test_affiliations = [
-        ("University of Lagos, Nigeria", "unilag"),
-        ("Department of Physics, University of Ibadan", "ui"),
-        ("OAU Ile-Ife, Nigeria", "oau"),
-        ("Ahmadu Bello University, Zaria", "abu"),
-    ]
-
-    for affiliation, expected_short_name in test_affiliations:
-        matched = False
-        for config in registry.list_all():
-            if config.matches_affiliation(affiliation):
-                print(f"  ✓ '{affiliation}' → {config.short_name}")
-                if config.short_name.lower() == expected_short_name.lower():
-                    matched = True
-                break
-        if not matched:
-            print(f"  ✗ '{affiliation}' not matched correctly")
-
-    return registry
-
-
-def test_staff_validator(registry):
-    """Test staff validator with multi-institution support"""
-    print("\n" + "=" * 60)
-    print("TEST 2: Staff Validator")
-    print("=" * 60)
-
-    # Test UNILAG validator
-    print("\nTesting UNILAG validator:")
-    unilag_config = registry.get("unilag")
-    if unilag_config:
-        validator = StaffValidator(institution_config=unilag_config)
-        print(f"  Institution: {validator.institution_name}")
-        print(f"  ROR: {validator.ror}")
-        print(f"  Staff count: {len(validator.staff_names)}")
-
-        # Test some known UNILAG staff (if any)
-        test_authors = [
-            "Prof. A. O. Adeyemi",
-            "Dr. John Smith",  # Should not match
-            "O. A. Ogunlana",
-        ]
-
-        print("\n  Testing author validation:")
-        for author in test_authors:
-            is_staff = validator.is_staff_member(author)
-            print(
-                f"    {'✓' if is_staff else '✗'} {author}: {'Staff' if is_staff else 'Not staff'}"
-            )
-
-    # Test UI validator (will have empty staff list for now)
-    print("\nTesting UI validator:")
-    ui_config = registry.get("ui")
-    if ui_config:
-        validator = StaffValidator(institution_config=ui_config)
-        print(f"  Institution: {validator.institution_name}")
-        print(f"  ROR: {validator.ror}")
-        print(f"  Staff count: {len(validator.staff_names)}")
-        print(f"  Note: Staff file not yet populated")
-
-
-def test_backward_compatibility():
-    """Test that old code still works"""
-    print("\n" + "=" * 60)
-    print("TEST 3: Backward Compatibility")
-    print("=" * 60)
-
-    # Test default validator (should still work for UNILAG)
-    from uraas.utils.staff_validator import staff_validator
-
-    print(f"\nDefault validator:")
-    print(f"  Institution: {staff_validator.institution_name}")
-    print(f"  Staff count: {len(staff_validator.staff_names)}")
-    print(f"  ✓ Backward compatibility maintained")
-
-
-def main():
-    """Run all tests"""
-    print("\n" + "=" * 60)
-    print("MULTI-INSTITUTION SUPPORT TEST SUITE")
-    print("=" * 60)
-
-    try:
-        # Test 1: Institution Registry
-        registry = test_institution_registry()
-
-        # Test 2: Staff Validator
-        test_staff_validator(registry)
-
-        # Test 3: Backward Compatibility
-        test_backward_compatibility()
-
-        print("\n" + "=" * 60)
-        print("ALL TESTS COMPLETED")
-        print("=" * 60)
-        print("\nSummary:")
-        print(f"  - {len(registry.institutions)} institutions configured")
-        print(f"  - Institution registry operational")
-        print(f"  - Staff validator supports multi-institution")
-        print(f"  - Backward compatibility maintained")
-        print("\nNext steps:")
-        print("  1. Populate staff files for UI, OAU, UNN, ABU")
-        print("  2. Update spiders to accept institution parameter")
-        print("  3. Test multi-institution crawling")
-
-    except Exception as e:
-        print(f"\n✗ TEST FAILED: {e}")
-        import traceback
-
-        traceback.print_exc()
-        return 1
-
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+"""
+Test script for multi-institution support
+Tests institution configuration and staff validation
+"""
+
+import os
+import sys
+
+import pytest
+
+# Add project root to path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from uraas.config.institutions import InstitutionRegistry, get_registry
+from uraas.utils.staff_validator import StaffValidator
+
+
+@pytest.fixture
+def registry():
+    """Provide the institution registry as a pytest fixture."""
+    return get_registry()
+
+
+def test_institution_registry():
+    """Test institution registry loading"""
+    print("=" * 60)
+    print("TEST 1: Institution Registry")
+    print("=" * 60)
+
+    registry = get_registry()
+
+    print(f"\nLoaded {len(registry.institutions)} institutions:")
+    for config in registry.list_all():
+        print(f"  - {config.name} ({config.short_name})")
+        print(f"    ROR: {config.ror}")
+        print(f"    Country: {config.country}")
+        print(f"    Staff count: {len(config.staff_names)}")
+        print(f"    Affiliation patterns: {len(config.affiliation_patterns)}")
+        print()
+
+    # Test retrieval by short name
+    print("\nTest retrieval by short name:")
+    unilag = registry.get("unilag")
+    if unilag:
+        print(f"  ✓ Found UNILAG: {unilag.name}")
+    else:
+        print(f"  ✗ UNILAG not found")
+
+    # Test retrieval by ROR
+    print("\nTest retrieval by ROR:")
+    ui = registry.get_by_ror("https://ror.org/01js2sh04")
+    if ui:
+        print(f"  ✓ Found UI: {ui.name}")
+    else:
+        print(f"  ✗ UI not found")
+
+    # Test affiliation matching
+    print("\nTest affiliation matching:")
+    test_affiliations = [
+        ("University of Lagos, Nigeria", "unilag"),
+        ("Department of Physics, University of Ibadan", "ui"),
+        ("OAU Ile-Ife, Nigeria", "oau"),
+        ("Ahmadu Bello University, Zaria", "abu"),
+    ]
+
+    for affiliation, expected_short_name in test_affiliations:
+        matched = False
+        for config in registry.list_all():
+            if config.matches_affiliation(affiliation):
+                print(f"  ✓ '{affiliation}' → {config.short_name}")
+                if config.short_name.lower() == expected_short_name.lower():
+                    matched = True
+                break
+        if not matched:
+            print(f"  ✗ '{affiliation}' not matched correctly")
+
+    return registry
+
+
+def test_staff_validator(registry):
+    """Test staff validator with multi-institution support"""
+    print("\n" + "=" * 60)
+    print("TEST 2: Staff Validator")
+    print("=" * 60)
+
+    # Test UNILAG validator
+    print("\nTesting UNILAG validator:")
+    unilag_config = registry.get("unilag")
+    if unilag_config:
+        validator = StaffValidator(institution_config=unilag_config)
+        print(f"  Institution: {validator.institution_name}")
+        print(f"  ROR: {validator.ror}")
+        print(f"  Staff count: {len(validator.staff_names)}")
+
+        # Test some known UNILAG staff (if any)
+        test_authors = [
+            "Prof. A. O. Adeyemi",
+            "Dr. John Smith",  # Should not match
+            "O. A. Ogunlana",
+        ]
+
+        print("\n  Testing author validation:")
+        for author in test_authors:
+            is_staff = validator.is_staff_member(author)
+            print(
+                f"    {'✓' if is_staff else '✗'} {author}: {'Staff' if is_staff else 'Not staff'}"
+            )
+
+    # Test UI validator (will have empty staff list for now)
+    print("\nTesting UI validator:")
+    ui_config = registry.get("ui")
+    if ui_config:
+        validator = StaffValidator(institution_config=ui_config)
+        print(f"  Institution: {validator.institution_name}")
+        print(f"  ROR: {validator.ror}")
+        print(f"  Staff count: {len(validator.staff_names)}")
+        print(f"  Note: Staff file not yet populated")
+
+
+def test_backward_compatibility():
+    """Test that old code still works"""
+    print("\n" + "=" * 60)
+    print("TEST 3: Backward Compatibility")
+    print("=" * 60)
+
+    # Test default validator (should still work for UNILAG)
+    from uraas.utils.staff_validator import staff_validator
+
+    print(f"\nDefault validator:")
+    print(f"  Institution: {staff_validator.institution_name}")
+    print(f"  Staff count: {len(staff_validator.staff_names)}")
+    print(f"  ✓ Backward compatibility maintained")
+
+
+def main():
+    """Run all tests"""
+    print("\n" + "=" * 60)
+    print("MULTI-INSTITUTION SUPPORT TEST SUITE")
+    print("=" * 60)
+
+    try:
+        # Test 1: Institution Registry
+        registry = test_institution_registry()
+
+        # Test 2: Staff Validator
+        test_staff_validator(registry)
+
+        # Test 3: Backward Compatibility
+        test_backward_compatibility()
+
+        print("\n" + "=" * 60)
+        print("ALL TESTS COMPLETED")
+        print("=" * 60)
+        print("\nSummary:")
+        print(f"  - {len(registry.institutions)} institutions configured")
+        print(f"  - Institution registry operational")
+        print(f"  - Staff validator supports multi-institution")
+        print(f"  - Backward compatibility maintained")
+        print("\nNext steps:")
+        print("  1. Populate staff files for UI, OAU, UNN, ABU")
+        print("  2. Update spiders to accept institution parameter")
+        print("  3. Test multi-institution crawling")
+
+    except Exception as e:
+        print(f"\n✗ TEST FAILED: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_multi_institution_crawl.py

@@ -1,207 +1,207 @@
-"""
-Test multi-institution crawling functionality
-Tests spider initialization and basic crawling for multiple institutions
-"""
-
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
-from uraas.config.institutions import get_registry
-from uraas.spiders.sources.openalex_spider import OpenAlexSpider
-
-
-def test_spider_initialization():
-    """Test that spiders can be initialized with different institutions"""
-    print("\n" + "=" * 60)
-    print("TEST: Spider Initialization")
-    print("=" * 60)
-
-    registry = get_registry()
-    institutions = ["unilag", "ui", "oau", "unn", "abu"]
-
-    results = {}
-
-    for inst in institutions:
-        try:
-            config = registry.get(inst)
-            if not config:
-                print(f"\n✗ {inst}: Configuration not found")
-                results[inst] = False
-                continue
-
-            # Try to initialize spider
-            spider = OpenAlexSpider(institution=inst)
-
-            print(f"\n✓ {inst}: {spider.institution_name}")
-            print(f"  ROR: {spider.ror_id}")
-            print(f"  ROR Short: {spider.ror_short}")
-            print(f"  Staff count: {len(config.staff_names)}")
-
-            results[inst] = True
-
-        except Exception as e:
-            print(f"\n✗ {inst}: Failed to initialize - {e}")
-            results[inst] = False
-
-    # Summary
-    print("\n" + "=" * 60)
-    print("INITIALIZATION SUMMARY")
-    print("=" * 60)
-
-    passed = sum(1 for v in results.values() if v)
-    total = len(results)
-
-    print(f"\nPassed: {passed}/{total}")
-
-    for inst, success in results.items():
-        status = "✓" if success else "✗"
-        print(f"  {status} {inst}")
-
-    return passed == total
-
-
-def test_affiliation_filter():
-    """Test affiliation filter with multi-institution support"""
-    print("\n" + "=" * 60)
-    print("TEST: Affiliation Filter")
-    print("=" * 60)
-
-    from uraas.config.institutions import get_registry
-    from uraas.pipelines.affiliation_filter import AffiliationFilterPipeline
-
-    # Create mock spider for each institution
-    class MockSpider:
-        def __init__(self, institution):
-            self.institution = institution
-            self.logger = MockLogger()
-
-    class MockLogger:
-        def info(self, msg):
-            pass
-
-        def warning(self, msg):
-            pass
-
-        def error(self, msg):
-            pass
-
-    registry = get_registry()
-    institutions = ["unilag", "ui"]
-
-    for inst in institutions:
-        config = registry.get(inst)
-        if not config:
-            continue
-
-        print(f"\n{config.name}:")
-
-        # Create pipeline
-        pipeline = AffiliationFilterPipeline()
-        spider = MockSpider(inst)
-        pipeline.open_spider(spider)
-
-        print(f"  Institution: {pipeline.current_institution.name}")
-        print(f"  Staff count: {len(pipeline.current_validator.staff_names)}")
-        print(f"  Patterns: {len(pipeline.current_patterns)}")
-
-        # Test affiliation matching
-        test_texts = [
-            (f"{config.name}, Nigeria", True),
-            (f"Department of Physics, {config.name}", True),
-            ("Random University", False),
-        ]
-
-        print(f"  Affiliation matching:")
-        for text, expected in test_texts:
-            result = pipeline.is_institution_affiliated(text)
-            status = "✓" if result == expected else "✗"
-            print(f"    {status} '{text}' → {result}")
-
-    return True
-
-
-def test_ror_extraction():
-    """Test ROR ID extraction from URLs"""
-    print("\n" + "=" * 60)
-    print("TEST: ROR ID Extraction")
-    print("=" * 60)
-
-    test_cases = [
-        ("https://ror.org/03qcnxw14", "03qcnxw14"),
-        ("https://ror.org/01js2sh04", "01js2sh04"),
-        ("https://ror.org/03yp73w09", "03yp73w09"),
-    ]
-
-    all_passed = True
-
-    for ror_url, expected_short in test_cases:
-        short = ror_url.split("/")[-1]
-        passed = short == expected_short
-        status = "✓" if passed else "✗"
-        print(f"  {status} {ror_url} → {short}")
-
-        if not passed:
-            all_passed = False
-
-    return all_passed
-
-
-def main():
-    """Run all tests"""
-    print("\n" + "=" * 60)
-    print("MULTI-INSTITUTION CRAWL TEST SUITE")
-    print("=" * 60)
-
-    tests = [
-        ("Spider Initialization", test_spider_initialization),
-        ("Affiliation Filter", test_affiliation_filter),
-        ("ROR Extraction", test_ror_extraction),
-    ]
-
-    results = {}
-
-    for test_name, test_func in tests:
-        try:
-            result = test_func()
-            results[test_name] = result
-        except Exception as e:
-            print(f"\n✗ {test_name} FAILED: {e}")
-            import traceback
-
-            traceback.print_exc()
-            results[test_name] = False
-
-    # Final summary
-    print("\n" + "=" * 60)
-    print("FINAL SUMMARY")
-    print("=" * 60)
-
-    passed = sum(1 for v in results.values() if v)
-    total = len(results)
-
-    print(f"\nTests passed: {passed}/{total}\n")
-
-    for test_name, success in results.items():
-        status = "✓ PASS" if success else "✗ FAIL"
-        print(f"  {status}: {test_name}")
-
-    if passed == total:
-        print("\n✓ ALL TESTS PASSED")
-        print("\nReady for production crawling!")
-        print("\nNext steps:")
-        print(
-            "  1. Run: python crawl_multi_institution.py --institutions unilag,ui --target 10"
-        )
-        print("  2. Monitor database for new papers with ROR tags")
-        print("  3. Verify multi-institution comparison in dashboard")
-        return 0
-    else:
-        print("\n✗ SOME TESTS FAILED")
-        print("\nPlease fix issues before proceeding.")
-        return 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())
+"""
+Test multi-institution crawling functionality
+Tests spider initialization and basic crawling for multiple institutions
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from uraas.config.institutions import get_registry
+from uraas.spiders.sources.openalex_spider import OpenAlexSpider
+
+
+def test_spider_initialization():
+    """Test that spiders can be initialized with different institutions"""
+    print("\n" + "=" * 60)
+    print("TEST: Spider Initialization")
+    print("=" * 60)
+
+    registry = get_registry()
+    institutions = ["unilag", "ui", "oau", "unn", "abu"]
+
+    results = {}
+
+    for inst in institutions:
+        try:
+            config = registry.get(inst)
+            if not config:
+                print(f"\n✗ {inst}: Configuration not found")
+                results[inst] = False
+                continue
+
+            # Try to initialize spider
+            spider = OpenAlexSpider(institution=inst)
+
+            print(f"\n✓ {inst}: {spider.institution_name}")
+            print(f"  ROR: {spider.ror_id}")
+            print(f"  ROR Short: {spider.ror_short}")
+            print(f"  Staff count: {len(config.staff_names)}")
+
+            results[inst] = True
+
+        except Exception as e:
+            print(f"\n✗ {inst}: Failed to initialize - {e}")
+            results[inst] = False
+
+    # Summary
+    print("\n" + "=" * 60)
+    print("INITIALIZATION SUMMARY")
+    print("=" * 60)
+
+    passed = sum(1 for v in results.values() if v)
+    total = len(results)
+
+    print(f"\nPassed: {passed}/{total}")
+
+    for inst, success in results.items():
+        status = "✓" if success else "✗"
+        print(f"  {status} {inst}")
+
+    return passed == total
+
+
+def test_affiliation_filter():
+    """Test affiliation filter with multi-institution support"""
+    print("\n" + "=" * 60)
+    print("TEST: Affiliation Filter")
+    print("=" * 60)
+
+    from uraas.config.institutions import get_registry
+    from uraas.pipelines.affiliation_filter import AffiliationFilterPipeline
+
+    # Create mock spider for each institution
+    class MockSpider:
+        def __init__(self, institution):
+            self.institution = institution
+            self.logger = MockLogger()
+
+    class MockLogger:
+        def info(self, msg):
+            pass
+
+        def warning(self, msg):
+            pass
+
+        def error(self, msg):
+            pass
+
+    registry = get_registry()
+    institutions = ["unilag", "ui"]
+
+    for inst in institutions:
+        config = registry.get(inst)
+        if not config:
+            continue
+
+        print(f"\n{config.name}:")
+
+        # Create pipeline
+        pipeline = AffiliationFilterPipeline()
+        spider = MockSpider(inst)
+        pipeline.open_spider(spider)
+
+        print(f"  Institution: {pipeline.current_institution.name}")
+        print(f"  Staff count: {len(pipeline.current_validator.staff_names)}")
+        print(f"  Patterns: {len(pipeline.current_patterns)}")
+
+        # Test affiliation matching
+        test_texts = [
+            (f"{config.name}, Nigeria", True),
+            (f"Department of Physics, {config.name}", True),
+            ("Random University", False),
+        ]
+
+        print(f"  Affiliation matching:")
+        for text, expected in test_texts:
+            result = pipeline.is_institution_affiliated(text)
+            status = "✓" if result == expected else "✗"
+            print(f"    {status} '{text}' → {result}")
+
+    return True
+
+
+def test_ror_extraction():
+    """Test ROR ID extraction from URLs"""
+    print("\n" + "=" * 60)
+    print("TEST: ROR ID Extraction")
+    print("=" * 60)
+
+    test_cases = [
+        ("https://ror.org/03qcnxw14", "03qcnxw14"),
+        ("https://ror.org/01js2sh04", "01js2sh04"),
+        ("https://ror.org/03yp73w09", "03yp73w09"),
+    ]
+
+    all_passed = True
+
+    for ror_url, expected_short in test_cases:
+        short = ror_url.split("/")[-1]
+        passed = short == expected_short
+        status = "✓" if passed else "✗"
+        print(f"  {status} {ror_url} → {short}")
+
+        if not passed:
+            all_passed = False
+
+    return all_passed
+
+
+def main():
+    """Run all tests"""
+    print("\n" + "=" * 60)
+    print("MULTI-INSTITUTION CRAWL TEST SUITE")
+    print("=" * 60)
+
+    tests = [
+        ("Spider Initialization", test_spider_initialization),
+        ("Affiliation Filter", test_affiliation_filter),
+        ("ROR Extraction", test_ror_extraction),
+    ]
+
+    results = {}
+
+    for test_name, test_func in tests:
+        try:
+            result = test_func()
+            results[test_name] = result
+        except Exception as e:
+            print(f"\n✗ {test_name} FAILED: {e}")
+            import traceback
+
+            traceback.print_exc()
+            results[test_name] = False
+
+    # Final summary
+    print("\n" + "=" * 60)
+    print("FINAL SUMMARY")
+    print("=" * 60)
+
+    passed = sum(1 for v in results.values() if v)
+    total = len(results)
+
+    print(f"\nTests passed: {passed}/{total}\n")
+
+    for test_name, success in results.items():
+        status = "✓ PASS" if success else "✗ FAIL"
+        print(f"  {status}: {test_name}")
+
+    if passed == total:
+        print("\n✓ ALL TESTS PASSED")
+        print("\nReady for production crawling!")
+        print("\nNext steps:")
+        print(
+            "  1. Run: python crawl_multi_institution.py --institutions unilag,ui --target 10"
+        )
+        print("  2. Monitor database for new papers with ROR tags")
+        print("  3. Verify multi-institution comparison in dashboard")
+        return 0
+    else:
+        print("\n✗ SOME TESTS FAILED")
+        print("\nPlease fix issues before proceeding.")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_new_features.py

@@ -1,208 +1,208 @@
-"""
-Test script for new Scopus-competitive features:
-1. Citation tracking
-2. H-index calculation
-3. Advanced search with Boolean operators
-"""
-
-import sys
-import time
-
-from uraas.database import Author, Base, Item, SessionLocal, engine
-from uraas.services.advanced_search import SearchQuery
-from uraas.services.citation_tracker import (
-    AuthorMetrics,
-    Citation,
-    CitationMetrics,
-    CitationTracker,
-    get_author_bibliometrics,
-    get_paper_citations,
-)
-
-if __name__ == "__main__":
-    # Create new tables
-    print("=" * 70)
-    print("Creating citation tracking tables...")
-    print("=" * 70)
-    Base.metadata.create_all(bind=engine)
-    print("✓ Tables created\n")
-
-    # Test 1: Citation Tracking
-    print("=" * 70)
-    print("TEST 1: Citation Tracking")
-    print("=" * 70)
-
-    session = SessionLocal()
-
-    # Find a paper with DOI
-    paper = session.query(Item).filter(Item.doi.isnot(None)).first()
-
-    if paper:
-        print(f"\nTesting with paper: {paper.title[:60]}...")
-        print(f"DOI: {paper.doi}")
-
-        print("\nFetching citations from OpenAlex...")
-        success = CitationTracker.update_paper_citations(paper.id)
-
-        if success:
-            print("✓ Citations fetched successfully")
-
-            # Get citation data
-            cite_data = get_paper_citations(paper.id)
-            print(f"\nCitation count: {cite_data['citation_count']}")
-            print(f"Citing papers in our DB: {len(cite_data['citing_papers'])}")
-
-            if cite_data["citing_papers"]:
-                print("\nSample citing papers:")
-                for cite in cite_data["citing_papers"][:3]:
-                    print(f"  - {cite['title'][:60]}... ({cite['year']})")
-        else:
-            print("⚠ Citation fetch failed (paper may not be in OpenAlex)")
-    else:
-        print("⚠ No papers with DOI found in database")
-
-    session.close()
-
-    # Test 2: H-index Calculation
-    print("\n" + "=" * 70)
-    print("TEST 2: H-index Calculation")
-    print("=" * 70)
-
-    session = SessionLocal()
-
-    # Test h-index calculation
-    test_citations = [100, 50, 30, 20, 15, 10, 8, 5, 3, 2, 1, 1, 0, 0]
-    h_index = CitationTracker.calculate_h_index(test_citations)
-    print(f"\nTest citation counts: {test_citations}")
-    print(f"Calculated h-index: {h_index}")
-    print(f"Expected: 10 (10 papers with ≥10 citations)")
-
-    # Find an author and calculate their metrics
-    author = session.query(Author).join(Author.items).first()
-
-    if author:
-        print(f"\nTesting with author: {author.name}")
-
-        # Update author metrics
-        success = CitationTracker.update_author_metrics(author.id)
-
-        if success:
-            metrics = get_author_bibliometrics(author.id)
-            print(f"\nAuthor Bibliometrics:")
-            print(f"  Total papers: {metrics.get('total_papers', 0)}")
-            print(f"  Total citations: {metrics.get('total_citations', 0)}")
-            print(f"  H-index: {metrics.get('h_index', 0)}")
-            print(f"  i10-index: {metrics.get('i10_index', 0)}")
-            print(f"  Citations per paper: {metrics.get('citations_per_paper', 0)}")
-        else:
-            print("⚠ Author metrics calculation failed (papers may lack citation data)")
-
-    session.close()
-
-    # Test 3: Advanced Search
-    print("\n" + "=" * 70)
-    print("TEST 3: Advanced Search with Boolean Operators")
-    print("=" * 70)
-
-    # Test query parsing
-    test_queries = [
-        "machine learning",
-        '"machine learning" AND author:smith',
-        "title:cancer NOT lung",
-        "author:okonkwo AND year:2020",
-        "(covid OR pandemic) AND faculty:medicine",
-    ]
-
-    print("\nQuery Parsing Tests:")
-    for query in test_queries:
-        parsed = SearchQuery.parse_boolean_query(query)
-        print(f"\nQuery: {query}")
-        print(f"Parsed: {parsed}")
-
-    # Test actual search execution
-    print("\n" + "=" * 70)
-    print("Search Execution Tests:")
-    print("=" * 70)
-
-    # Test 1: Simple keyword search
-    print("\n1. Simple keyword search: 'health'")
-    results = SearchQuery.execute_search("health", limit=5)
-    print(f"   Found {results['total']} papers in {results['took_ms']}ms")
-    if results["results"]:
-        print(f"   Top result: {results['results'][0]['title'][:60]}...")
-
-    # Test 2: Field-specific search
-    print("\n2. Field-specific search: 'year:2020'")
-    results = SearchQuery.execute_search("year:2020", limit=5)
-    print(f"   Found {results['total']} papers from 2020")
-
-    # Test 3: Boolean AND
-    print("\n3. Boolean AND: 'health AND education'")
-    results = SearchQuery.execute_search("health AND education", limit=5)
-    print(f"   Found {results['total']} papers")
-
-    # Test 4: Phrase search
-    print("\n4. Phrase search: '\"machine learning\"'")
-    results = SearchQuery.execute_search('"machine learning"', limit=5)
-    print(f"   Found {results['total']} papers")
-
-    # Test 5: Complex query
-    print("\n5. Complex query: 'author:okonkwo AND faculty:science'")
-    results = SearchQuery.execute_search("author:okonkwo AND faculty:science", limit=5)
-    print(f"   Found {results['total']} papers")
-
-    # Test 6: Sort by date
-    print("\n6. Sort by date: 'health' sorted by publication date")
-    results = SearchQuery.execute_search("health", limit=5, sort_by="date")
-    print(f"   Found {results['total']} papers")
-    if results["results"]:
-        print(
-            f"   Most recent: {results['results'][0]['title'][:60]}... ({results['results'][0]['year']})"
-        )
-
-    # Test autocomplete
-    print("\n" + "=" * 70)
-    print("Autocomplete Suggestions:")
-    print("=" * 70)
-
-    test_partials = ["health", "machine", "science"]
-    for partial in test_partials:
-        suggestions = SearchQuery.get_search_suggestions(partial)
-        print(f"\n'{partial}' → {len(suggestions)} suggestions")
-        for sug in suggestions[:5]:
-            print(f"  - {sug}")
-
-    # Summary
-    print("\n" + "=" * 70)
-    print("FEATURE COMPARISON SUMMARY")
-    print("=" * 70)
-
-    print("\n✓ IMPLEMENTED:")
-    print("  1. Citation tracking (OpenAlex + Crossref APIs)")
-    print("  2. H-index calculation (standard algorithm)")
-    print("  3. i10-index (papers with 10+ citations)")
-    print("  4. Author bibliometrics (total citations, papers, indices)")
-    print("  5. Advanced search with Boolean operators (AND, OR, NOT)")
-    print("  6. Field-specific queries (title:, author:, year:, faculty:, etc.)")
-    print("  7. Phrase searches with quotes")
-    print("  8. Multiple sort options (relevance, date, citations, title)")
-    print("  9. Autocomplete suggestions")
-    print(" 10. Pagination support")
-
-    print("\n⚠ LIMITATIONS vs Scopus:")
-    print("  - Scale: ~1K papers vs 80M+ (institutional focus)")
-    print("  - Citation data: Depends on OpenAlex coverage")
-    print("  - Update frequency: Weekly vs daily (configurable)")
-    print("  - Journal metrics: Not included (focus on institutional output)")
-
-    print("\n✓ ADVANTAGES over Scopus:")
-    print("  - Zero false positives (staff validation)")
-    print("  - Free (no $40K/year subscription)")
-    print("  - Customizable (open source)")
-    print("  - African focus (indigenous knowledge metrics)")
-    print("  - Local PDF storage")
-    print("  - DocID™ persistent identifiers")
-
-    print("\n" + "=" * 70)
-    print("Tests complete!")
-    print("=" * 70)
+"""
+Test script for new Scopus-competitive features:
+1. Citation tracking
+2. H-index calculation
+3. Advanced search with Boolean operators
+"""
+
+import sys
+import time
+
+from uraas.database import Author, Base, Item, SessionLocal, engine
+from uraas.services.advanced_search import SearchQuery
+from uraas.services.citation_tracker import (
+    AuthorMetrics,
+    Citation,
+    CitationMetrics,
+    CitationTracker,
+    get_author_bibliometrics,
+    get_paper_citations,
+)
+
+if __name__ == "__main__":
+    # Create new tables
+    print("=" * 70)
+    print("Creating citation tracking tables...")
+    print("=" * 70)
+    Base.metadata.create_all(bind=engine)
+    print("✓ Tables created\n")
+
+    # Test 1: Citation Tracking
+    print("=" * 70)
+    print("TEST 1: Citation Tracking")
+    print("=" * 70)
+
+    session = SessionLocal()
+
+    # Find a paper with DOI
+    paper = session.query(Item).filter(Item.doi.isnot(None)).first()
+
+    if paper:
+        print(f"\nTesting with paper: {paper.title[:60]}...")
+        print(f"DOI: {paper.doi}")
+
+        print("\nFetching citations from OpenAlex...")
+        success = CitationTracker.update_paper_citations(paper.id)
+
+        if success:
+            print("✓ Citations fetched successfully")
+
+            # Get citation data
+            cite_data = get_paper_citations(paper.id)
+            print(f"\nCitation count: {cite_data['citation_count']}")
+            print(f"Citing papers in our DB: {len(cite_data['citing_papers'])}")
+
+            if cite_data["citing_papers"]:
+                print("\nSample citing papers:")
+                for cite in cite_data["citing_papers"][:3]:
+                    print(f"  - {cite['title'][:60]}... ({cite['year']})")
+        else:
+            print("⚠ Citation fetch failed (paper may not be in OpenAlex)")
+    else:
+        print("⚠ No papers with DOI found in database")
+
+    session.close()
+
+    # Test 2: H-index Calculation
+    print("\n" + "=" * 70)
+    print("TEST 2: H-index Calculation")
+    print("=" * 70)
+
+    session = SessionLocal()
+
+    # Test h-index calculation
+    test_citations = [100, 50, 30, 20, 15, 10, 8, 5, 3, 2, 1, 1, 0, 0]
+    h_index = CitationTracker.calculate_h_index(test_citations)
+    print(f"\nTest citation counts: {test_citations}")
+    print(f"Calculated h-index: {h_index}")
+    print(f"Expected: 10 (10 papers with ≥10 citations)")
+
+    # Find an author and calculate their metrics
+    author = session.query(Author).join(Author.items).first()
+
+    if author:
+        print(f"\nTesting with author: {author.name}")
+
+        # Update author metrics
+        success = CitationTracker.update_author_metrics(author.id)
+
+        if success:
+            metrics = get_author_bibliometrics(author.id)
+            print(f"\nAuthor Bibliometrics:")
+            print(f"  Total papers: {metrics.get('total_papers', 0)}")
+            print(f"  Total citations: {metrics.get('total_citations', 0)}")
+            print(f"  H-index: {metrics.get('h_index', 0)}")
+            print(f"  i10-index: {metrics.get('i10_index', 0)}")
+            print(f"  Citations per paper: {metrics.get('citations_per_paper', 0)}")
+        else:
+            print("⚠ Author metrics calculation failed (papers may lack citation data)")
+
+    session.close()
+
+    # Test 3: Advanced Search
+    print("\n" + "=" * 70)
+    print("TEST 3: Advanced Search with Boolean Operators")
+    print("=" * 70)
+
+    # Test query parsing
+    test_queries = [
+        "machine learning",
+        '"machine learning" AND author:smith',
+        "title:cancer NOT lung",
+        "author:okonkwo AND year:2020",
+        "(covid OR pandemic) AND faculty:medicine",
+    ]
+
+    print("\nQuery Parsing Tests:")
+    for query in test_queries:
+        parsed = SearchQuery.parse_boolean_query(query)
+        print(f"\nQuery: {query}")
+        print(f"Parsed: {parsed}")
+
+    # Test actual search execution
+    print("\n" + "=" * 70)
+    print("Search Execution Tests:")
+    print("=" * 70)
+
+    # Test 1: Simple keyword search
+    print("\n1. Simple keyword search: 'health'")
+    results = SearchQuery.execute_search("health", limit=5)
+    print(f"   Found {results['total']} papers in {results['took_ms']}ms")
+    if results["results"]:
+        print(f"   Top result: {results['results'][0]['title'][:60]}...")
+
+    # Test 2: Field-specific search
+    print("\n2. Field-specific search: 'year:2020'")
+    results = SearchQuery.execute_search("year:2020", limit=5)
+    print(f"   Found {results['total']} papers from 2020")
+
+    # Test 3: Boolean AND
+    print("\n3. Boolean AND: 'health AND education'")
+    results = SearchQuery.execute_search("health AND education", limit=5)
+    print(f"   Found {results['total']} papers")
+
+    # Test 4: Phrase search
+    print("\n4. Phrase search: '\"machine learning\"'")
+    results = SearchQuery.execute_search('"machine learning"', limit=5)
+    print(f"   Found {results['total']} papers")
+
+    # Test 5: Complex query
+    print("\n5. Complex query: 'author:okonkwo AND faculty:science'")
+    results = SearchQuery.execute_search("author:okonkwo AND faculty:science", limit=5)
+    print(f"   Found {results['total']} papers")
+
+    # Test 6: Sort by date
+    print("\n6. Sort by date: 'health' sorted by publication date")
+    results = SearchQuery.execute_search("health", limit=5, sort_by="date")
+    print(f"   Found {results['total']} papers")
+    if results["results"]:
+        print(
+            f"   Most recent: {results['results'][0]['title'][:60]}... ({results['results'][0]['year']})"
+        )
+
+    # Test autocomplete
+    print("\n" + "=" * 70)
+    print("Autocomplete Suggestions:")
+    print("=" * 70)
+
+    test_partials = ["health", "machine", "science"]
+    for partial in test_partials:
+        suggestions = SearchQuery.get_search_suggestions(partial)
+        print(f"\n'{partial}' → {len(suggestions)} suggestions")
+        for sug in suggestions[:5]:
+            print(f"  - {sug}")
+
+    # Summary
+    print("\n" + "=" * 70)
+    print("FEATURE COMPARISON SUMMARY")
+    print("=" * 70)
+
+    print("\n✓ IMPLEMENTED:")
+    print("  1. Citation tracking (OpenAlex + Crossref APIs)")
+    print("  2. H-index calculation (standard algorithm)")
+    print("  3. i10-index (papers with 10+ citations)")
+    print("  4. Author bibliometrics (total citations, papers, indices)")
+    print("  5. Advanced search with Boolean operators (AND, OR, NOT)")
+    print("  6. Field-specific queries (title:, author:, year:, faculty:, etc.)")
+    print("  7. Phrase searches with quotes")
+    print("  8. Multiple sort options (relevance, date, citations, title)")
+    print("  9. Autocomplete suggestions")
+    print(" 10. Pagination support")
+
+    print("\n⚠ LIMITATIONS vs Scopus:")
+    print("  - Scale: ~1K papers vs 80M+ (institutional focus)")
+    print("  - Citation data: Depends on OpenAlex coverage")
+    print("  - Update frequency: Weekly vs daily (configurable)")
+    print("  - Journal metrics: Not included (focus on institutional output)")
+
+    print("\n✓ ADVANTAGES over Scopus:")
+    print("  - Zero false positives (staff validation)")
+    print("  - Free (no $40K/year subscription)")
+    print("  - Customizable (open source)")
+    print("  - African focus (indigenous knowledge metrics)")
+    print("  - Local PDF storage")
+    print("  - DocID™ persistent identifiers")
+
+    print("\n" + "=" * 70)
+    print("Tests complete!")
+    print("=" * 70)

tests/test_production_ready.py

@@ -1,375 +1,375 @@
-"""
-URAAS Production Readiness Test Suite
-Comprehensive tests ensuring zero defects before deployment
-"""
-
-import sys
-from pathlib import Path
-
-import pytest
-
-sys.path.insert(0, str(Path(__file__).parent.parent))
-
-from uraas.database import Author, Item, SessionLocal
-from uraas.utils.ai_keyword_extractor import ai_extractor
-from uraas.utils.staff_validator import staff_validator
-
-
-class TestAIKeywordExtractor:
-    """Test AI keyword extraction system"""
-
-    def test_keyword_extraction_basic(self):
-        """Test basic keyword extraction"""
-        text = "machine learning deep neural networks artificial intelligence"
-        keywords = ai_extractor.extract_keywords(text, top_n=5)
-
-        assert len(keywords) > 0, "Should extract keywords"
-        assert all(
-            isinstance(k, tuple) and len(k) == 2 for k in keywords
-        ), "Keywords should be (word, score) tuples"
-        assert all(
-            0 <= score <= 1 for _, score in keywords
-        ), "Scores should be between 0 and 1"
-
-    def test_keyword_extraction_empty(self):
-        """Test with empty text"""
-        keywords = ai_extractor.extract_keywords("", top_n=5)
-        assert keywords == [], "Empty text should return no keywords"
-
-    def test_domain_classification(self):
-        """Test domain classification"""
-        text = "algorithm data structure programming software engineering"
-        domains = ai_extractor.classify_domain(text)
-
-        assert len(domains) > 0, "Should classify domains"
-        assert (
-            domains[0][0] == "computer_science"
-        ), "Should identify computer science domain"
-
-    def test_entity_extraction(self):
-        """Test entity extraction"""
-        text = "The study was conducted at University of Lagos using 50 mg of compound"
-        entities = ai_extractor.extract_entities(text)
-
-        assert "organizations" in entities, "Should extract organizations"
-        assert "measurements" in entities, "Should extract measurements"
-
-    def test_paper_scoring(self):
-        """Test paper quality scoring"""
-        title = "Machine Learning Applications in Medical Diagnosis"
-        abstract = (
-            "This study investigates the application of machine learning "
-            "algorithms for medical diagnosis. We developed a novel approach "
-            "using deep neural networks to classify medical images. Our results "
-            "show significant improvement over existing methods."
-        )
-
-        score = ai_extractor.score_paper(title, abstract)
-
-        assert "quality_score" in score, "Should have quality score"
-        assert 0 <= score["quality_score"] <= 1, "Quality score should be 0-1"
-        assert "keywords" in score, "Should have keywords"
-        assert "domains" in score, "Should have domains"
-
-
-class TestDashboardUI:
-    """Test dashboard UI for emoji removal"""
-
-    def test_dashboard_no_emojis(self):
-        """Verify dashboard HTML has no emojis in critical UI text"""
-        dashboard_path = Path("uraas/dashboard/templates/index.html")
-        assert dashboard_path.exists(), "Dashboard file should exist"
-
-        content = dashboard_path.read_text(encoding="utf-8")
-
-        # Check for common emoji patterns in non-optgroup areas (optgroups intentionally use flag emojis)
-        import re
-
-        # Strip out optgroup labels before checking
-        stripped = re.sub(r'<optgroup[^>]*label="[^"]*"[^>]*>', "", content)
-        emoji_pattern = r"[\U0001F680-\U0001F9FF]"  # Rockets, charts, etc.
-        matches = re.findall(emoji_pattern, stripped)
-        assert len(matches) == 0, f"Found unexpected emojis: {matches[:5]}"
-
-    def test_dashboard_minimalist_design(self):
-        """Verify dashboard uses consistent design tokens"""
-        dashboard_path = Path("uraas/dashboard/templates/index.html")
-        content = dashboard_path.read_text(encoding="utf-8")
-
-        # Check for design system elements present in index.html
-        assert "Inter" in content, "Should use Inter font"
-        assert (
-            "gradient-text" in content or "var(--accent)" in content
-        ), "Should use CSS design tokens"
-
-    def test_dashboard_accessibility(self):
-        """Verify dashboard has accessibility features"""
-        dashboard_path = Path("uraas/dashboard/templates/index.html")
-        content = dashboard_path.read_text(encoding="utf-8")
-
-        # Check for accessibility features
-        assert (
-            "aria-" in content or "role=" in content or "title=" in content
-        ), "Should have ARIA attributes or title attributes"
-        assert "lang=" in content, "HTML element should have lang attribute"
-
-
-class TestMetadataExtraction:
-    """Test metadata extraction accuracy"""
-
-    def test_paper_metadata_completeness(self):
-        """Test that papers have complete metadata"""
-        session = SessionLocal()
-        try:
-            papers = session.query(Item).limit(10).all()
-
-            for paper in papers:
-                assert paper.title, "Paper should have title"
-                assert paper.dc_title, "Paper should have Dublin Core title"
-
-                # dc_identifier_doi stores the repository handle (OAI/DocID),
-                # while paper.doi stores the scholarly DOI — these are distinct fields.
-                # Both can coexist; just verify dc_identifier_doi is non-empty when doi is set.
-                if paper.doi and paper.dc_identifier_doi:
-                    assert (
-                        len(paper.dc_identifier_doi) > 0
-                    ), "dc_identifier_doi should be non-empty when present"
-        finally:
-            session.close()
-
-    def test_author_extraction_accuracy(self):
-        """Test author extraction"""
-        session = SessionLocal()
-        try:
-            papers = session.query(Item).filter(Item.authors.any()).limit(5).all()
-
-            for paper in papers:
-                assert len(paper.authors) > 0, "Paper should have authors"
-
-                for author in paper.authors:
-                    assert author.name, "Author should have name"
-                    assert author.normalized_name, "Author should have normalized name"
-                    assert (
-                        author.normalized_name == author.name.lower().strip()
-                    ), "Normalized name should be lowercase"
-        finally:
-            session.close()
-
-
-class TestStaffValidation:
-    """Test staff validation accuracy"""
-
-    def test_staff_cache_loaded(self):
-        """Verify staff cache can be loaded from the data directory"""
-        import os
-
-        # The data file is always relative to the project root
-        staff_path = os.path.join(
-            os.path.dirname(__file__), "..", "data", "unilag_staff.json"
-        )
-        assert os.path.exists(
-            staff_path
-        ), f"Staff data file should exist at {staff_path}"
-        # If staff_validator already loaded correctly, that is the best evidence
-        if len(staff_validator.staff_names) == 0:
-            # Reload using absolute path so tests pass regardless of working dir
-            from uraas.utils.staff_validator import StaffValidator
-
-            sv = StaffValidator(staff_cache_path=os.path.abspath(staff_path))
-            assert (
-                len(sv.staff_names) > 0
-            ), "Staff cache should load from data/unilag_staff.json"
-
-    def test_exact_staff_match(self):
-        """Test exact staff matching"""
-        if staff_validator.staff_names:
-            test_name = list(staff_validator.staff_names)[0]
-            assert staff_validator.is_staff_member(
-                test_name, fuzzy_threshold=100
-            ), "Exact match should work"
-
-    def test_fuzzy_staff_match(self):
-        """Test fuzzy staff matching"""
-        if staff_validator.staff_names:
-            test_name = list(staff_validator.staff_names)[0]
-            # Slightly modify the name
-            modified = test_name.replace("a", "e", 1) if "a" in test_name else test_name
-            result = staff_validator.is_staff_member(modified, fuzzy_threshold=75)
-            assert isinstance(result, bool), "Should return boolean"
-
-
-class TestAPIEndpoints:
-    """Test API endpoints"""
-
-    @pytest.fixture
-    def client(self):
-        """Create test client"""
-        from uraas.dashboard.app import app
-
-        app.config["TESTING"] = True
-        with app.test_client() as client:
-            yield client
-
-    def test_dashboard_loads(self, client):
-        """Test dashboard loads"""
-        response = client.get("/")
-        assert response.status_code == 200, "Dashboard should load"
-
-    def test_analytics_overview(self, client):
-        """Test analytics overview endpoint"""
-        response = client.get("/api/analytics/overview")
-        assert response.status_code == 200, "Analytics overview should work"
-
-        data = response.get_json()
-        assert "total_papers" in data, "Should have total papers"
-        assert "total_authors" in data, "Should have total authors"
-
-    def test_search_endpoint(self, client):
-        """Test search endpoint"""
-        response = client.get("/api/analytics/search?q=test&limit=10")
-        assert response.status_code == 200, "Search should work"
-
-        data = response.get_json()
-        assert isinstance(data, list), "Search should return list"
-
-
-class TestPerformance:
-    """Test performance requirements"""
-
-    def test_keyword_extraction_speed(self):
-        """Test keyword extraction is fast"""
-        import time
-
-        text = "machine learning deep neural networks artificial intelligence " * 10
-
-        start = time.time()
-        for _ in range(100):
-            ai_extractor.extract_keywords(text, top_n=10)
-        duration = time.time() - start
-
-        avg_time = duration / 100
-        assert (
-            avg_time < 0.05
-        ), f"Keyword extraction took {avg_time}s, should be < 0.05s"
-
-    def test_domain_classification_speed(self):
-        """Test domain classification is fast"""
-        import time
-
-        text = "algorithm data structure programming software engineering" * 5
-
-        start = time.time()
-        for _ in range(100):
-            ai_extractor.classify_domain(text)
-        duration = time.time() - start
-
-        avg_time = duration / 100
-        assert avg_time < 0.01, f"Classification took {avg_time}s, should be < 0.01s"
-
-
-class TestSecurity:
-    """Test security measures"""
-
-    def test_no_sql_injection_in_search(self):
-        """Test SQL injection protection"""
-        from uraas.dashboard.app import app
-
-        app.config["TESTING"] = True
-
-        with app.test_client() as client:
-            malicious_query = "'; DROP TABLE items; --"
-            response = client.get(f"/api/analytics/search?q={malicious_query}")
-
-            # Should not crash
-            assert response.status_code in [200, 400], "Should handle malicious input"
-
-    def test_xss_protection(self):
-        """Test XSS protection"""
-        session = SessionLocal()
-        try:
-            malicious_title = "<script>alert('XSS')</script>"
-            item = Item(
-                title=malicious_title, dc_title=malicious_title, doi="10.test/xss.001"
-            )
-            session.add(item)
-            session.commit()
-
-            # Retrieve and verify
-            retrieved = session.query(Item).filter_by(doi="10.test/xss.001").first()
-            assert retrieved.title == malicious_title, "Should store as-is"
-
-            # Cleanup
-            session.delete(retrieved)
-            session.commit()
-        finally:
-            session.close()
-
-
-class TestCodeQuality:
-    """Test code quality"""
-
-    def test_no_print_statements(self):
-        """Verify no debug print statements in production code"""
-        import os
-        import re
-
-        production_dirs = ["uraas/dashboard", "uraas/utils", "uraas/pipelines"]
-
-        # Files that legitimately use print() for IPC/stdout protocols
-        ALLOWED_FILES = {
-            os.path.normpath(
-                "uraas/pipelines/database.py"
-            ),  # URAAS_DOWNLOAD: stdout IPC
-        }
-
-        for dir_path in production_dirs:
-            for root, dirs, files in os.walk(dir_path):
-                for file in files:
-                    if file.endswith(".py"):
-                        filepath = os.path.join(root, file)
-                        if os.path.normpath(filepath) in ALLOWED_FILES:
-                            continue  # Skip intentional stdout IPC files
-                        with open(
-                            filepath, "r", encoding="utf-8", errors="replace"
-                        ) as f:
-                            content = f.read()
-                            # Check for debug print statements (not logging)
-                            debug_prints = re.findall(
-                                r"^\s*print\(", content, re.MULTILINE
-                            )
-                            assert (
-                                len(debug_prints) == 0
-                            ), f"Found debug print statements in {filepath}"
-
-    def test_imports_organized(self):
-        """Verify imports are organized"""
-        import os
-
-        for root, dirs, files in os.walk("uraas"):
-            for file in files:
-                if file.endswith(".py"):
-                    filepath = os.path.join(root, file)
-                    with open(filepath, "r") as f:
-                        lines = f.readlines()
-
-                        # Check that imports are at the top
-                        import_section_ended = False
-                        for i, line in enumerate(lines[:50]):
-                            if line.strip() and not line.startswith(
-                                ("import", "from", "#", '"""', "'''")
-                            ):
-                                import_section_ended = True
-                            elif import_section_ended and line.startswith(
-                                ("import", "from")
-                            ):
-                                # Imports after code is bad
-                                pass  # Allow for now
-
-
-def run_all_tests():
-    """Run all tests"""
-    pytest.main([__file__, "-v", "--tb=short", "--color=yes"])
-
-
-if __name__ == "__main__":
-    run_all_tests()
+"""
+URAAS Production Readiness Test Suite
+Comprehensive tests ensuring zero defects before deployment
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from uraas.database import Author, Item, SessionLocal
+from uraas.utils.ai_keyword_extractor import ai_extractor
+from uraas.utils.staff_validator import staff_validator
+
+
+class TestAIKeywordExtractor:
+    """Test AI keyword extraction system"""
+
+    def test_keyword_extraction_basic(self):
+        """Test basic keyword extraction"""
+        text = "machine learning deep neural networks artificial intelligence"
+        keywords = ai_extractor.extract_keywords(text, top_n=5)
+
+        assert len(keywords) > 0, "Should extract keywords"
+        assert all(
+            isinstance(k, tuple) and len(k) == 2 for k in keywords
+        ), "Keywords should be (word, score) tuples"
+        assert all(
+            0 <= score <= 1 for _, score in keywords
+        ), "Scores should be between 0 and 1"
+
+    def test_keyword_extraction_empty(self):
+        """Test with empty text"""
+        keywords = ai_extractor.extract_keywords("", top_n=5)
+        assert keywords == [], "Empty text should return no keywords"
+
+    def test_domain_classification(self):
+        """Test domain classification"""
+        text = "algorithm data structure programming software engineering"
+        domains = ai_extractor.classify_domain(text)
+
+        assert len(domains) > 0, "Should classify domains"
+        assert (
+            domains[0][0] == "computer_science"
+        ), "Should identify computer science domain"
+
+    def test_entity_extraction(self):
+        """Test entity extraction"""
+        text = "The study was conducted at University of Lagos using 50 mg of compound"
+        entities = ai_extractor.extract_entities(text)
+
+        assert "organizations" in entities, "Should extract organizations"
+        assert "measurements" in entities, "Should extract measurements"
+
+    def test_paper_scoring(self):
+        """Test paper quality scoring"""
+        title = "Machine Learning Applications in Medical Diagnosis"
+        abstract = (
+            "This study investigates the application of machine learning "
+            "algorithms for medical diagnosis. We developed a novel approach "
+            "using deep neural networks to classify medical images. Our results "
+            "show significant improvement over existing methods."
+        )
+
+        score = ai_extractor.score_paper(title, abstract)
+
+        assert "quality_score" in score, "Should have quality score"
+        assert 0 <= score["quality_score"] <= 1, "Quality score should be 0-1"
+        assert "keywords" in score, "Should have keywords"
+        assert "domains" in score, "Should have domains"
+
+
+class TestDashboardUI:
+    """Test dashboard UI for emoji removal"""
+
+    def test_dashboard_no_emojis(self):
+        """Verify dashboard HTML has no emojis in critical UI text"""
+        dashboard_path = Path("uraas/dashboard/templates/index.html")
+        assert dashboard_path.exists(), "Dashboard file should exist"
+
+        content = dashboard_path.read_text(encoding="utf-8")
+
+        # Check for common emoji patterns in non-optgroup areas (optgroups intentionally use flag emojis)
+        import re
+
+        # Strip out optgroup labels before checking
+        stripped = re.sub(r'<optgroup[^>]*label="[^"]*"[^>]*>', "", content)
+        emoji_pattern = r"[\U0001F680-\U0001F9FF]"  # Rockets, charts, etc.
+        matches = re.findall(emoji_pattern, stripped)
+        assert len(matches) == 0, f"Found unexpected emojis: {matches[:5]}"
+
+    def test_dashboard_minimalist_design(self):
+        """Verify dashboard uses consistent design tokens"""
+        dashboard_path = Path("uraas/dashboard/templates/index.html")
+        content = dashboard_path.read_text(encoding="utf-8")
+
+        # Check for design system elements present in index.html
+        assert "Inter" in content, "Should use Inter font"
+        assert (
+            "gradient-text" in content or "var(--accent)" in content
+        ), "Should use CSS design tokens"
+
+    def test_dashboard_accessibility(self):
+        """Verify dashboard has accessibility features"""
+        dashboard_path = Path("uraas/dashboard/templates/index.html")
+        content = dashboard_path.read_text(encoding="utf-8")
+
+        # Check for accessibility features
+        assert (
+            "aria-" in content or "role=" in content or "title=" in content
+        ), "Should have ARIA attributes or title attributes"
+        assert "lang=" in content, "HTML element should have lang attribute"
+
+
+class TestMetadataExtraction:
+    """Test metadata extraction accuracy"""
+
+    def test_paper_metadata_completeness(self):
+        """Test that papers have complete metadata"""
+        session = SessionLocal()
+        try:
+            papers = session.query(Item).limit(10).all()
+
+            for paper in papers:
+                assert paper.title, "Paper should have title"
+                assert paper.dc_title, "Paper should have Dublin Core title"
+
+                # dc_identifier_doi stores the repository handle (OAI/DocID),
+                # while paper.doi stores the scholarly DOI — these are distinct fields.
+                # Both can coexist; just verify dc_identifier_doi is non-empty when doi is set.
+                if paper.doi and paper.dc_identifier_doi:
+                    assert (
+                        len(paper.dc_identifier_doi) > 0
+                    ), "dc_identifier_doi should be non-empty when present"
+        finally:
+            session.close()
+
+    def test_author_extraction_accuracy(self):
+        """Test author extraction"""
+        session = SessionLocal()
+        try:
+            papers = session.query(Item).filter(Item.authors.any()).limit(5).all()
+
+            for paper in papers:
+                assert len(paper.authors) > 0, "Paper should have authors"
+
+                for author in paper.authors:
+                    assert author.name, "Author should have name"
+                    assert author.normalized_name, "Author should have normalized name"
+                    assert (
+                        author.normalized_name == author.name.lower().strip()
+                    ), "Normalized name should be lowercase"
+        finally:
+            session.close()
+
+
+class TestStaffValidation:
+    """Test staff validation accuracy"""
+
+    def test_staff_cache_loaded(self):
+        """Verify staff cache can be loaded from the data directory"""
+        import os
+
+        # The data file is always relative to the project root
+        staff_path = os.path.join(
+            os.path.dirname(__file__), "..", "data", "unilag_staff.json"
+        )
+        assert os.path.exists(
+            staff_path
+        ), f"Staff data file should exist at {staff_path}"
+        # If staff_validator already loaded correctly, that is the best evidence
+        if len(staff_validator.staff_names) == 0:
+            # Reload using absolute path so tests pass regardless of working dir
+            from uraas.utils.staff_validator import StaffValidator
+
+            sv = StaffValidator(staff_cache_path=os.path.abspath(staff_path))
+            assert (
+                len(sv.staff_names) > 0
+            ), "Staff cache should load from data/unilag_staff.json"
+
+    def test_exact_staff_match(self):
+        """Test exact staff matching"""
+        if staff_validator.staff_names:
+            test_name = list(staff_validator.staff_names)[0]
+            assert staff_validator.is_staff_member(
+                test_name, fuzzy_threshold=100
+            ), "Exact match should work"
+
+    def test_fuzzy_staff_match(self):
+        """Test fuzzy staff matching"""
+        if staff_validator.staff_names:
+            test_name = list(staff_validator.staff_names)[0]
+            # Slightly modify the name
+            modified = test_name.replace("a", "e", 1) if "a" in test_name else test_name
+            result = staff_validator.is_staff_member(modified, fuzzy_threshold=75)
+            assert isinstance(result, bool), "Should return boolean"
+
+
+class TestAPIEndpoints:
+    """Test API endpoints"""
+
+    @pytest.fixture
+    def client(self):
+        """Create test client"""
+        from uraas.dashboard.app import app
+
+        app.config["TESTING"] = True
+        with app.test_client() as client:
+            yield client
+
+    def test_dashboard_loads(self, client):
+        """Test dashboard loads"""
+        response = client.get("/")
+        assert response.status_code == 200, "Dashboard should load"
+
+    def test_analytics_overview(self, client):
+        """Test analytics overview endpoint"""
+        response = client.get("/api/analytics/overview")
+        assert response.status_code == 200, "Analytics overview should work"
+
+        data = response.get_json()
+        assert "total_papers" in data, "Should have total papers"
+        assert "total_authors" in data, "Should have total authors"
+
+    def test_search_endpoint(self, client):
+        """Test search endpoint"""
+        response = client.get("/api/analytics/search?q=test&limit=10")
+        assert response.status_code == 200, "Search should work"
+
+        data = response.get_json()
+        assert isinstance(data, list), "Search should return list"
+
+
+class TestPerformance:
+    """Test performance requirements"""
+
+    def test_keyword_extraction_speed(self):
+        """Test keyword extraction is fast"""
+        import time
+
+        text = "machine learning deep neural networks artificial intelligence " * 10
+
+        start = time.time()
+        for _ in range(100):
+            ai_extractor.extract_keywords(text, top_n=10)
+        duration = time.time() - start
+
+        avg_time = duration / 100
+        assert (
+            avg_time < 0.05
+        ), f"Keyword extraction took {avg_time}s, should be < 0.05s"
+
+    def test_domain_classification_speed(self):
+        """Test domain classification is fast"""
+        import time
+
+        text = "algorithm data structure programming software engineering" * 5
+
+        start = time.time()
+        for _ in range(100):
+            ai_extractor.classify_domain(text)
+        duration = time.time() - start
+
+        avg_time = duration / 100
+        assert avg_time < 0.01, f"Classification took {avg_time}s, should be < 0.01s"
+
+
+class TestSecurity:
+    """Test security measures"""
+
+    def test_no_sql_injection_in_search(self):
+        """Test SQL injection protection"""
+        from uraas.dashboard.app import app
+
+        app.config["TESTING"] = True
+
+        with app.test_client() as client:
+            malicious_query = "'; DROP TABLE items; --"
+            response = client.get(f"/api/analytics/search?q={malicious_query}")
+
+            # Should not crash
+            assert response.status_code in [200, 400], "Should handle malicious input"
+
+    def test_xss_protection(self):
+        """Test XSS protection"""
+        session = SessionLocal()
+        try:
+            malicious_title = "<script>alert('XSS')</script>"
+            item = Item(
+                title=malicious_title, dc_title=malicious_title, doi="10.test/xss.001"
+            )
+            session.add(item)
+            session.commit()
+
+            # Retrieve and verify
+            retrieved = session.query(Item).filter_by(doi="10.test/xss.001").first()
+            assert retrieved.title == malicious_title, "Should store as-is"
+
+            # Cleanup
+            session.delete(retrieved)
+            session.commit()
+        finally:
+            session.close()
+
+
+class TestCodeQuality:
+    """Test code quality"""
+
+    def test_no_print_statements(self):
+        """Verify no debug print statements in production code"""
+        import os
+        import re
+
+        production_dirs = ["uraas/dashboard", "uraas/utils", "uraas/pipelines"]
+
+        # Files that legitimately use print() for IPC/stdout protocols
+        ALLOWED_FILES = {
+            os.path.normpath(
+                "uraas/pipelines/database.py"
+            ),  # URAAS_DOWNLOAD: stdout IPC
+        }
+
+        for dir_path in production_dirs:
+            for root, dirs, files in os.walk(dir_path):
+                for file in files:
+                    if file.endswith(".py"):
+                        filepath = os.path.join(root, file)
+                        if os.path.normpath(filepath) in ALLOWED_FILES:
+                            continue  # Skip intentional stdout IPC files
+                        with open(
+                            filepath, "r", encoding="utf-8", errors="replace"
+                        ) as f:
+                            content = f.read()
+                            # Check for debug print statements (not logging)
+                            debug_prints = re.findall(
+                                r"^\s*print\(", content, re.MULTILINE
+                            )
+                            assert (
+                                len(debug_prints) == 0
+                            ), f"Found debug print statements in {filepath}"
+
+    def test_imports_organized(self):
+        """Verify imports are organized"""
+        import os
+
+        for root, dirs, files in os.walk("uraas"):
+            for file in files:
+                if file.endswith(".py"):
+                    filepath = os.path.join(root, file)
+                    with open(filepath, "r") as f:
+                        lines = f.readlines()
+
+                        # Check that imports are at the top
+                        import_section_ended = False
+                        for i, line in enumerate(lines[:50]):
+                            if line.strip() and not line.startswith(
+                                ("import", "from", "#", '"""', "'''")
+                            ):
+                                import_section_ended = True
+                            elif import_section_ended and line.startswith(
+                                ("import", "from")
+                            ):
+                                # Imports after code is bad
+                                pass  # Allow for now
+
+
+def run_all_tests():
+    """Run all tests"""
+    pytest.main([__file__, "-v", "--tb=short", "--color=yes"])
+
+
+if __name__ == "__main__":
+    run_all_tests()

tests/test_staff_res.py

@@ -1,12 +1,12 @@
-import os
-
-from uraas.config.institutions import get_registry
-
-if __name__ == "__main__":
-    registry = get_registry()
-    for inst in registry.list_all():
-        print(
-            f"{inst.short_name}: {len(inst.staff_names)} names, resolved path: {inst._resolve_staff_file()}"
-        )
-        if not os.path.exists(inst._resolve_staff_file()):
-            print(f"  ERROR: Path does not exist!")
+import os
+
+from uraas.config.institutions import get_registry
+
+if __name__ == "__main__":
+    registry = get_registry()
+    for inst in registry.list_all():
+        print(
+            f"{inst.short_name}: {len(inst.staff_names)} names, resolved path: {inst._resolve_staff_file()}"
+        )
+        if not os.path.exists(inst._resolve_staff_file()):
+            print(f"  ERROR: Path does not exist!")