diff --git a/.gitignore b/.gitignore index 048fea58c90acff676ee75310e3cd4e78e0d2a55..3e1f71a360f5b773cd1dc7f1b28a27a635d37262 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ env/ .env.* !.env.example +# Claude Code local settings (may contain tokens/keys from shell history) +.claude/settings.local.json + # Databases *.db *.db-journal diff --git a/Dockerfile b/Dockerfile index 4639379f869f36ef0496e13be78acb5628d57b02..fbf66296539341ea7e7a186160ffc43a867fe8fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,40 +1,40 @@ -# ── URAAS — Hugging Face Spaces Dockerfile ──────────────────────────────────── -# Single container: SQLite on /data (persistent bucket), gunicorn on port 7860. -# No PostgreSQL or Redis needed — SQLite stored in HF persistent storage. -# ────────────────────────────────────────────────────────────────────────────── -FROM python:3.11-slim - -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - URAAS_ENV=production \ - PORT=7860 - -# System deps -RUN apt-get update && apt-get install -y \ - gcc g++ curl \ - && rm -rf /var/lib/apt/lists/* - -# App user — HF Spaces runs as root but we keep the same uid as prod -RUN useradd -m -u 1000 uraas && \ - mkdir -p /app /app/storage/pdfs /app/data /app/logs && \ - chown -R uraas:uraas /app - -WORKDIR /app - -# Install Python dependencies (no libpq — SQLite only) -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt && \ - python -m spacy download en_core_web_sm - -# Copy application code -COPY --chown=uraas:uraas . . - -# Make startup script executable -RUN chmod +x scripts/start_hf.sh - -USER uraas - -EXPOSE 7860 - -# Startup: init DB in /data then start gunicorn -CMD ["bash", "scripts/start_hf.sh"] +# ── URAAS — Hugging Face Spaces Dockerfile ──────────────────────────────────── +# Single container: SQLite on /data (persistent bucket), gunicorn on port 7860. +# No PostgreSQL or Redis needed — SQLite stored in HF persistent storage. +# ────────────────────────────────────────────────────────────────────────────── +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + URAAS_ENV=production \ + PORT=7860 + +# System deps +RUN apt-get update && apt-get install -y \ + gcc g++ curl \ + && rm -rf /var/lib/apt/lists/* + +# App user — HF Spaces runs as root but we keep the same uid as prod +RUN useradd -m -u 1000 uraas && \ + mkdir -p /app /app/storage/pdfs /app/data /app/logs && \ + chown -R uraas:uraas /app + +WORKDIR /app + +# Install Python dependencies (no libpq — SQLite only) +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && \ + python -m spacy download en_core_web_sm + +# Copy application code +COPY --chown=uraas:uraas . . + +# Make startup script executable +RUN chmod +x scripts/start_hf.sh + +USER uraas + +EXPOSE 7860 + +# Startup: init DB in /data then start gunicorn +CMD ["bash", "scripts/start_hf.sh"] diff --git a/api/index.py b/api/index.py index 1866ee97ff7e7ec3b1c3779907c4a203f9e5e8cc..560328e3f84286c1059cb5b2813d6f4d225f7302 100644 --- a/api/index.py +++ b/api/index.py @@ -1,17 +1,17 @@ -import os -import sys - -# Add root directory to sys.path so 'uraas' package is findable -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -# Set environment variables for Vercel -# Vercel filesystem is read-only, so we point SQLite to a temp dir if we want to write, -# or just keep it in the project root if it's read-only. -os.environ["DATABASE_URL"] = "sqlite:///uraas.db" - -from uraas.dashboard.app import app - -# For Vercel, the variable must be named 'app' -# but since we imported 'app' from uraas.dashboard.app, it's already there. -# We just need to make sure it's exported at the module level. -handler = app +import os +import sys + +# Add root directory to sys.path so 'uraas' package is findable +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Set environment variables for Vercel +# Vercel filesystem is read-only, so we point SQLite to a temp dir if we want to write, +# or just keep it in the project root if it's read-only. +os.environ["DATABASE_URL"] = "sqlite:///uraas.db" + +from uraas.dashboard.app import app + +# For Vercel, the variable must be named 'app' +# but since we imported 'app' from uraas.dashboard.app, it's already there. +# We just need to make sure it's exported at the module level. +handler = app diff --git a/gunicorn_config.py b/gunicorn_config.py index 2129d131efb8b38e6c9401e922134c118aa255fb..d9d5bb406ccec77728bc813345c7ca9471ffafb5 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -1,64 +1,64 @@ -""" -Gunicorn configuration for production deployment on Render. -Optimized for Flask-SocketIO with WebSocket support. -""" - -import multiprocessing -import os - -# Server socket — default 8080 matches Dockerfile EXPOSE and health checks. -# HF Spaces overrides this with PORT=7860 via Space config. -port = os.getenv("PORT", "8080") -bind = f"0.0.0.0:{port}" - -# Worker processes -# Free tier: 2 workers, Starter tier: 4 workers -workers = int(os.getenv("GUNICORN_WORKERS", "2")) - -# Worker class — must match Flask-SocketIO async_mode. -# app.py uses async_mode="threading", so we use gthread (synchronous + threads). -# Do NOT use eventlet or gevent here without also changing async_mode in SocketIO. -worker_class = "gthread" - -# Threads per worker (for gthread worker_class) -threads = 4 - -# Worker connections -worker_connections = 1000 - -# Restart workers after handling this many requests (prevents memory leaks) -max_requests = 1000 -max_requests_jitter = 50 - -# Timeout for requests (120 seconds for long-running crawler operations) -timeout = 120 - -# Keep-alive connections -keepalive = 5 - -# Logging -accesslog = "-" # Log to stdout (Render captures this) -errorlog = "-" # Log to stderr (Render captures this) -loglevel = "info" -access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' - -# Process naming -proc_name = "uraas-dashboard" - -# Graceful shutdown timeout -graceful_timeout = 30 - -# Preload app for faster worker spawning -preload_app = True - -# Server mechanics -daemon = False -pidfile = None -umask = 0 -user = None -group = None -tmp_upload_dir = None - -# SSL (handled by Render's load balancer) -keyfile = None -certfile = None +""" +Gunicorn configuration for production deployment on Render. +Optimized for Flask-SocketIO with WebSocket support. +""" + +import multiprocessing +import os + +# Server socket — default 8080 matches Dockerfile EXPOSE and health checks. +# HF Spaces overrides this with PORT=7860 via Space config. +port = os.getenv("PORT", "8080") +bind = f"0.0.0.0:{port}" + +# Worker processes +# Free tier: 2 workers, Starter tier: 4 workers +workers = int(os.getenv("GUNICORN_WORKERS", "2")) + +# Worker class — must match Flask-SocketIO async_mode. +# app.py uses async_mode="threading", so we use gthread (synchronous + threads). +# Do NOT use eventlet or gevent here without also changing async_mode in SocketIO. +worker_class = "gthread" + +# Threads per worker (for gthread worker_class) +threads = 4 + +# Worker connections +worker_connections = 1000 + +# Restart workers after handling this many requests (prevents memory leaks) +max_requests = 1000 +max_requests_jitter = 50 + +# Timeout for requests (120 seconds for long-running crawler operations) +timeout = 120 + +# Keep-alive connections +keepalive = 5 + +# Logging +accesslog = "-" # Log to stdout (Render captures this) +errorlog = "-" # Log to stderr (Render captures this) +loglevel = "info" +access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' + +# Process naming +proc_name = "uraas-dashboard" + +# Graceful shutdown timeout +graceful_timeout = 30 + +# Preload app for faster worker spawning +preload_app = True + +# Server mechanics +daemon = False +pidfile = None +umask = 0 +user = None +group = None +tmp_upload_dir = None + +# SSL (handled by Render's load balancer) +keyfile = None +certfile = None diff --git a/netlify/functions/app.py b/netlify/functions/app.py index 779c897dc26c3de56c582e71fb8f1fb14f523fb3..c8bad4f35e4b172ffacb72054dafe8afcc8ab3f5 100644 --- a/netlify/functions/app.py +++ b/netlify/functions/app.py @@ -1,49 +1,49 @@ -import os -import sys - -# Add project root to path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../..")) - -from uraas.dashboard.app import app - - -def handler(event, context): - """Netlify serverless function handler""" - from io import BytesIO - - from werkzeug.wrappers import Request, Response - - # Convert Netlify event to WSGI environ - environ = { - "REQUEST_METHOD": event["httpMethod"], - "SCRIPT_NAME": "", - "PATH_INFO": event["path"], - "QUERY_STRING": event.get("rawQuery", ""), - "CONTENT_TYPE": event["headers"].get("content-type", ""), - "CONTENT_LENGTH": str(len(event.get("body", ""))), - "SERVER_NAME": event["headers"].get("host", "localhost"), - "SERVER_PORT": "443", - "SERVER_PROTOCOL": "HTTP/1.1", - "wsgi.version": (1, 0), - "wsgi.url_scheme": "https", - "wsgi.input": BytesIO(event.get("body", "").encode()), - "wsgi.errors": sys.stderr, - "wsgi.multithread": False, - "wsgi.multiprocess": True, - "wsgi.run_once": False, - } - - # Add headers - for key, value in event.get("headers", {}).items(): - key = key.upper().replace("-", "_") - if key not in ("CONTENT_TYPE", "CONTENT_LENGTH"): - environ[f"HTTP_{key}"] = value - - # Call Flask app - response = Response.from_app(app, environ) - - return { - "statusCode": response.status_code, - "headers": dict(response.headers), - "body": response.get_data(as_text=True), - } +import os +import sys + +# Add project root to path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../..")) + +from uraas.dashboard.app import app + + +def handler(event, context): + """Netlify serverless function handler""" + from io import BytesIO + + from werkzeug.wrappers import Request, Response + + # Convert Netlify event to WSGI environ + environ = { + "REQUEST_METHOD": event["httpMethod"], + "SCRIPT_NAME": "", + "PATH_INFO": event["path"], + "QUERY_STRING": event.get("rawQuery", ""), + "CONTENT_TYPE": event["headers"].get("content-type", ""), + "CONTENT_LENGTH": str(len(event.get("body", ""))), + "SERVER_NAME": event["headers"].get("host", "localhost"), + "SERVER_PORT": "443", + "SERVER_PROTOCOL": "HTTP/1.1", + "wsgi.version": (1, 0), + "wsgi.url_scheme": "https", + "wsgi.input": BytesIO(event.get("body", "").encode()), + "wsgi.errors": sys.stderr, + "wsgi.multithread": False, + "wsgi.multiprocess": True, + "wsgi.run_once": False, + } + + # Add headers + for key, value in event.get("headers", {}).items(): + key = key.upper().replace("-", "_") + if key not in ("CONTENT_TYPE", "CONTENT_LENGTH"): + environ[f"HTTP_{key}"] = value + + # Call Flask app + response = Response.from_app(app, environ) + + return { + "statusCode": response.status_code, + "headers": dict(response.headers), + "body": response.get_data(as_text=True), + } diff --git a/scratch/apply_remaining_ror_fixes.py b/scratch/apply_remaining_ror_fixes.py index 60f29c737af3e2bc9a6d65f763ed01c6fc8a5cf9..95f08ed5fafe7974e87be46d2855f3bc3bd945fb 100644 --- a/scratch/apply_remaining_ror_fixes.py +++ b/scratch/apply_remaining_ror_fixes.py @@ -1,56 +1,56 @@ -""" -Apply the manually-found ROR corrections for the 7 remaining institutions. -""" -import json -from pathlib import Path - -config_dir = Path(__file__).parent.parent / "config" / "institutions" - -CORRECTIONS = { - "agostinhoneto.json": "https://ror.org/0057ag334", # Agostinho Neto University (5726 works) - "kinshasa.json": "https://ror.org/05rrz2q74", # University of Kinshasa (10374 works) - "marienngouabi.json": "https://ror.org/00tt5kf04", # Marien Ngouabi University (4236 works) - "masuku.json": "https://ror.org/03f0njg03", # Univ. Sciences et Techniques de Masuku (1522) - "mohammedv.json": "https://ror.org/00r8w8f84", # Mohammed V University (49646 works) - "tunis.json": "https://ror.org/029cgt552", # Tunis El Manar University (37992 works) - "yaoundei.json": None, # Need to search for Université de Yaoundé I specifically -} - -# For Yaoundé I, search for the proper institution (not the hospital) -import urllib.request, urllib.parse -q = urllib.parse.quote("Universite de Yaounde") -url = f"https://api.openalex.org/institutions?search={q}&per-page=5&mailto=cokiki@unilag.edu.ng" -req = urllib.request.urlopen(url, timeout=15) -resp = json.loads(req.read()) -print("=== Yaoundé I search results ===") -for r in resp.get("results", []): - print(f" [{r['works_count']:6d}] {r['display_name']} | ROR: {r['ror']}") - -# The actual Université de Yaoundé I -# Will manually set from search result -CORRECTIONS["yaoundei.json"] = "https://ror.org/01ktt0j77" # Université de Yaoundé I (verified below) - -# Re-verify with direct lookup -import urllib.request as ur -try: - check_url = "https://api.openalex.org/works?filter=institutions.ror:01ktt0j77&select=id&per-page=1&mailto=cokiki@unilag.edu.ng" - r2 = json.loads(ur.urlopen(check_url, timeout=10).read()) - print(f"\nYaoundé I (01ktt0j77): count={r2['meta']['count']}") -except Exception as e: - print(f"Check failed: {e}") - -print("\n--- APPLYING REMAINING PATCHES ---") -for fname, new_ror in CORRECTIONS.items(): - if new_ror is None: - print(f"[SKIP] {fname}") - continue - jf = config_dir / fname - with open(jf) as f: - data = json.load(f) - old_ror = data["ror"] - data["ror"] = new_ror - with open(jf, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - print(f"[PATCHED] {fname}: {old_ror} -> {new_ror}") - -print("\nAll done!") +""" +Apply the manually-found ROR corrections for the 7 remaining institutions. +""" +import json +from pathlib import Path + +config_dir = Path(__file__).parent.parent / "config" / "institutions" + +CORRECTIONS = { + "agostinhoneto.json": "https://ror.org/0057ag334", # Agostinho Neto University (5726 works) + "kinshasa.json": "https://ror.org/05rrz2q74", # University of Kinshasa (10374 works) + "marienngouabi.json": "https://ror.org/00tt5kf04", # Marien Ngouabi University (4236 works) + "masuku.json": "https://ror.org/03f0njg03", # Univ. Sciences et Techniques de Masuku (1522) + "mohammedv.json": "https://ror.org/00r8w8f84", # Mohammed V University (49646 works) + "tunis.json": "https://ror.org/029cgt552", # Tunis El Manar University (37992 works) + "yaoundei.json": None, # Need to search for Université de Yaoundé I specifically +} + +# For Yaoundé I, search for the proper institution (not the hospital) +import urllib.request, urllib.parse +q = urllib.parse.quote("Universite de Yaounde") +url = f"https://api.openalex.org/institutions?search={q}&per-page=5&mailto=cokiki@unilag.edu.ng" +req = urllib.request.urlopen(url, timeout=15) +resp = json.loads(req.read()) +print("=== Yaoundé I search results ===") +for r in resp.get("results", []): + print(f" [{r['works_count']:6d}] {r['display_name']} | ROR: {r['ror']}") + +# The actual Université de Yaoundé I +# Will manually set from search result +CORRECTIONS["yaoundei.json"] = "https://ror.org/01ktt0j77" # Université de Yaoundé I (verified below) + +# Re-verify with direct lookup +import urllib.request as ur +try: + check_url = "https://api.openalex.org/works?filter=institutions.ror:01ktt0j77&select=id&per-page=1&mailto=cokiki@unilag.edu.ng" + r2 = json.loads(ur.urlopen(check_url, timeout=10).read()) + print(f"\nYaoundé I (01ktt0j77): count={r2['meta']['count']}") +except Exception as e: + print(f"Check failed: {e}") + +print("\n--- APPLYING REMAINING PATCHES ---") +for fname, new_ror in CORRECTIONS.items(): + if new_ror is None: + print(f"[SKIP] {fname}") + continue + jf = config_dir / fname + with open(jf) as f: + data = json.load(f) + old_ror = data["ror"] + data["ror"] = new_ror + with open(jf, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + print(f"[PATCHED] {fname}: {old_ror} -> {new_ror}") + +print("\nAll done!") diff --git a/scratch/check_db.py b/scratch/check_db.py index f5a59adbd255956918f331f2e62c40d19ab75aff..bad5cde0f8624ea2fe51b632e2e9b500c3b2fcfc 100644 --- a/scratch/check_db.py +++ b/scratch/check_db.py @@ -1,18 +1,18 @@ -import sqlite3 - -conn = sqlite3.connect("uraas.db") -cursor = conn.cursor() - -cursor.execute( - "SELECT title, institution, created_at FROM items WHERE institution = 'Addis Ababa University' ORDER BY created_at DESC;" -) -rows = cursor.fetchall() - -if not rows: - print("No papers found for Addis Ababa University.") -else: - print(f"Found {len(rows)} papers total:") - for row in rows: - print(f"- {row[0][:50]}... | {row[1]} | {row[2]}") - -conn.close() +import sqlite3 + +conn = sqlite3.connect("uraas.db") +cursor = conn.cursor() + +cursor.execute( + "SELECT title, institution, created_at FROM items WHERE institution = 'Addis Ababa University' ORDER BY created_at DESC;" +) +rows = cursor.fetchall() + +if not rows: + print("No papers found for Addis Ababa University.") +else: + print(f"Found {len(rows)} papers total:") + for row in rows: + print(f"- {row[0][:50]}... | {row[1]} | {row[2]}") + +conn.close() diff --git a/scratch/fix_rors.py b/scratch/fix_rors.py index bba86cd036eabc4f411fb788f0340145b348dc3b..d122fe4b39a0ebeb069a72bd05d4d3f96735a59a 100644 --- a/scratch/fix_rors.py +++ b/scratch/fix_rors.py @@ -1,91 +1,91 @@ -""" -Lookup correct RORs from OpenAlex for all broken institutions, -then patch the JSON files automatically. -""" -import json -import time -import urllib.request -import urllib.parse -from pathlib import Path - -BASE = "https://api.openalex.org" -MAILTO = "cokiki@unilag.edu.ng" - -# Institutions we know are broken (from verify_rors.py output) -BROKEN = [ - "agostinhoneto.json", - "ainshams.json", - "alexandria.json", - "cairo.json", - "daressalaam.json", - "kinshasa.json", - "marienngouabi.json", - "masuku.json", - "mohammedv.json", - "pretoria.json", - "rwanda.json", - "tunis.json", - "wits.json", - "yaoundei.json", - "zimbabwe.json", -] - -config_dir = Path(__file__).parent.parent / "config" / "institutions" -fixes = {} - -for fname in BROKEN: - jf = config_dir / fname - with open(jf) as f: - data = json.load(f) - - name = data["name"] - q = urllib.parse.quote(name) - url = f"{BASE}/institutions?search={q}&per-page=3&mailto={MAILTO}" - - try: - req = urllib.request.urlopen(url, timeout=15) - resp = json.loads(req.read()) - results = resp.get("results", []) - except Exception as e: - print(f"[ERROR] {name}: {e}") - fixes[fname] = None - time.sleep(1) - continue - - if not results: - print(f"[NOT FOUND] {name}") - fixes[fname] = None - else: - best = results[0] - ror = best["ror"] # e.g. "https://ror.org/00cb9w016" - ror_short = ror.split("/")[-1] - count = best.get("works_count", "?") - display = best["display_name"] - print(f"[FOUND] {name!r}") - print(f" OpenAlex: {display!r}") - print(f" ROR: {ror} (works: {count})") - if count == 0: - print(f" WARNING: works_count=0 — double check!") - fixes[fname] = ror - - time.sleep(0.4) - -# Now apply the patches -print("\n--- APPLYING PATCHES ---") -for fname, new_ror in fixes.items(): - if new_ror is None: - print(f"[SKIP] {fname} — no ROR found") - continue - jf = config_dir / fname - with open(jf) as f: - data = json.load(f) - old_ror = data["ror"] - if old_ror == new_ror: - print(f"[SAME] {fname} — already correct") - continue - data["ror"] = new_ror - with open(jf, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - print(f"[PATCHED] {fname}: {old_ror} -> {new_ror}") - -print("\nDone. Run verify_rors.py again to confirm all are fixed.") +""" +Lookup correct RORs from OpenAlex for all broken institutions, +then patch the JSON files automatically. +""" +import json +import time +import urllib.request +import urllib.parse +from pathlib import Path + +BASE = "https://api.openalex.org" +MAILTO = "cokiki@unilag.edu.ng" + +# Institutions we know are broken (from verify_rors.py output) +BROKEN = [ + "agostinhoneto.json", + "ainshams.json", + "alexandria.json", + "cairo.json", + "daressalaam.json", + "kinshasa.json", + "marienngouabi.json", + "masuku.json", + "mohammedv.json", + "pretoria.json", + "rwanda.json", + "tunis.json", + "wits.json", + "yaoundei.json", + "zimbabwe.json", +] + +config_dir = Path(__file__).parent.parent / "config" / "institutions" +fixes = {} + +for fname in BROKEN: + jf = config_dir / fname + with open(jf) as f: + data = json.load(f) + + name = data["name"] + q = urllib.parse.quote(name) + url = f"{BASE}/institutions?search={q}&per-page=3&mailto={MAILTO}" + + try: + req = urllib.request.urlopen(url, timeout=15) + resp = json.loads(req.read()) + results = resp.get("results", []) + except Exception as e: + print(f"[ERROR] {name}: {e}") + fixes[fname] = None + time.sleep(1) + continue + + if not results: + print(f"[NOT FOUND] {name}") + fixes[fname] = None + else: + best = results[0] + ror = best["ror"] # e.g. "https://ror.org/00cb9w016" + ror_short = ror.split("/")[-1] + count = best.get("works_count", "?") + display = best["display_name"] + print(f"[FOUND] {name!r}") + print(f" OpenAlex: {display!r}") + print(f" ROR: {ror} (works: {count})") + if count == 0: + print(f" WARNING: works_count=0 — double check!") + fixes[fname] = ror + + time.sleep(0.4) + +# Now apply the patches +print("\n--- APPLYING PATCHES ---") +for fname, new_ror in fixes.items(): + if new_ror is None: + print(f"[SKIP] {fname} — no ROR found") + continue + jf = config_dir / fname + with open(jf) as f: + data = json.load(f) + old_ror = data["ror"] + if old_ror == new_ror: + print(f"[SAME] {fname} — already correct") + continue + data["ror"] = new_ror + with open(jf, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + print(f"[PATCHED] {fname}: {old_ror} -> {new_ror}") + +print("\nDone. Run verify_rors.py again to confirm all are fixed.") diff --git a/scratch/fix_rors_manual.py b/scratch/fix_rors_manual.py index 2a27e0d56e7475fac4a8b88ebbde6cf5dc910cc1..39e42e3c3d61139a4d5ebaea14b876459e12024f 100644 --- a/scratch/fix_rors_manual.py +++ b/scratch/fix_rors_manual.py @@ -1,56 +1,56 @@ -""" -Manual ROR lookup for institutions not found by name search. -Uses OpenAlex institution search with alternate spellings/names. -""" -import json -import time -import urllib.request -import urllib.parse -from pathlib import Path - -BASE = "https://api.openalex.org" -MAILTO = "cokiki@unilag.edu.ng" - -# Alternate search terms for institutions not found by direct name -ALTERNATES = { - "agostinhoneto.json": ["Agostinho Neto", "UAN Angola", "Luanda university"], - "kinshasa.json": ["Kinshasa university", "UNIKIN", "Congo kinshasa"], - "marienngouabi.json": ["Marien Ngouabi", "Brazzaville university", "Congo Brazzaville"], - "masuku.json": ["Masuku", "Franceville", "Gabon university science"], - "mohammedv.json": ["Mohammed V", "Rabat university", "Mohammed 5"], - "tunis.json": ["Tunis El Manar", "Tunis university", "UTM Tunisia"], - "yaoundei.json": ["Yaounde", "Cameroon university", "Yaounde 1"], -} - -config_dir = Path(__file__).parent.parent / "config" / "institutions" - -for fname, search_terms in ALTERNATES.items(): - jf = config_dir / fname - with open(jf) as f: - data = json.load(f) - name = data["name"] - print(f"\n=== {name} ===") - - found = False - for term in search_terms: - q = urllib.parse.quote(term) - url = f"{BASE}/institutions?search={q}&per-page=5&mailto={MAILTO}" - try: - req = urllib.request.urlopen(url, timeout=15) - resp = json.loads(req.read()) - results = resp.get("results", []) - except Exception as e: - print(f" ERROR searching {term!r}: {e}") - time.sleep(1) - continue - - if results: - print(f" Search '{term}' -> {len(results)} results:") - for r in results[:3]: - print(f" [{r['works_count']:6d} works] {r['display_name']} | ROR: {r['ror']}") - found = True - else: - print(f" Search '{term}' -> no results") - time.sleep(0.4) - if found: - break +""" +Manual ROR lookup for institutions not found by name search. +Uses OpenAlex institution search with alternate spellings/names. +""" +import json +import time +import urllib.request +import urllib.parse +from pathlib import Path + +BASE = "https://api.openalex.org" +MAILTO = "cokiki@unilag.edu.ng" + +# Alternate search terms for institutions not found by direct name +ALTERNATES = { + "agostinhoneto.json": ["Agostinho Neto", "UAN Angola", "Luanda university"], + "kinshasa.json": ["Kinshasa university", "UNIKIN", "Congo kinshasa"], + "marienngouabi.json": ["Marien Ngouabi", "Brazzaville university", "Congo Brazzaville"], + "masuku.json": ["Masuku", "Franceville", "Gabon university science"], + "mohammedv.json": ["Mohammed V", "Rabat university", "Mohammed 5"], + "tunis.json": ["Tunis El Manar", "Tunis university", "UTM Tunisia"], + "yaoundei.json": ["Yaounde", "Cameroon university", "Yaounde 1"], +} + +config_dir = Path(__file__).parent.parent / "config" / "institutions" + +for fname, search_terms in ALTERNATES.items(): + jf = config_dir / fname + with open(jf) as f: + data = json.load(f) + name = data["name"] + print(f"\n=== {name} ===") + + found = False + for term in search_terms: + q = urllib.parse.quote(term) + url = f"{BASE}/institutions?search={q}&per-page=5&mailto={MAILTO}" + try: + req = urllib.request.urlopen(url, timeout=15) + resp = json.loads(req.read()) + results = resp.get("results", []) + except Exception as e: + print(f" ERROR searching {term!r}: {e}") + time.sleep(1) + continue + + if results: + print(f" Search '{term}' -> {len(results)} results:") + for r in results[:3]: + print(f" [{r['works_count']:6d} works] {r['display_name']} | ROR: {r['ror']}") + found = True + else: + print(f" Search '{term}' -> no results") + time.sleep(0.4) + if found: + break diff --git a/scratch/generate_all_configs.py b/scratch/generate_all_configs.py index d9544ba42adc003cd4350d2493e4601680d6d4a5..afc3616fc85d2bce6124015815ec662700ac0345 100644 --- a/scratch/generate_all_configs.py +++ b/scratch/generate_all_configs.py @@ -1,145 +1,145 @@ -import json -import os - -# Sub-regions and countries -subregions = { - "North Africa": ["Egypt", "Morocco", "Algeria", "Tunisia", "Libya", "Sudan"], - "West Africa": ["Nigeria", "Ghana", "Senegal", "Cote d'Ivoire", "Benin", "Burkina Faso", "Cape Verde", "Gambia", "Guinea", "Guinea-Bissau", "Liberia", "Mali", "Mauritania", "Niger", "Sierra Leone", "Togo"], - "East Africa": ["Kenya", "Uganda", "Tanzania", "Ethiopia", "Rwanda", "Burundi", "Djibouti", "Eritrea", "Somalia", "South Sudan", "Madagascar", "Mauritius", "Seychelles", "Comoros"], - "Southern Africa": ["South Africa", "Zimbabwe", "Zambia", "Namibia", "Botswana", "Lesotho", "Eswatini", "Malawi", "Mozambique"], - "Central Africa": ["Cameroon", "DR Congo", "Angola", "Gabon", "Republic of the Congo", "Central African Republic", "Chad", "Equatorial Guinea", "Sao Tome and Principe"], -} - -curated_universities = { - "Egypt": [ - {"name": "Cairo University", "ror": "https://ror.org/03c4mpy73"}, - {"name": "Ain Shams University", "ror": "https://ror.org/034x7p097"}, - {"name": "Alexandria University", "ror": "https://ror.org/02078r490"}, - {"name": "Mansoura University", "ror": "https://ror.org/032p18087"}, - {"name": "Assiut University", "ror": "https://ror.org/047fpp722"}, - ], - "Morocco": [ - {"name": "Université Mohammed V de Rabat", "ror": "https://ror.org/03vpy3v17"}, - {"name": "Université Cadi Ayyad", "ror": "https://ror.org/0154pcf71"}, - {"name": "Université Hassan II de Casablanca", "ror": "https://ror.org/013y27r38"}, - ], - "Tunisia": [ - {"name": "Université de Tunis El Manar", "ror": "https://ror.org/050j3a172"}, - {"name": "Université de Sfax", "ror": "https://ror.org/02157p641"}, - {"name": "Université de Carthage", "ror": "https://ror.org/011yvpy71"}, - ], - "Cameroon": [ - {"name": "Université de Yaoundé I", "ror": "https://ror.org/04h7g6177"}, - {"name": "Université de Dschang", "ror": "https://ror.org/012y7p041"}, - {"name": "Université de Douala", "ror": "https://ror.org/041y27r28"}, - ], - "DR Congo": [ - {"name": "Université de Kinshasa", "ror": "https://ror.org/05vzwad88"}, - {"name": "Université de Lubumbashi", "ror": "https://ror.org/02rry3m21"}, - ], - "Angola": [ - {"name": "Université Agostinho Neto", "ror": "https://ror.org/00z2bpt98"} - ], - "Gabon": [ - {"name": "Université des Sciences et Techniques de Masuku", "ror": "https://ror.org/059gqse72"}, - {"name": "Université Omar Bongo", "ror": "https://ror.org/041yp1812"}, - ], - "Republic of the Congo": [ - {"name": "Université Marien Ngouabi", "ror": "https://ror.org/02y1sra05"} - ], - "Nigeria": [ - {"name": "University of Lagos", "ror": "https://ror.org/05rk03822"}, - {"name": "University of Ibadan", "ror": "https://ror.org/01es5me90"}, - {"name": "Covenant University", "ror": "https://ror.org/02n05rk12"}, - {"name": "Obafemi Awolowo University", "ror": "https://ror.org/013pcr241"}, - {"name": "University of Nigeria Nsukka", "ror": "https://ror.org/02kpy5732"}, - ], - "Ghana": [ - {"name": "University of Ghana", "ror": "https://ror.org/00zpy3v12"}, - {"name": "Kwame Nkrumah University of Science and Technology", "ror": "https://ror.org/00x4mpy73"}, - {"name": "University of Cape Coast", "ror": "https://ror.org/01es3v123"}, - ], - "South Africa": [ - {"name": "University of Cape Town", "ror": "https://ror.org/017620319"}, - {"name": "Stellenbosch University", "ror": "https://ror.org/05777p686"}, - {"name": "University of the Witwatersrand", "ror": "https://ror.org/039482g93"}, - {"name": "University of Pretoria", "ror": "https://ror.org/047fpp722"}, - {"name": "University of KwaZulu-Natal", "ror": "https://ror.org/01267r312"}, - ], - "Zimbabwe": [ - {"name": "University of Zimbabwe", "ror": "https://ror.org/03w489125"}, - {"name": "National University of Science and Technology", "ror": "https://ror.org/01y6mpy73"}, - ], - "Kenya": [ - {"name": "University of Nairobi", "ror": "https://ror.org/01078r490"}, - {"name": "Kenyatta University", "ror": "https://ror.org/01py3v171"}, - {"name": "Jomo Kenyatta University of Agriculture and Technology", "ror": "https://ror.org/03pyvpy71"}, - ], - "Uganda": [ - {"name": "Makerere University", "ror": "https://ror.org/05vzwad88"}, - {"name": "Mbarara University of Science and Technology", "ror": "https://ror.org/0155pcf71"}, - ], - "Tanzania": [ - {"name": "University of Dar es Salaam", "ror": "https://ror.org/0199e1957"}, - {"name": "Sokoine University of Agriculture", "ror": "https://ror.org/011y27r38"}, - ], - "Ethiopia": [ - {"name": "Addis Ababa University", "ror": "https://ror.org/01py3v171"} - ], - "Rwanda": [ - {"name": "University of Rwanda", "ror": "https://ror.org/02yr01r27"} - ], -} - -os.makedirs("config/institutions", exist_ok=True) -count = 0 - -for country, unis in curated_universities.items(): - region = next(r for r, c in subregions.items() if country in c) - for u in unis: - # Create a safe shortname / filename - short_name = u["name"].replace("University of ", "").replace("Université de ", "").replace("Université ", "") - if len(short_name.split()) > 3: - short_name = "".join([word[0] for word in short_name.split() if word.istitle()]) - if not short_name: - short_name = u["name"].split()[0] - - # Overrides for some known ones - if "Lagos" in u["name"]: short_name = "UNILAG" - elif "Ibadan" in u["name"]: short_name = "UI" - elif "Cape Town" in u["name"]: short_name = "UCT" - elif "Witwatersrand" in u["name"]: short_name = "Wits" - elif "Kwame Nkrumah" in u["name"]: short_name = "KNUST" - elif "Yaoundé" in u["name"]: short_name = "Yaounde I" - - file_name = "".join(x for x in short_name.lower() if x.isalnum()) + ".json" - - cfg = { - "ror": u["ror"], - "name": u["name"], - "short_name": short_name, - "country": country, - "sub_region": region, - "staff_file": f"data/{short_name.lower().replace(' ', '_')}_staff.json", - "affiliation_patterns": [u["name"], short_name, f"{u['name']} Department"], - "faculties": [ - "Science", - "Humanities", - "Engineering", - "Medicine", - "Social Sciences", - "Arts", - "Law", - ], - "crawler_settings": { - "rate_limit": 2.0, - "concurrent_requests": 8, - "retry_times": 3, - "download_delay": 2.0, - }, - } - with open(f"config/institutions/{file_name}", "w", encoding="utf-8") as f: - json.dump(cfg, f, indent=2, ensure_ascii=False) - count += 1 - -print(f"Generated {count} university configurations.") +import json +import os + +# Sub-regions and countries +subregions = { + "North Africa": ["Egypt", "Morocco", "Algeria", "Tunisia", "Libya", "Sudan"], + "West Africa": ["Nigeria", "Ghana", "Senegal", "Cote d'Ivoire", "Benin", "Burkina Faso", "Cape Verde", "Gambia", "Guinea", "Guinea-Bissau", "Liberia", "Mali", "Mauritania", "Niger", "Sierra Leone", "Togo"], + "East Africa": ["Kenya", "Uganda", "Tanzania", "Ethiopia", "Rwanda", "Burundi", "Djibouti", "Eritrea", "Somalia", "South Sudan", "Madagascar", "Mauritius", "Seychelles", "Comoros"], + "Southern Africa": ["South Africa", "Zimbabwe", "Zambia", "Namibia", "Botswana", "Lesotho", "Eswatini", "Malawi", "Mozambique"], + "Central Africa": ["Cameroon", "DR Congo", "Angola", "Gabon", "Republic of the Congo", "Central African Republic", "Chad", "Equatorial Guinea", "Sao Tome and Principe"], +} + +curated_universities = { + "Egypt": [ + {"name": "Cairo University", "ror": "https://ror.org/03c4mpy73"}, + {"name": "Ain Shams University", "ror": "https://ror.org/034x7p097"}, + {"name": "Alexandria University", "ror": "https://ror.org/02078r490"}, + {"name": "Mansoura University", "ror": "https://ror.org/032p18087"}, + {"name": "Assiut University", "ror": "https://ror.org/047fpp722"}, + ], + "Morocco": [ + {"name": "Université Mohammed V de Rabat", "ror": "https://ror.org/03vpy3v17"}, + {"name": "Université Cadi Ayyad", "ror": "https://ror.org/0154pcf71"}, + {"name": "Université Hassan II de Casablanca", "ror": "https://ror.org/013y27r38"}, + ], + "Tunisia": [ + {"name": "Université de Tunis El Manar", "ror": "https://ror.org/050j3a172"}, + {"name": "Université de Sfax", "ror": "https://ror.org/02157p641"}, + {"name": "Université de Carthage", "ror": "https://ror.org/011yvpy71"}, + ], + "Cameroon": [ + {"name": "Université de Yaoundé I", "ror": "https://ror.org/04h7g6177"}, + {"name": "Université de Dschang", "ror": "https://ror.org/012y7p041"}, + {"name": "Université de Douala", "ror": "https://ror.org/041y27r28"}, + ], + "DR Congo": [ + {"name": "Université de Kinshasa", "ror": "https://ror.org/05vzwad88"}, + {"name": "Université de Lubumbashi", "ror": "https://ror.org/02rry3m21"}, + ], + "Angola": [ + {"name": "Université Agostinho Neto", "ror": "https://ror.org/00z2bpt98"} + ], + "Gabon": [ + {"name": "Université des Sciences et Techniques de Masuku", "ror": "https://ror.org/059gqse72"}, + {"name": "Université Omar Bongo", "ror": "https://ror.org/041yp1812"}, + ], + "Republic of the Congo": [ + {"name": "Université Marien Ngouabi", "ror": "https://ror.org/02y1sra05"} + ], + "Nigeria": [ + {"name": "University of Lagos", "ror": "https://ror.org/05rk03822"}, + {"name": "University of Ibadan", "ror": "https://ror.org/01es5me90"}, + {"name": "Covenant University", "ror": "https://ror.org/02n05rk12"}, + {"name": "Obafemi Awolowo University", "ror": "https://ror.org/013pcr241"}, + {"name": "University of Nigeria Nsukka", "ror": "https://ror.org/02kpy5732"}, + ], + "Ghana": [ + {"name": "University of Ghana", "ror": "https://ror.org/00zpy3v12"}, + {"name": "Kwame Nkrumah University of Science and Technology", "ror": "https://ror.org/00x4mpy73"}, + {"name": "University of Cape Coast", "ror": "https://ror.org/01es3v123"}, + ], + "South Africa": [ + {"name": "University of Cape Town", "ror": "https://ror.org/017620319"}, + {"name": "Stellenbosch University", "ror": "https://ror.org/05777p686"}, + {"name": "University of the Witwatersrand", "ror": "https://ror.org/039482g93"}, + {"name": "University of Pretoria", "ror": "https://ror.org/047fpp722"}, + {"name": "University of KwaZulu-Natal", "ror": "https://ror.org/01267r312"}, + ], + "Zimbabwe": [ + {"name": "University of Zimbabwe", "ror": "https://ror.org/03w489125"}, + {"name": "National University of Science and Technology", "ror": "https://ror.org/01y6mpy73"}, + ], + "Kenya": [ + {"name": "University of Nairobi", "ror": "https://ror.org/01078r490"}, + {"name": "Kenyatta University", "ror": "https://ror.org/01py3v171"}, + {"name": "Jomo Kenyatta University of Agriculture and Technology", "ror": "https://ror.org/03pyvpy71"}, + ], + "Uganda": [ + {"name": "Makerere University", "ror": "https://ror.org/05vzwad88"}, + {"name": "Mbarara University of Science and Technology", "ror": "https://ror.org/0155pcf71"}, + ], + "Tanzania": [ + {"name": "University of Dar es Salaam", "ror": "https://ror.org/0199e1957"}, + {"name": "Sokoine University of Agriculture", "ror": "https://ror.org/011y27r38"}, + ], + "Ethiopia": [ + {"name": "Addis Ababa University", "ror": "https://ror.org/01py3v171"} + ], + "Rwanda": [ + {"name": "University of Rwanda", "ror": "https://ror.org/02yr01r27"} + ], +} + +os.makedirs("config/institutions", exist_ok=True) +count = 0 + +for country, unis in curated_universities.items(): + region = next(r for r, c in subregions.items() if country in c) + for u in unis: + # Create a safe shortname / filename + short_name = u["name"].replace("University of ", "").replace("Université de ", "").replace("Université ", "") + if len(short_name.split()) > 3: + short_name = "".join([word[0] for word in short_name.split() if word.istitle()]) + if not short_name: + short_name = u["name"].split()[0] + + # Overrides for some known ones + if "Lagos" in u["name"]: short_name = "UNILAG" + elif "Ibadan" in u["name"]: short_name = "UI" + elif "Cape Town" in u["name"]: short_name = "UCT" + elif "Witwatersrand" in u["name"]: short_name = "Wits" + elif "Kwame Nkrumah" in u["name"]: short_name = "KNUST" + elif "Yaoundé" in u["name"]: short_name = "Yaounde I" + + file_name = "".join(x for x in short_name.lower() if x.isalnum()) + ".json" + + cfg = { + "ror": u["ror"], + "name": u["name"], + "short_name": short_name, + "country": country, + "sub_region": region, + "staff_file": f"data/{short_name.lower().replace(' ', '_')}_staff.json", + "affiliation_patterns": [u["name"], short_name, f"{u['name']} Department"], + "faculties": [ + "Science", + "Humanities", + "Engineering", + "Medicine", + "Social Sciences", + "Arts", + "Law", + ], + "crawler_settings": { + "rate_limit": 2.0, + "concurrent_requests": 8, + "retry_times": 3, + "download_delay": 2.0, + }, + } + with open(f"config/institutions/{file_name}", "w", encoding="utf-8") as f: + json.dump(cfg, f, indent=2, ensure_ascii=False) + count += 1 + +print(f"Generated {count} university configurations.") diff --git a/scratch/generate_creds.py b/scratch/generate_creds.py index 9957c2108b4bb6214dd266b1c8c99f3b55448e62..0d628515a4719221b03917efbb3fb6637d5461fc 100644 --- a/scratch/generate_creds.py +++ b/scratch/generate_creds.py @@ -1,18 +1,18 @@ -import secrets -from werkzeug.security import generate_password_hash - -secret_key = secrets.token_hex(32) -admin_pw = "uraas_admin_2026" -viewer_pw = "uraas_viewer_2026" - -print("=== NEW URAAS CREDENTIALS ===") -print("DASHBOARD_SECRET_KEY=" + secret_key) -print("ADMIN_USERNAME=admin") -print("ADMIN_PASSWORD_HASH=" + generate_password_hash(admin_pw)) -print("VIEWER_USERNAME=viewer") -print("VIEWER_PASSWORD_HASH=" + generate_password_hash(viewer_pw)) -print() -print(f"Admin plain password: {admin_pw}") -print(f"Viewer plain password: {viewer_pw}") -print() -print("Store these safely in the server environment!") +import secrets +from werkzeug.security import generate_password_hash + +secret_key = secrets.token_hex(32) +admin_pw = "uraas_admin_2026" +viewer_pw = "uraas_viewer_2026" + +print("=== NEW URAAS CREDENTIALS ===") +print("DASHBOARD_SECRET_KEY=" + secret_key) +print("ADMIN_USERNAME=admin") +print("ADMIN_PASSWORD_HASH=" + generate_password_hash(admin_pw)) +print("VIEWER_USERNAME=viewer") +print("VIEWER_PASSWORD_HASH=" + generate_password_hash(viewer_pw)) +print() +print(f"Admin plain password: {admin_pw}") +print(f"Viewer plain password: {viewer_pw}") +print() +print("Store these safely in the server environment!") diff --git a/scratch/inspect_citations.py b/scratch/inspect_citations.py index 220e99da5381a2748c0399cbc60a77f56448b3e4..78785dfd3ee4a1807bbd5afb83b49a3383f0187b 100644 --- a/scratch/inspect_citations.py +++ b/scratch/inspect_citations.py @@ -1,30 +1,30 @@ -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from sqlalchemy import text - -from uraas.database import Base, SessionLocal, engine - - -def inspect(): - session = SessionLocal() - try: - # Check if tables exist first - tables = ["citations", "citation_metrics", "author_metrics"] - for table in tables: - try: - res = session.execute(text(f"SELECT COUNT(*) FROM {table}")).scalar() - print(f"Table '{table}' has {res} rows") - except Exception as e: - print(f"Table '{table}' error: {e}") - - except Exception as e: - print(f"Error: {e}") - finally: - session.close() - - -if __name__ == "__main__": - inspect() +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from sqlalchemy import text + +from uraas.database import Base, SessionLocal, engine + + +def inspect(): + session = SessionLocal() + try: + # Check if tables exist first + tables = ["citations", "citation_metrics", "author_metrics"] + for table in tables: + try: + res = session.execute(text(f"SELECT COUNT(*) FROM {table}")).scalar() + print(f"Table '{table}' has {res} rows") + except Exception as e: + print(f"Table '{table}' error: {e}") + + except Exception as e: + print(f"Error: {e}") + finally: + session.close() + + +if __name__ == "__main__": + inspect() diff --git a/scratch/inspect_db.py b/scratch/inspect_db.py index 93a32915ae0a8f0146832c5f2397b8456c5fe51e..ea7d96bb9853086bd3b0a39516744550e1ac278e 100644 --- a/scratch/inspect_db.py +++ b/scratch/inspect_db.py @@ -1,73 +1,73 @@ -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from sqlalchemy import func - -from uraas.database import Item, SessionLocal - - -def inspect(): - session = SessionLocal() - try: - total_items = session.query(Item).count() - print(f"Total items: {total_items}") - - # Group by institution/ror - inst_counts = ( - session.query(Item.institution, Item.ror, func.count(Item.id)) - .group_by(Item.institution, Item.ror) - .all() - ) - print("\nItems per institution:") - for inst, ror, count in inst_counts: - print(f" - {inst} ({ror}): {count} papers") - - # African language papers - african_lang_count = ( - session.query(func.count(Item.id)) - .filter(Item.is_african_language == True) - .scalar() - ) - print(f"\nAfrican language papers: {african_lang_count}") - - # TK vitality papers - tk_count = ( - session.query(func.count(Item.id)) - .filter( - (Item.tk_label.isnot(None)) - | (Item.content_type == "indigenous_knowledge") - ) - .scalar() - ) - print(f"Indigenous knowledge / TK papers: {tk_count}") - - # Patents - patent_count = ( - session.query(func.count(Item.id)) - .filter(Item.patent_id.isnot(None)) - .scalar() - ) - print(f"Patents: {patent_count}") - - # DocID coverage - docid_count = ( - session.query(func.count(Item.id)).filter(Item.docid.isnot(None)).scalar() - ) - print(f"DocID assigned papers: {docid_count}") - - # Access policy / PDFs - from uraas.database import File - - pdf_count = session.query(File).count() - print(f"Downloaded PDFs: {pdf_count}") - - except Exception as e: - print(f"Error: {e}") - finally: - session.close() - - -if __name__ == "__main__": - inspect() +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from sqlalchemy import func + +from uraas.database import Item, SessionLocal + + +def inspect(): + session = SessionLocal() + try: + total_items = session.query(Item).count() + print(f"Total items: {total_items}") + + # Group by institution/ror + inst_counts = ( + session.query(Item.institution, Item.ror, func.count(Item.id)) + .group_by(Item.institution, Item.ror) + .all() + ) + print("\nItems per institution:") + for inst, ror, count in inst_counts: + print(f" - {inst} ({ror}): {count} papers") + + # African language papers + african_lang_count = ( + session.query(func.count(Item.id)) + .filter(Item.is_african_language == True) + .scalar() + ) + print(f"\nAfrican language papers: {african_lang_count}") + + # TK vitality papers + tk_count = ( + session.query(func.count(Item.id)) + .filter( + (Item.tk_label.isnot(None)) + | (Item.content_type == "indigenous_knowledge") + ) + .scalar() + ) + print(f"Indigenous knowledge / TK papers: {tk_count}") + + # Patents + patent_count = ( + session.query(func.count(Item.id)) + .filter(Item.patent_id.isnot(None)) + .scalar() + ) + print(f"Patents: {patent_count}") + + # DocID coverage + docid_count = ( + session.query(func.count(Item.id)).filter(Item.docid.isnot(None)).scalar() + ) + print(f"DocID assigned papers: {docid_count}") + + # Access policy / PDFs + from uraas.database import File + + pdf_count = session.query(File).count() + print(f"Downloaded PDFs: {pdf_count}") + + except Exception as e: + print(f"Error: {e}") + finally: + session.close() + + +if __name__ == "__main__": + inspect() diff --git a/scratch/replica_smoke_test.sh b/scratch/replica_smoke_test.sh index 5f9f04d2dc5591796b0638b25cc2ffd503f53436..cf3a75d8d8dac45c37b23f9488a1aafd4c9d747c 100644 --- a/scratch/replica_smoke_test.sh +++ b/scratch/replica_smoke_test.sh @@ -1,80 +1,80 @@ -#!/usr/bin/env bash -# UNILAG mounting dry-run — Phase G smoke tests (mounting guide §5). -# Runs against the production replica stack. -# -# NOTE on cookies: the app sets SESSION_COOKIE_SECURE=True in production, so the -# session cookie is only sent over HTTPS. All authenticated flows therefore go -# through the nginx TLS reverse proxy (https://localhost:8443, -k = self-signed), -# which is the real production request path anyway. Unauthenticated checks use -# the direct gunicorn port (18080) to prove the app itself is hardened. -set -u - -APP=http://localhost:18080 # gunicorn app (direct, bypasses nginx) -TLS=https://localhost:8443 # nginx TLS reverse proxy (prod path) -J=/tmp/uraas_cookies.txt -PASS=0; FAIL=0 -ok(){ echo " PASS: $1"; PASS=$((PASS+1)); } -no(){ echo " FAIL: $1"; FAIL=$((FAIL+1)); } -CURL="curl -sk" # -k: trust the self-signed dry-run cert - -echo "=== 1. /health returns 200 (app direct + nginx TLS) ===" -code=$($CURL -o /dev/null -w '%{http_code}' $APP/health) -[ "$code" = "200" ] && ok "app /health = 200" || no "app /health = $code" -code=$($CURL -o /dev/null -w '%{http_code}' $TLS/health) -[ "$code" = "200" ] && ok "nginx TLS /health = 200" || no "nginx TLS /health = $code" - -echo "=== 2. HTTP -> HTTPS redirect (nginx :8081) ===" -loc=$($CURL -o /dev/null -w '%{http_code} %{redirect_url}' http://localhost:8081/) -echo " $loc" -echo "$loc" | grep -q "301" && ok "HTTP returns 301 redirect to https" || no "no 301 redirect: $loc" - -echo "=== 3. Anonymous API control route -> 401 ===" -code=$($CURL -o /dev/null -w '%{http_code}' $APP/api/crawler/status) -[ "$code" = "401" ] && ok "anon /api/crawler/status = 401" || no "anon crawler status = $code" -code=$($CURL -o /dev/null -w '%{http_code}' $APP/api/analytics/overview) -[ "$code" = "401" ] && ok "anon /api/analytics/overview = 401" || no "anon analytics = $code" -code=$($CURL -o /dev/null -w '%{http_code}' $APP/api/staff/directory) -echo " (staff directory PII, anon): $code" -[ "$code" = "401" ] && ok "anon staff directory (PII) = 401" || no "anon staff directory = $code" - -echo "=== 4. Anonymous HTML route -> redirect to /login ===" -loc=$($CURL -o /dev/null -w '%{http_code} %{redirect_url}' "$APP/") -echo " $loc" -echo "$loc" | grep -qE "302.*/login" && ok "anon / redirects to /login" || no "anon / = $loc" - -echo "=== 5. Security headers (CSP, XFO, nosniff, HSTS over TLS) ===" -h=$($CURL -D - -o /dev/null $APP/login) -echo "$h" | grep -qi "Content-Security-Policy" && ok "CSP header present" || no "CSP missing" -echo "$h" | grep -qi "X-Frame-Options: DENY" && ok "X-Frame-Options DENY" || no "XFO missing" -echo "$h" | grep -qi "X-Content-Type-Options: nosniff" && ok "nosniff present" || no "nosniff missing" -echo "$h" | grep -qi "Strict-Transport-Security" && ok "HSTS present (prod)" || no "HSTS missing" -ht=$($CURL -D - -o /dev/null $TLS/login) -echo "$ht" | grep -qi "Strict-Transport-Security" && ok "HSTS present via nginx TLS" || no "HSTS via nginx missing" - -echo "=== 6. Bad login -> 401, no session granted ===" -code=$($CURL -o /dev/null -w '%{http_code}' -d "username=admin&password=wrong" $TLS/login) -[ "$code" = "401" ] && ok "bad login = 401" || no "bad login = $code" - -echo "=== 7. Admin login works + reaches admin-only route (over TLS) ===" -rm -f $J -code=$($CURL -o /dev/null -w '%{http_code}' -c $J -d "username=admin&password=UnilagAdmin#2026" $TLS/login) -echo " admin login status (302 expected): $code" -[ "$code" = "302" ] && ok "admin login = 302" || no "admin login = $code" -code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/crawler/status) -[ "$code" = "200" ] && ok "admin reaches crawler status (200)" || no "admin crawler status = $code" - -echo "=== 8. Viewer login works but is NOT admin (crawler -> 403) ===" -rm -f $J -$CURL -o /dev/null -c $J -d "username=viewer&password=UnilagView#2026" $TLS/login -code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/crawler/status) -[ "$code" = "403" ] && ok "viewer crawler status = 403 (admin-only)" || no "viewer crawler status = $code" -code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/analytics/overview) -[ "$code" = "200" ] && ok "viewer reads analytics (200)" || no "viewer analytics = $code" -code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/staff/directory) -[ "$code" = "403" ] && ok "viewer staff directory (PII) = 403" || no "viewer staff directory = $code" - -echo -echo "================= SMOKE TEST SUMMARY =================" -echo " PASSED: $PASS FAILED: $FAIL" -echo "=====================================================" -[ "$FAIL" = "0" ] && exit 0 || exit 1 +#!/usr/bin/env bash +# UNILAG mounting dry-run — Phase G smoke tests (mounting guide §5). +# Runs against the production replica stack. +# +# NOTE on cookies: the app sets SESSION_COOKIE_SECURE=True in production, so the +# session cookie is only sent over HTTPS. All authenticated flows therefore go +# through the nginx TLS reverse proxy (https://localhost:8443, -k = self-signed), +# which is the real production request path anyway. Unauthenticated checks use +# the direct gunicorn port (18080) to prove the app itself is hardened. +set -u + +APP=http://localhost:18080 # gunicorn app (direct, bypasses nginx) +TLS=https://localhost:8443 # nginx TLS reverse proxy (prod path) +J=/tmp/uraas_cookies.txt +PASS=0; FAIL=0 +ok(){ echo " PASS: $1"; PASS=$((PASS+1)); } +no(){ echo " FAIL: $1"; FAIL=$((FAIL+1)); } +CURL="curl -sk" # -k: trust the self-signed dry-run cert + +echo "=== 1. /health returns 200 (app direct + nginx TLS) ===" +code=$($CURL -o /dev/null -w '%{http_code}' $APP/health) +[ "$code" = "200" ] && ok "app /health = 200" || no "app /health = $code" +code=$($CURL -o /dev/null -w '%{http_code}' $TLS/health) +[ "$code" = "200" ] && ok "nginx TLS /health = 200" || no "nginx TLS /health = $code" + +echo "=== 2. HTTP -> HTTPS redirect (nginx :8081) ===" +loc=$($CURL -o /dev/null -w '%{http_code} %{redirect_url}' http://localhost:8081/) +echo " $loc" +echo "$loc" | grep -q "301" && ok "HTTP returns 301 redirect to https" || no "no 301 redirect: $loc" + +echo "=== 3. Anonymous API control route -> 401 ===" +code=$($CURL -o /dev/null -w '%{http_code}' $APP/api/crawler/status) +[ "$code" = "401" ] && ok "anon /api/crawler/status = 401" || no "anon crawler status = $code" +code=$($CURL -o /dev/null -w '%{http_code}' $APP/api/analytics/overview) +[ "$code" = "401" ] && ok "anon /api/analytics/overview = 401" || no "anon analytics = $code" +code=$($CURL -o /dev/null -w '%{http_code}' $APP/api/staff/directory) +echo " (staff directory PII, anon): $code" +[ "$code" = "401" ] && ok "anon staff directory (PII) = 401" || no "anon staff directory = $code" + +echo "=== 4. Anonymous HTML route -> redirect to /login ===" +loc=$($CURL -o /dev/null -w '%{http_code} %{redirect_url}' "$APP/") +echo " $loc" +echo "$loc" | grep -qE "302.*/login" && ok "anon / redirects to /login" || no "anon / = $loc" + +echo "=== 5. Security headers (CSP, XFO, nosniff, HSTS over TLS) ===" +h=$($CURL -D - -o /dev/null $APP/login) +echo "$h" | grep -qi "Content-Security-Policy" && ok "CSP header present" || no "CSP missing" +echo "$h" | grep -qi "X-Frame-Options: DENY" && ok "X-Frame-Options DENY" || no "XFO missing" +echo "$h" | grep -qi "X-Content-Type-Options: nosniff" && ok "nosniff present" || no "nosniff missing" +echo "$h" | grep -qi "Strict-Transport-Security" && ok "HSTS present (prod)" || no "HSTS missing" +ht=$($CURL -D - -o /dev/null $TLS/login) +echo "$ht" | grep -qi "Strict-Transport-Security" && ok "HSTS present via nginx TLS" || no "HSTS via nginx missing" + +echo "=== 6. Bad login -> 401, no session granted ===" +code=$($CURL -o /dev/null -w '%{http_code}' -d "username=admin&password=wrong" $TLS/login) +[ "$code" = "401" ] && ok "bad login = 401" || no "bad login = $code" + +echo "=== 7. Admin login works + reaches admin-only route (over TLS) ===" +rm -f $J +code=$($CURL -o /dev/null -w '%{http_code}' -c $J -d "username=admin&password=UnilagAdmin#2026" $TLS/login) +echo " admin login status (302 expected): $code" +[ "$code" = "302" ] && ok "admin login = 302" || no "admin login = $code" +code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/crawler/status) +[ "$code" = "200" ] && ok "admin reaches crawler status (200)" || no "admin crawler status = $code" + +echo "=== 8. Viewer login works but is NOT admin (crawler -> 403) ===" +rm -f $J +$CURL -o /dev/null -c $J -d "username=viewer&password=UnilagView#2026" $TLS/login +code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/crawler/status) +[ "$code" = "403" ] && ok "viewer crawler status = 403 (admin-only)" || no "viewer crawler status = $code" +code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/analytics/overview) +[ "$code" = "200" ] && ok "viewer reads analytics (200)" || no "viewer analytics = $code" +code=$($CURL -o /dev/null -w '%{http_code}' -b $J $TLS/api/staff/directory) +[ "$code" = "403" ] && ok "viewer staff directory (PII) = 403" || no "viewer staff directory = $code" + +echo +echo "================= SMOKE TEST SUMMARY =================" +echo " PASSED: $PASS FAILED: $FAIL" +echo "=====================================================" +[ "$FAIL" = "0" ] && exit 0 || exit 1 diff --git a/scratch/verify_rors.py b/scratch/verify_rors.py index 6b378a2e20dc8d9c3aa8f4da85b05531d182dd63..8e9741eebeee78da1129aff12294810c3a1c44fc 100644 --- a/scratch/verify_rors.py +++ b/scratch/verify_rors.py @@ -1,39 +1,39 @@ -""" -Script to verify all institution RORs against OpenAlex API. -Identifies wrong RORs by checking if count > 0. -""" -import json -import time -import urllib.request -from pathlib import Path - -BASE = "https://api.openalex.org" -MAILTO = "cokiki@unilag.edu.ng" - -config_dir = Path(__file__).parent.parent / "config" / "institutions" -results = {} - -for jf in sorted(config_dir.glob("*.json")): - with open(jf) as f: - data = json.load(f) - name = data["name"] - ror_full = data["ror"] - ror_short = ror_full.split("/")[-1] - - url = f"{BASE}/works?filter=institutions.ror:{ror_short}&select=id&per-page=1&mailto={MAILTO}" - try: - req = urllib.request.urlopen(url, timeout=10) - resp = json.loads(req.read()) - count = resp["meta"]["count"] - except Exception as e: - count = f"ERROR: {e}" - - status = "OK" if isinstance(count, int) and count > 0 else "ZERO/ERROR" - print(f"[{status:5}] {name:40s} ROR: {ror_short} count={count}") - results[jf.name] = {"name": name, "ror": ror_short, "count": count, "status": status} - time.sleep(0.5) - -print("\n--- PROBLEM INSTITUTIONS ---") -for fname, r in results.items(): - if r["status"] != "OK": - print(f" {fname}: {r['name']} -- ROR {r['ror']} returns {r['count']}") +""" +Script to verify all institution RORs against OpenAlex API. +Identifies wrong RORs by checking if count > 0. +""" +import json +import time +import urllib.request +from pathlib import Path + +BASE = "https://api.openalex.org" +MAILTO = "cokiki@unilag.edu.ng" + +config_dir = Path(__file__).parent.parent / "config" / "institutions" +results = {} + +for jf in sorted(config_dir.glob("*.json")): + with open(jf) as f: + data = json.load(f) + name = data["name"] + ror_full = data["ror"] + ror_short = ror_full.split("/")[-1] + + url = f"{BASE}/works?filter=institutions.ror:{ror_short}&select=id&per-page=1&mailto={MAILTO}" + try: + req = urllib.request.urlopen(url, timeout=10) + resp = json.loads(req.read()) + count = resp["meta"]["count"] + except Exception as e: + count = f"ERROR: {e}" + + status = "OK" if isinstance(count, int) and count > 0 else "ZERO/ERROR" + print(f"[{status:5}] {name:40s} ROR: {ror_short} count={count}") + results[jf.name] = {"name": name, "ror": ror_short, "count": count, "status": status} + time.sleep(0.5) + +print("\n--- PROBLEM INSTITUTIONS ---") +for fname, r in results.items(): + if r["status"] != "OK": + print(f" {fname}: {r['name']} -- ROR {r['ror']} returns {r['count']}") diff --git a/scripts/backfill_alignment.py b/scripts/backfill_alignment.py index a5dd6bae31aee0fc869b86e638506eb3a859077c..756ecd2f7b44f4c9131d8fd0243c98d7b18fec3e 100644 --- a/scripts/backfill_alignment.py +++ b/scripts/backfill_alignment.py @@ -1,99 +1,99 @@ -""" -Backfill framework alignment scores for existing items and rebuild the -AlignmentAggregate table (per institution + global). - -Usage: - python scripts/backfill_alignment.py # DRY RUN — counts only - python scripts/backfill_alignment.py --apply - python scripts/backfill_alignment.py --apply --force # re-score current-version items - -Safe to re-run: items already at ALIGNMENT_VERSION are skipped unless --force. -No network needed beyond the one-time embedding-model download. -""" - -import argparse -import json -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from uraas.config.alignment_frameworks import ALIGNMENT_VERSION -from uraas.database import Item, SessionLocal -from uraas.services.alignment_engine import ( - recompute_aggregates, - score_item_alignment, - scoring_mode, -) -from uraas.utils.analytics_cache import analytics_cache - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)") - parser.add_argument("--force", action="store_true", help="Re-score items already at current version") - parser.add_argument("--batch", type=int, default=500) - args = parser.parse_args() - - session = SessionLocal() - try: - q = session.query(Item) - if not args.force: - q = q.filter( - (Item.alignment_version.is_(None)) - | (Item.alignment_version < ALIGNMENT_VERSION) - ) - todo = q.count() - total = session.query(Item).count() - print("=" * 64) - print(f"Scoring mode: {scoring_mode()} | version: {ALIGNMENT_VERSION}") - print(f"Items to score: {todo} / {total}") - print("=" * 64) - if not args.apply: - print("[DRY RUN] No writes. Re-run with --apply.") - return 0 - - scored = aligned = 0 - framework_hits = {} - while True: - batch = q.limit(args.batch).all() - if not batch: - break - for it in batch: - j, v = score_item_alignment( - it.title or "", it.abstract or "", it.dc_subject or "" - ) - it.alignment_scores = j - it.alignment_version = v - scored += 1 - if j: - aligned += 1 - for fk in json.loads(j): - framework_hits[fk] = framework_hits.get(fk, 0) + 1 - session.commit() - print(f" scored {scored}/{todo}") - - print("\nPer-framework items with alignment:") - for fk, n in sorted(framework_hits.items(), key=lambda kv: -kv[1]): - print(f" {fk:24s} {n}") - - # Aggregates: global + each distinct institution - rows = recompute_aggregates(session, None) - institutions = [ - i for (i,) in session.query(Item.institution).distinct() if i - ] - for inst in institutions: - rows += recompute_aggregates(session, inst) - print(f"\nAggregate rows written: {rows} ({1 + len(institutions)} scopes)") - - analytics_cache.invalidate_all() - print("\n" + "=" * 64) - print(f"DONE. scored={scored} with_alignment={aligned}") - print("=" * 64) - return 0 - finally: - session.close() - - -if __name__ == "__main__": - sys.exit(main()) +""" +Backfill framework alignment scores for existing items and rebuild the +AlignmentAggregate table (per institution + global). + +Usage: + python scripts/backfill_alignment.py # DRY RUN — counts only + python scripts/backfill_alignment.py --apply + python scripts/backfill_alignment.py --apply --force # re-score current-version items + +Safe to re-run: items already at ALIGNMENT_VERSION are skipped unless --force. +No network needed beyond the one-time embedding-model download. +""" + +import argparse +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from uraas.config.alignment_frameworks import ALIGNMENT_VERSION +from uraas.database import Item, SessionLocal +from uraas.services.alignment_engine import ( + recompute_aggregates, + score_item_alignment, + scoring_mode, +) +from uraas.utils.analytics_cache import analytics_cache + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)") + parser.add_argument("--force", action="store_true", help="Re-score items already at current version") + parser.add_argument("--batch", type=int, default=500) + args = parser.parse_args() + + session = SessionLocal() + try: + q = session.query(Item) + if not args.force: + q = q.filter( + (Item.alignment_version.is_(None)) + | (Item.alignment_version < ALIGNMENT_VERSION) + ) + todo = q.count() + total = session.query(Item).count() + print("=" * 64) + print(f"Scoring mode: {scoring_mode()} | version: {ALIGNMENT_VERSION}") + print(f"Items to score: {todo} / {total}") + print("=" * 64) + if not args.apply: + print("[DRY RUN] No writes. Re-run with --apply.") + return 0 + + scored = aligned = 0 + framework_hits = {} + while True: + batch = q.limit(args.batch).all() + if not batch: + break + for it in batch: + j, v = score_item_alignment( + it.title or "", it.abstract or "", it.dc_subject or "" + ) + it.alignment_scores = j + it.alignment_version = v + scored += 1 + if j: + aligned += 1 + for fk in json.loads(j): + framework_hits[fk] = framework_hits.get(fk, 0) + 1 + session.commit() + print(f" scored {scored}/{todo}") + + print("\nPer-framework items with alignment:") + for fk, n in sorted(framework_hits.items(), key=lambda kv: -kv[1]): + print(f" {fk:24s} {n}") + + # Aggregates: global + each distinct institution + rows = recompute_aggregates(session, None) + institutions = [ + i for (i,) in session.query(Item.institution).distinct() if i + ] + for inst in institutions: + rows += recompute_aggregates(session, inst) + print(f"\nAggregate rows written: {rows} ({1 + len(institutions)} scopes)") + + analytics_cache.invalidate_all() + print("\n" + "=" * 64) + print(f"DONE. scored={scored} with_alignment={aligned}") + print("=" * 64) + return 0 + finally: + session.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/backfill_citation_velocity.py b/scripts/backfill_citation_velocity.py index 896135d3238d943710b6e7a208e5ef7027575baa..face98443e76b0b4117c664112b721c0539ebbcf 100644 --- a/scripts/backfill_citation_velocity.py +++ b/scripts/backfill_citation_velocity.py @@ -1,81 +1,81 @@ -""" -Backfill Pan-African citation share (and citation velocity for items missed -by the collaboration backfill). - -The share is one OpenAlex request per item (filter=cites:W... grouped by -citing-country), so run it for the most-cited works first: - - python scripts/backfill_citation_velocity.py # DRY RUN - python scripts/backfill_citation_velocity.py --apply --limit 200 - -Skips items whose share is already computed unless --force. -""" - -import argparse -import json -import os -import sys -import time - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from uraas.database import Item, SessionLocal -from uraas.services.citation_tracker import CitationTracker -from uraas.utils.analytics_cache import analytics_cache - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)") - parser.add_argument("--limit", type=int, default=200, help="Max items (most-cited first)") - parser.add_argument("--force", action="store_true", help="Recompute existing shares") - args = parser.parse_args() - - session = SessionLocal() - try: - q = ( - session.query(Item) - .filter(Item.openalex_id.isnot(None), Item.cited_by_count > 0) - .order_by(Item.cited_by_count.desc()) - ) - if not args.force: - q = q.filter(Item.african_citation_share.is_(None)) - items = q.limit(args.limit).all() - - print("=" * 64) - print(f"Items to process (most-cited first): {len(items)}") - print("=" * 64) - if not args.apply: - print("[DRY RUN] No API calls or writes. Re-run with --apply.") - return 0 - - updated = velocity_fixed = 0 - for i, it in enumerate(items, 1): - share = CitationTracker.fetch_african_citation_share(it.openalex_id) - if share is not None: - it.african_citation_share = share - updated += 1 - # Opportunistic velocity fix for items missing counts_by_year - if not it.counts_by_year: - vel = CitationTracker.fetch_work_velocity(it.openalex_id) - if vel and vel["counts_by_year"]: - it.counts_by_year = json.dumps(vel["counts_by_year"]) - it.cited_by_count = vel["cited_by_count"] - velocity_fixed += 1 - if i % 25 == 0: - session.commit() - print(f" {i}/{len(items)} processed (share set: {updated})") - time.sleep(1.0) - session.commit() - analytics_cache.invalidate_all() - - print("\n" + "=" * 64) - print(f"DONE. shares set={updated} velocity backfilled={velocity_fixed}") - print("=" * 64) - return 0 - finally: - session.close() - - -if __name__ == "__main__": - sys.exit(main()) +""" +Backfill Pan-African citation share (and citation velocity for items missed +by the collaboration backfill). + +The share is one OpenAlex request per item (filter=cites:W... grouped by +citing-country), so run it for the most-cited works first: + + python scripts/backfill_citation_velocity.py # DRY RUN + python scripts/backfill_citation_velocity.py --apply --limit 200 + +Skips items whose share is already computed unless --force. +""" + +import argparse +import json +import os +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from uraas.database import Item, SessionLocal +from uraas.services.citation_tracker import CitationTracker +from uraas.utils.analytics_cache import analytics_cache + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)") + parser.add_argument("--limit", type=int, default=200, help="Max items (most-cited first)") + parser.add_argument("--force", action="store_true", help="Recompute existing shares") + args = parser.parse_args() + + session = SessionLocal() + try: + q = ( + session.query(Item) + .filter(Item.openalex_id.isnot(None), Item.cited_by_count > 0) + .order_by(Item.cited_by_count.desc()) + ) + if not args.force: + q = q.filter(Item.african_citation_share.is_(None)) + items = q.limit(args.limit).all() + + print("=" * 64) + print(f"Items to process (most-cited first): {len(items)}") + print("=" * 64) + if not args.apply: + print("[DRY RUN] No API calls or writes. Re-run with --apply.") + return 0 + + updated = velocity_fixed = 0 + for i, it in enumerate(items, 1): + share = CitationTracker.fetch_african_citation_share(it.openalex_id) + if share is not None: + it.african_citation_share = share + updated += 1 + # Opportunistic velocity fix for items missing counts_by_year + if not it.counts_by_year: + vel = CitationTracker.fetch_work_velocity(it.openalex_id) + if vel and vel["counts_by_year"]: + it.counts_by_year = json.dumps(vel["counts_by_year"]) + it.cited_by_count = vel["cited_by_count"] + velocity_fixed += 1 + if i % 25 == 0: + session.commit() + print(f" {i}/{len(items)} processed (share set: {updated})") + time.sleep(1.0) + session.commit() + analytics_cache.invalidate_all() + + print("\n" + "=" * 64) + print(f"DONE. shares set={updated} velocity backfilled={velocity_fixed}") + print("=" * 64) + return 0 + finally: + session.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/backfill_collaboration_data.py b/scripts/backfill_collaboration_data.py index f60cdfa2f7384af2965d729ebf54b2c3a364dfc5..ace1aafc2955a5a30decec63106e11424ec7653d 100644 --- a/scripts/backfill_collaboration_data.py +++ b/scripts/backfill_collaboration_data.py @@ -1,227 +1,227 @@ -""" -Backfill collaboration + citation data for existing items from OpenAlex. - -Re-fetches each item's OpenAlex record (batched 50 DOIs per request to -conserve API quota) and populates: - - item_affiliations rows (institution / ROR / country per authorship) - - items.coauthor_countries / african_country_count / is_intra_african - - items.openalex_id / cited_by_count / counts_by_year - -Usage: - python scripts/backfill_collaboration_data.py # DRY RUN - python scripts/backfill_collaboration_data.py --apply - python scripts/backfill_collaboration_data.py --apply --limit 500 - python scripts/backfill_collaboration_data.py --apply --force # redo enriched rows - -Idempotent: items that already have affiliation rows are skipped unless ---force. Respects ~1 req/sec. Set OPENALEX_API_KEY in the environment. -""" - -import argparse -import json -import os -import sys -import time - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from uraas.config.african_countries import african_countries_in -from uraas.database import Item, ItemAffiliation, SessionLocal -from uraas.utils.analytics_cache import analytics_cache -from uraas.utils.openalex_client import oa_get - -BATCH = 50 -SELECT = "id,doi,authorships,cited_by_count,counts_by_year" - - -def _norm_doi(doi: str) -> str: - return ( - (doi or "") - .replace("https://doi.org/", "") - .replace("http://dx.doi.org/", "") - .strip() - .lower() - ) - - -def fetch_batch_by_doi(dois): - """One OpenAlex call for up to 50 DOIs. Returns {normalized_doi: work}. - - DOIs are pipe-joined raw — requests URL-encodes the whole filter param; - pre-quoting each DOI double-encodes and matches nothing.""" - flt = "doi:" + "|".join(dois) - data = oa_get("/works", {"filter": flt, "select": SELECT, "per-page": BATCH}) - out = {} - for work in (data or {}).get("results", []): - nd = _norm_doi(work.get("doi", "")) - if nd: - out[nd] = work - return out - - -def fetch_batch_by_openalex_id(ids): - """One OpenAlex call for up to 50 OpenAlex work IDs.""" - flt = "openalex_id:" + "|".join(ids) - data = oa_get("/works", {"filter": flt, "select": SELECT, "per-page": BATCH}) - out = {} - for work in (data or {}).get("results", []): - wid = work.get("id", "").replace("https://openalex.org/", "") - if wid: - out[wid] = work - return out - - -def extract_affiliations(work): - """(ror_short, name) -> {ror, name, country_code, author_count}.""" - rows = {} - for authorship in work.get("authorships", []): - for inst in authorship.get("institutions", []): - name = inst.get("display_name", "") or "" - ror = (inst.get("ror") or "").replace("https://ror.org/", "") - cc = (inst.get("country_code") or "").upper() - if not (name or ror): - continue - row = rows.setdefault( - (ror, name), - {"ror": ror, "name": name, "country_code": cc, "author_count": 0}, - ) - row["author_count"] += 1 - if cc and not row["country_code"]: - row["country_code"] = cc - return list(rows.values()) - - -def apply_work(session, item, work): - """Write affiliation rows + collaboration/citation columns for one item.""" - affs = extract_affiliations(work) - - # Idempotency: replace any existing affiliation rows for this item. - session.query(ItemAffiliation).filter_by(item_id=item.id).delete() - for aff in affs: - session.add( - ItemAffiliation( - item_id=item.id, - ror=(aff["ror"] or "")[:128] or None, - institution_name=(aff["name"] or "")[:255] or None, - country_code=(aff["country_code"] or "")[:2] or None, - author_count=aff["author_count"], - ) - ) - - african = african_countries_in(a["country_code"] for a in affs) - item.coauthor_countries = ",".join(african) or None - item.african_country_count = len(african) - item.is_intra_african = len(african) >= 2 - - item.openalex_id = work.get("id", "").replace("https://openalex.org/", "") or None - item.cited_by_count = work.get("cited_by_count", 0) or 0 - cby = work.get("counts_by_year") or [] - item.counts_by_year = json.dumps(cby) if cby else None - return len(affs), item.is_intra_african - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)") - parser.add_argument("--limit", type=int, default=0, help="Max items to process (0 = all)") - parser.add_argument( - "--force", action="store_true", help="Re-fetch items that already have affiliation data" - ) - args = parser.parse_args() - - session = SessionLocal() - try: - q = session.query(Item) - if not args.force: - enriched = {i for (i,) in session.query(ItemAffiliation.item_id).distinct()} - else: - enriched = set() - - items = [ - it - for it in q.all() - if it.id not in enriched and (it.doi or "openalex.org" in (it.url or "")) - ] - skipped_no_id = q.count() - len(items) - len(enriched & {it.id for it in q}) - if args.limit: - items = items[: args.limit] - - print("=" * 64) - print(f"Items to enrich: {len(items)} (already enriched, skipped: {len(enriched)})") - print("=" * 64) - if not args.apply: - print("[DRY RUN] No API calls or writes. Re-run with --apply.") - return 0 - - by_doi = [it for it in items if it.doi] - by_oaid = [ - it for it in items if not it.doi and "openalex.org" in (it.url or "") - ] - - updated = intra = not_found = 0 - - # ── DOI batches ────────────────────────────────────────────────── - doi_map = {_norm_doi(it.doi): it for it in by_doi} - doi_keys = list(doi_map) - for start in range(0, len(doi_keys), BATCH): - chunk = doi_keys[start : start + BATCH] - works = fetch_batch_by_doi(chunk) - for nd in chunk: - it = doi_map[nd] - work = works.get(nd) - if not work: - not_found += 1 - continue - _, is_ia = apply_work(session, it, work) - updated += 1 - intra += int(is_ia) - session.commit() - print( - f" [doi {start + len(chunk)}/{len(doi_keys)}] " - f"updated={updated} intra_african={intra} not_found={not_found}" - ) - time.sleep(1.0) - - # ── OpenAlex-ID batches (items without DOI) ────────────────────── - oaid_map = {} - for it in by_oaid: - wid = (it.url or "").rstrip("/").split("/")[-1] - if wid.startswith("W"): - oaid_map[wid] = it - oaid_keys = list(oaid_map) - for start in range(0, len(oaid_keys), BATCH): - chunk = oaid_keys[start : start + BATCH] - works = fetch_batch_by_openalex_id(chunk) - for wid in chunk: - it = oaid_map[wid] - work = works.get(wid) - if not work: - not_found += 1 - continue - _, is_ia = apply_work(session, it, work) - updated += 1 - intra += int(is_ia) - session.commit() - print( - f" [oaid {start + len(chunk)}/{len(oaid_keys)}] " - f"updated={updated} intra_african={intra} not_found={not_found}" - ) - time.sleep(1.0) - - analytics_cache.invalidate_all() - total_ia = session.query(Item).filter(Item.is_intra_african.is_(True)).count() - total = session.query(Item).count() - print("\n" + "=" * 64) - print(f"DONE. updated={updated} not_found={not_found}") - print( - f"Repository intra-African collaboration: {total_ia}/{total} " - f"({(total_ia / total * 100) if total else 0:.1f}%) — continental baseline ~8.4%" - ) - print("=" * 64) - return 0 - finally: - session.close() - - -if __name__ == "__main__": - sys.exit(main()) +""" +Backfill collaboration + citation data for existing items from OpenAlex. + +Re-fetches each item's OpenAlex record (batched 50 DOIs per request to +conserve API quota) and populates: + - item_affiliations rows (institution / ROR / country per authorship) + - items.coauthor_countries / african_country_count / is_intra_african + - items.openalex_id / cited_by_count / counts_by_year + +Usage: + python scripts/backfill_collaboration_data.py # DRY RUN + python scripts/backfill_collaboration_data.py --apply + python scripts/backfill_collaboration_data.py --apply --limit 500 + python scripts/backfill_collaboration_data.py --apply --force # redo enriched rows + +Idempotent: items that already have affiliation rows are skipped unless +--force. Respects ~1 req/sec. Set OPENALEX_API_KEY in the environment. +""" + +import argparse +import json +import os +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from uraas.config.african_countries import african_countries_in +from uraas.database import Item, ItemAffiliation, SessionLocal +from uraas.utils.analytics_cache import analytics_cache +from uraas.utils.openalex_client import oa_get + +BATCH = 50 +SELECT = "id,doi,authorships,cited_by_count,counts_by_year" + + +def _norm_doi(doi: str) -> str: + return ( + (doi or "") + .replace("https://doi.org/", "") + .replace("http://dx.doi.org/", "") + .strip() + .lower() + ) + + +def fetch_batch_by_doi(dois): + """One OpenAlex call for up to 50 DOIs. Returns {normalized_doi: work}. + + DOIs are pipe-joined raw — requests URL-encodes the whole filter param; + pre-quoting each DOI double-encodes and matches nothing.""" + flt = "doi:" + "|".join(dois) + data = oa_get("/works", {"filter": flt, "select": SELECT, "per-page": BATCH}) + out = {} + for work in (data or {}).get("results", []): + nd = _norm_doi(work.get("doi", "")) + if nd: + out[nd] = work + return out + + +def fetch_batch_by_openalex_id(ids): + """One OpenAlex call for up to 50 OpenAlex work IDs.""" + flt = "openalex_id:" + "|".join(ids) + data = oa_get("/works", {"filter": flt, "select": SELECT, "per-page": BATCH}) + out = {} + for work in (data or {}).get("results", []): + wid = work.get("id", "").replace("https://openalex.org/", "") + if wid: + out[wid] = work + return out + + +def extract_affiliations(work): + """(ror_short, name) -> {ror, name, country_code, author_count}.""" + rows = {} + for authorship in work.get("authorships", []): + for inst in authorship.get("institutions", []): + name = inst.get("display_name", "") or "" + ror = (inst.get("ror") or "").replace("https://ror.org/", "") + cc = (inst.get("country_code") or "").upper() + if not (name or ror): + continue + row = rows.setdefault( + (ror, name), + {"ror": ror, "name": name, "country_code": cc, "author_count": 0}, + ) + row["author_count"] += 1 + if cc and not row["country_code"]: + row["country_code"] = cc + return list(rows.values()) + + +def apply_work(session, item, work): + """Write affiliation rows + collaboration/citation columns for one item.""" + affs = extract_affiliations(work) + + # Idempotency: replace any existing affiliation rows for this item. + session.query(ItemAffiliation).filter_by(item_id=item.id).delete() + for aff in affs: + session.add( + ItemAffiliation( + item_id=item.id, + ror=(aff["ror"] or "")[:128] or None, + institution_name=(aff["name"] or "")[:255] or None, + country_code=(aff["country_code"] or "")[:2] or None, + author_count=aff["author_count"], + ) + ) + + african = african_countries_in(a["country_code"] for a in affs) + item.coauthor_countries = ",".join(african) or None + item.african_country_count = len(african) + item.is_intra_african = len(african) >= 2 + + item.openalex_id = work.get("id", "").replace("https://openalex.org/", "") or None + item.cited_by_count = work.get("cited_by_count", 0) or 0 + cby = work.get("counts_by_year") or [] + item.counts_by_year = json.dumps(cby) if cby else None + return len(affs), item.is_intra_african + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)") + parser.add_argument("--limit", type=int, default=0, help="Max items to process (0 = all)") + parser.add_argument( + "--force", action="store_true", help="Re-fetch items that already have affiliation data" + ) + args = parser.parse_args() + + session = SessionLocal() + try: + q = session.query(Item) + if not args.force: + enriched = {i for (i,) in session.query(ItemAffiliation.item_id).distinct()} + else: + enriched = set() + + items = [ + it + for it in q.all() + if it.id not in enriched and (it.doi or "openalex.org" in (it.url or "")) + ] + skipped_no_id = q.count() - len(items) - len(enriched & {it.id for it in q}) + if args.limit: + items = items[: args.limit] + + print("=" * 64) + print(f"Items to enrich: {len(items)} (already enriched, skipped: {len(enriched)})") + print("=" * 64) + if not args.apply: + print("[DRY RUN] No API calls or writes. Re-run with --apply.") + return 0 + + by_doi = [it for it in items if it.doi] + by_oaid = [ + it for it in items if not it.doi and "openalex.org" in (it.url or "") + ] + + updated = intra = not_found = 0 + + # ── DOI batches ────────────────────────────────────────────────── + doi_map = {_norm_doi(it.doi): it for it in by_doi} + doi_keys = list(doi_map) + for start in range(0, len(doi_keys), BATCH): + chunk = doi_keys[start : start + BATCH] + works = fetch_batch_by_doi(chunk) + for nd in chunk: + it = doi_map[nd] + work = works.get(nd) + if not work: + not_found += 1 + continue + _, is_ia = apply_work(session, it, work) + updated += 1 + intra += int(is_ia) + session.commit() + print( + f" [doi {start + len(chunk)}/{len(doi_keys)}] " + f"updated={updated} intra_african={intra} not_found={not_found}" + ) + time.sleep(1.0) + + # ── OpenAlex-ID batches (items without DOI) ────────────────────── + oaid_map = {} + for it in by_oaid: + wid = (it.url or "").rstrip("/").split("/")[-1] + if wid.startswith("W"): + oaid_map[wid] = it + oaid_keys = list(oaid_map) + for start in range(0, len(oaid_keys), BATCH): + chunk = oaid_keys[start : start + BATCH] + works = fetch_batch_by_openalex_id(chunk) + for wid in chunk: + it = oaid_map[wid] + work = works.get(wid) + if not work: + not_found += 1 + continue + _, is_ia = apply_work(session, it, work) + updated += 1 + intra += int(is_ia) + session.commit() + print( + f" [oaid {start + len(chunk)}/{len(oaid_keys)}] " + f"updated={updated} intra_african={intra} not_found={not_found}" + ) + time.sleep(1.0) + + analytics_cache.invalidate_all() + total_ia = session.query(Item).filter(Item.is_intra_african.is_(True)).count() + total = session.query(Item).count() + print("\n" + "=" * 64) + print(f"DONE. updated={updated} not_found={not_found}") + print( + f"Repository intra-African collaboration: {total_ia}/{total} " + f"({(total_ia / total * 100) if total else 0:.1f}%) — continental baseline ~8.4%" + ) + print("=" * 64) + return 0 + finally: + session.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/backfill_pids.py b/scripts/backfill_pids.py index 786797a3fcec3007f955fbc7331a4231681418b4..a4b08671245d172f4320c8fc08c8914f14f7d3f8 100644 --- a/scripts/backfill_pids.py +++ b/scripts/backfill_pids.py @@ -1,70 +1,70 @@ -""" -Backfill persistent identifiers: DocID™ + ARK for items missing them. - -ARKs are minted deterministically from the item's DocID hash, so re-running -is idempotent. No network access needed. - -Usage: - python scripts/backfill_pids.py # DRY RUN - python scripts/backfill_pids.py --apply -""" - -import argparse -import os -import sys -from datetime import datetime - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from uraas.database import Item, SessionLocal -from uraas.utils.ark_generator import ark_generator -from uraas.utils.docid_generator import docid_generator - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)") - args = parser.parse_args() - - session = SessionLocal() - try: - need_docid = session.query(Item).filter(Item.docid.is_(None)).count() - need_ark = session.query(Item).filter(Item.ark.is_(None)).count() - print("=" * 64) - print(f"Items missing DocID: {need_docid} missing ARK: {need_ark}") - print("=" * 64) - if not args.apply: - print("[DRY RUN] No writes. Re-run with --apply.") - return 0 - - minted_docid = minted_ark = 0 - now = datetime.utcnow() - for it in session.query(Item).filter( - (Item.docid.is_(None)) | (Item.ark.is_(None)) - ): - if not it.docid: - it.docid = docid_generator.generate_docid( - title=it.title or "", - doi=it.doi, - institution=it.institution or "Unknown", - timestamp=it.publication_date, - ) - it.docid_assigned_at = now - minted_docid += 1 - if not it.ark: - it.ark = ark_generator.mint(it.docid) - it.ark_assigned_at = now - minted_ark += 1 - session.commit() - - print(f"DONE. DocIDs minted: {minted_docid} ARKs minted: {minted_ark}") - sample = session.query(Item.ark).filter(Item.ark.isnot(None)).first() - if sample: - print(f"Sample ARK: {sample[0]} (valid: {ark_generator.validate(sample[0])})") - return 0 - finally: - session.close() - - -if __name__ == "__main__": - sys.exit(main()) +""" +Backfill persistent identifiers: DocID™ + ARK for items missing them. + +ARKs are minted deterministically from the item's DocID hash, so re-running +is idempotent. No network access needed. + +Usage: + python scripts/backfill_pids.py # DRY RUN + python scripts/backfill_pids.py --apply +""" + +import argparse +import os +import sys +from datetime import datetime + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from uraas.database import Item, SessionLocal +from uraas.utils.ark_generator import ark_generator +from uraas.utils.docid_generator import docid_generator + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)") + args = parser.parse_args() + + session = SessionLocal() + try: + need_docid = session.query(Item).filter(Item.docid.is_(None)).count() + need_ark = session.query(Item).filter(Item.ark.is_(None)).count() + print("=" * 64) + print(f"Items missing DocID: {need_docid} missing ARK: {need_ark}") + print("=" * 64) + if not args.apply: + print("[DRY RUN] No writes. Re-run with --apply.") + return 0 + + minted_docid = minted_ark = 0 + now = datetime.utcnow() + for it in session.query(Item).filter( + (Item.docid.is_(None)) | (Item.ark.is_(None)) + ): + if not it.docid: + it.docid = docid_generator.generate_docid( + title=it.title or "", + doi=it.doi, + institution=it.institution or "Unknown", + timestamp=it.publication_date, + ) + it.docid_assigned_at = now + minted_docid += 1 + if not it.ark: + it.ark = ark_generator.mint(it.docid) + it.ark_assigned_at = now + minted_ark += 1 + session.commit() + + print(f"DONE. DocIDs minted: {minted_docid} ARKs minted: {minted_ark}") + sample = session.query(Item.ark).filter(Item.ark.isnot(None)).first() + if sample: + print(f"Sample ARK: {sample[0]} (valid: {ark_generator.validate(sample[0])})") + return 0 + finally: + session.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/backfill_special_collections.py b/scripts/backfill_special_collections.py index ac643d004d4ea7893566157def09fed64466e687..0efab77f9229693f84b7505aeb7ab2bdfd830614 100644 --- a/scripts/backfill_special_collections.py +++ b/scripts/backfill_special_collections.py @@ -1,69 +1,69 @@ -""" -Backfill special_collection_score + special_collection_categories on existing items. - -Runs classify_special_collections() over every Item (title + abstract + dc_subject) -and writes the score/categories. Idempotent — re-running on already-scored rows -produces the same values. -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from uraas.database import Item, SessionLocal -from uraas.utils.ai_classifier import classify_special_collections - -BATCH_SIZE = 500 - - -def main() -> int: - session = SessionLocal() - try: - total = session.query(Item).count() - print(f"Backfilling SC score for {total} items...") - - scored = 0 - hits = 0 - offset = 0 - while offset < total: - batch = ( - session.query(Item) - .order_by(Item.id) - .offset(offset) - .limit(BATCH_SIZE) - .all() - ) - if not batch: - break - - for item in batch: - sc = classify_special_collections( - item.title or "", - item.abstract or "", - item.dc_subject or "", - ) - if sc: - item.special_collection_score = float(sum(h["score"] for h in sc)) - item.special_collection_categories = ",".join( - h["category"] for h in sc - ) - hits += 1 - else: - item.special_collection_score = 0.0 - item.special_collection_categories = "" - scored += 1 - - session.commit() - offset += len(batch) - print(f" {scored}/{total} scored ({hits} SC hits so far)") - - print() - print(f"Done. {scored} items scored, {hits} matched a special collection.") - return 0 - finally: - session.close() - - -if __name__ == "__main__": - sys.exit(main()) +""" +Backfill special_collection_score + special_collection_categories on existing items. + +Runs classify_special_collections() over every Item (title + abstract + dc_subject) +and writes the score/categories. Idempotent — re-running on already-scored rows +produces the same values. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from uraas.database import Item, SessionLocal +from uraas.utils.ai_classifier import classify_special_collections + +BATCH_SIZE = 500 + + +def main() -> int: + session = SessionLocal() + try: + total = session.query(Item).count() + print(f"Backfilling SC score for {total} items...") + + scored = 0 + hits = 0 + offset = 0 + while offset < total: + batch = ( + session.query(Item) + .order_by(Item.id) + .offset(offset) + .limit(BATCH_SIZE) + .all() + ) + if not batch: + break + + for item in batch: + sc = classify_special_collections( + item.title or "", + item.abstract or "", + item.dc_subject or "", + ) + if sc: + item.special_collection_score = float(sum(h["score"] for h in sc)) + item.special_collection_categories = ",".join( + h["category"] for h in sc + ) + hits += 1 + else: + item.special_collection_score = 0.0 + item.special_collection_categories = "" + scored += 1 + + session.commit() + offset += len(batch) + print(f" {scored}/{total} scored ({hits} SC hits so far)") + + print() + print(f"Done. {scored} items scored, {hits} matched a special collection.") + return 0 + finally: + session.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/build_app.py b/scripts/build_app.py index c414d0622844560c7a107004b82b47c5ebe961bc..2ed027c77cf6a3ad11cd8351a6a2cb83ba1859da 100644 --- a/scripts/build_app.py +++ b/scripts/build_app.py @@ -1,26 +1,26 @@ -import os -import sys - -sys.path.insert(0, ".") - -APP = """import os,csv,io,subprocess,threading,re,logging -from flask import Flask,render_template,jsonify,send_file,request,Response -from flask_socketio import SocketIO -from uraas.config import config -from uraas.analytics.engine import analytics -from uraas.database import SessionLocal,Item,File,Author,Community,Collection -from uraas.utils.docid_generator import docid_generator -from sqlalchemy import func,extract,desc,or_ - -app = Flask(__name__) -app.config["SECRET_KEY"] = config.DASHBOARD_SECRET_KEY -socketio = SocketIO(app, cors_allowed_origins="*") -logger = logging.getLogger(__name__) -crawler_process = None -crawler_lock = threading.Lock() -docid_crawler_process = None -docid_crawler_lock = threading.Lock() -""" - -open("uraas/dashboard/app.py", "w", encoding="utf-8").write(APP) -print("wrote", len(APP), "chars") +import os +import sys + +sys.path.insert(0, ".") + +APP = """import os,csv,io,subprocess,threading,re,logging +from flask import Flask,render_template,jsonify,send_file,request,Response +from flask_socketio import SocketIO +from uraas.config import config +from uraas.analytics.engine import analytics +from uraas.database import SessionLocal,Item,File,Author,Community,Collection +from uraas.utils.docid_generator import docid_generator +from sqlalchemy import func,extract,desc,or_ + +app = Flask(__name__) +app.config["SECRET_KEY"] = config.DASHBOARD_SECRET_KEY +socketio = SocketIO(app, cors_allowed_origins="*") +logger = logging.getLogger(__name__) +crawler_process = None +crawler_lock = threading.Lock() +docid_crawler_process = None +docid_crawler_lock = threading.Lock() +""" + +open("uraas/dashboard/app.py", "w", encoding="utf-8").write(APP) +print("wrote", len(APP), "chars") diff --git a/scripts/check_staff.py b/scripts/check_staff.py index 295b0e5bc374f7020e60d8cda12fd246e7bd3698..14ee5f9c6e595114215995f0f5884aeafdf48153 100644 --- a/scripts/check_staff.py +++ b/scripts/check_staff.py @@ -1,9 +1,9 @@ -import os -import sys - -sys.path.insert(0, os.getcwd()) -from uraas.config.institutions import get_registry - -registry = get_registry() -for inst in registry.list_all(): - print(f"{inst.short_name}: {len(inst.staff_names)} staff (File: {inst.staff_file})") +import os +import sys + +sys.path.insert(0, os.getcwd()) +from uraas.config.institutions import get_registry + +registry = get_registry() +for inst in registry.list_all(): + print(f"{inst.short_name}: {len(inst.staff_names)} staff (File: {inst.staff_file})") diff --git a/scripts/clean_database.py b/scripts/clean_database.py index 87b38c1ad5554d8d1e1c42f73c81f63919afa4da..6497c951cc3a73ae0e14390e13b32d5b760c749f 100644 --- a/scripts/clean_database.py +++ b/scripts/clean_database.py @@ -1,156 +1,156 @@ -""" -Database Cleanup Script -Removes bad/misattributed data from the URAAS database. - -Cleanup rules: -1. Remove items from institutions no longer in the registry -2. Remove items with no title, no DOI, no URL, and no authors -3. Remove exact DOI duplicates (keep first by id) -4. Remove items whose title is fewer than 10 chars -5. Flag (do not delete) items with institution mismatch in affiliation -6. Report before/after counts -""" - -import logging -import os -import sys -from datetime import datetime - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) - - -def run_cleanup(dry_run: bool = False): - from sqlalchemy import func - - from uraas.config.institutions import get_registry - from uraas.database import Author, Collection, Community, Item, SessionLocal - - registry = get_registry() - valid_inst_names = {c.name for c in registry.list_all()} - - session = SessionLocal() - try: - total_before = session.query(Item).count() - log.info(f"Starting cleanup | Items before: {total_before}") - - removed = 0 - - # ── Rule 1: Remove items from removed institutions ──────────────────── - all_institutions_in_db = session.query(Item.institution).distinct().all() - stale_insts = [ - r[0] - for r in all_institutions_in_db - if r[0] and r[0] not in valid_inst_names - ] - - if stale_insts: - log.info(f"Found stale institutions: {stale_insts}") - for stale in stale_insts: - stale_items = ( - session.query(Item).filter(Item.institution == stale).all() - ) - log.info(f" [{stale}] {len(stale_items)} items to remove") - if not dry_run: - for item in stale_items: - session.delete(item) - session.commit() - removed += len(stale_items) - - # ── Rule 2: Remove items with no title ──────────────────────────────── - no_title = ( - session.query(Item) - .filter( - (Item.title == None) | (Item.title == "") | (Item.title == "Untitled") - ) - .all() - ) - log.info(f"Items with no/empty title: {len(no_title)}") - if not dry_run: - for item in no_title: - session.delete(item) - session.commit() - removed += len(no_title) - - # ── Rule 3: Remove items with title < 10 chars and no DOI ──────────── - all_short = session.query(Item).filter(Item.doi == None).all() - short_items = [i for i in all_short if i.title and len(i.title.strip()) < 10] - log.info( - f"Items with very short title (<10 chars) and no DOI: {len(short_items)}" - ) - if not dry_run: - for item in short_items: - session.delete(item) - session.commit() - removed += len(short_items) - - # ── Rule 4: Remove exact DOI duplicates (keep lowest id) ───────────── - doi_subq = ( - session.query(Item.doi, func.min(Item.id).label("min_id")) - .filter(Item.doi != None) - .group_by(Item.doi) - .subquery() - ) - - dup_dois = ( - session.query(Item) - .filter(Item.doi != None, Item.id.notin_(session.query(doi_subq.c.min_id))) - .all() - ) - log.info(f"Duplicate DOI items to remove: {len(dup_dois)}") - if not dry_run: - for item in dup_dois: - session.delete(item) - session.commit() - removed += len(dup_dois) - - # ── Rule 5: Remove items with no institution tag ────────────────────── - no_inst = ( - session.query(Item) - .filter((Item.institution == None) | (Item.institution == "")) - .all() - ) - # Only remove those that have no authors either - truly_orphan = [i for i in no_inst if not i.authors] - log.info(f"Items with no institution AND no authors: {len(truly_orphan)}") - if not dry_run: - for item in truly_orphan: - session.delete(item) - session.commit() - removed += len(truly_orphan) - - total_after = session.query(Item).count() - - log.info(f"\n{'='*50}") - log.info(f"CLEANUP COMPLETE {'(DRY RUN)' if dry_run else ''}") - log.info(f" Items before: {total_before}") - log.info(f" Items removed: {removed}") - log.info(f" Items after: {total_after}") - log.info(f"{'='*50}") - - return { - "total_before": total_before, - "removed": removed, - "total_after": total_after, - "dry_run": dry_run, - } - - except Exception as e: - log.error(f"Cleanup error: {e}") - session.rollback() - raise - finally: - session.close() - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser(description="URAAS Database Cleanup") - parser.add_argument( - "--dry-run", action="store_true", help="Report without deleting" - ) - args = parser.parse_args() - run_cleanup(dry_run=args.dry_run) +""" +Database Cleanup Script +Removes bad/misattributed data from the URAAS database. + +Cleanup rules: +1. Remove items from institutions no longer in the registry +2. Remove items with no title, no DOI, no URL, and no authors +3. Remove exact DOI duplicates (keep first by id) +4. Remove items whose title is fewer than 10 chars +5. Flag (do not delete) items with institution mismatch in affiliation +6. Report before/after counts +""" + +import logging +import os +import sys +from datetime import datetime + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) + + +def run_cleanup(dry_run: bool = False): + from sqlalchemy import func + + from uraas.config.institutions import get_registry + from uraas.database import Author, Collection, Community, Item, SessionLocal + + registry = get_registry() + valid_inst_names = {c.name for c in registry.list_all()} + + session = SessionLocal() + try: + total_before = session.query(Item).count() + log.info(f"Starting cleanup | Items before: {total_before}") + + removed = 0 + + # ── Rule 1: Remove items from removed institutions ──────────────────── + all_institutions_in_db = session.query(Item.institution).distinct().all() + stale_insts = [ + r[0] + for r in all_institutions_in_db + if r[0] and r[0] not in valid_inst_names + ] + + if stale_insts: + log.info(f"Found stale institutions: {stale_insts}") + for stale in stale_insts: + stale_items = ( + session.query(Item).filter(Item.institution == stale).all() + ) + log.info(f" [{stale}] {len(stale_items)} items to remove") + if not dry_run: + for item in stale_items: + session.delete(item) + session.commit() + removed += len(stale_items) + + # ── Rule 2: Remove items with no title ──────────────────────────────── + no_title = ( + session.query(Item) + .filter( + (Item.title == None) | (Item.title == "") | (Item.title == "Untitled") + ) + .all() + ) + log.info(f"Items with no/empty title: {len(no_title)}") + if not dry_run: + for item in no_title: + session.delete(item) + session.commit() + removed += len(no_title) + + # ── Rule 3: Remove items with title < 10 chars and no DOI ──────────── + all_short = session.query(Item).filter(Item.doi == None).all() + short_items = [i for i in all_short if i.title and len(i.title.strip()) < 10] + log.info( + f"Items with very short title (<10 chars) and no DOI: {len(short_items)}" + ) + if not dry_run: + for item in short_items: + session.delete(item) + session.commit() + removed += len(short_items) + + # ── Rule 4: Remove exact DOI duplicates (keep lowest id) ───────────── + doi_subq = ( + session.query(Item.doi, func.min(Item.id).label("min_id")) + .filter(Item.doi != None) + .group_by(Item.doi) + .subquery() + ) + + dup_dois = ( + session.query(Item) + .filter(Item.doi != None, Item.id.notin_(session.query(doi_subq.c.min_id))) + .all() + ) + log.info(f"Duplicate DOI items to remove: {len(dup_dois)}") + if not dry_run: + for item in dup_dois: + session.delete(item) + session.commit() + removed += len(dup_dois) + + # ── Rule 5: Remove items with no institution tag ────────────────────── + no_inst = ( + session.query(Item) + .filter((Item.institution == None) | (Item.institution == "")) + .all() + ) + # Only remove those that have no authors either + truly_orphan = [i for i in no_inst if not i.authors] + log.info(f"Items with no institution AND no authors: {len(truly_orphan)}") + if not dry_run: + for item in truly_orphan: + session.delete(item) + session.commit() + removed += len(truly_orphan) + + total_after = session.query(Item).count() + + log.info(f"\n{'='*50}") + log.info(f"CLEANUP COMPLETE {'(DRY RUN)' if dry_run else ''}") + log.info(f" Items before: {total_before}") + log.info(f" Items removed: {removed}") + log.info(f" Items after: {total_after}") + log.info(f"{'='*50}") + + return { + "total_before": total_before, + "removed": removed, + "total_after": total_after, + "dry_run": dry_run, + } + + except Exception as e: + log.error(f"Cleanup error: {e}") + session.rollback() + raise + finally: + session.close() + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="URAAS Database Cleanup") + parser.add_argument( + "--dry-run", action="store_true", help="Report without deleting" + ) + args = parser.parse_args() + run_cleanup(dry_run=args.dry_run) diff --git a/scripts/crawl_multi_institution.py b/scripts/crawl_multi_institution.py index c07f58552e16ea7518152669414f4d17f7a2abca..33f86edb9ada6d8821ce0b756ad9eed525fc67f0 100644 --- a/scripts/crawl_multi_institution.py +++ b/scripts/crawl_multi_institution.py @@ -1,242 +1,242 @@ -""" -Multi-Institution Crawler -Crawls papers for multiple Nigerian universities simultaneously -""" - -import argparse -import os -import subprocess -import sys - -from scrapy.crawler import CrawlerProcess -from scrapy.utils.project import get_project_settings - -# Force unbuffered output so terminal log is in correct order -sys.stdout.reconfigure(line_buffering=True) - -# Add project root to path (parent of scripts/) -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from uraas.config.institutions import get_registry - - -def main(): - parser = argparse.ArgumentParser( - description="Multi-institution research paper crawler" - ) - parser.add_argument( - "--institutions", - type=str, - default="all", - help='Comma-separated list of institution short names, or "all" (default: all)', - ) - parser.add_argument( - "--target", type=int, default=20, help="Target number of papers per institution" - ) - parser.add_argument( - "--spider", - type=str, - default="openalex", - choices=["openalex", "crossref", "arxiv", "orcid", "oai", - "semantic_scholar", "europepmc", "core", "pubmed", - "openaire", "doaj", "ajol", "all"], - help=( - "Spider to use for crawling. " - "'all' fans out across every web source (openalex + crossref + " - "semantic_scholar + europepmc + arxiv + orcid) for maximum coverage." - ), - ) - parser.add_argument( - "--from-date", - dest="from_date", - type=str, - default=None, - help="OAI harvest lower bound YYYY-MM-DD (oai spider only; " - "defaults to a recent look-back window)", - ) - parser.add_argument( - "--until-date", - dest="until_date", - type=str, - default=None, - help="OAI harvest upper bound YYYY-MM-DD (oai spider only; optional)", - ) - parser.add_argument( - "--clean", - action="store_true", - help="Run database cleanup script before crawling", - ) - parser.add_argument( - "--no-boost-special", - dest="boost_special", - action="store_false", - help="Disable Special Collections boost waves (default: boost ON)", - ) - parser.add_argument( - "--sc-only", - action="store_true", - help="Crawl ONLY Special Collections seed waves (skip generic ROR pass)", - ) - parser.set_defaults(boost_special=True) - - args = parser.parse_args() - - if args.clean: - print("\n" + "=" * 60) - print("RUNNING DATABASE CLEANUP") - print("=" * 60) - try: - subprocess.run([sys.executable, "scripts/clean_database.py"], check=True) - print("Cleanup completed successfully.") - except subprocess.CalledProcessError as e: - print(f"Cleanup failed: {e}") - return 1 - - registry = get_registry() - - if args.institutions.lower() == "all": - valid_institutions = [inst.short_name.lower() for inst in registry.list_all()] - else: - # Parse institutions - institution_list = [inst.strip() for inst in args.institutions.split(",")] - valid_institutions = [] - for inst in institution_list: - config = registry.get(inst) - if config: - valid_institutions.append(config.short_name.lower()) - else: - print(f" [NOT FOUND] '{inst}' not found in registry") - - print("\n" + "=" * 60, flush=True) - print("MULTI-INSTITUTION CRAWLER", flush=True) - print("=" * 60, flush=True) - print(f"\nTarget: {args.target} papers total per institution", flush=True) - print(f"Spider: {args.spider}", flush=True) - print(f"\nValidating institutions...", flush=True) - - # Map spider names to classes (defined early so we can validate) - spider_map = { - "openalex": "uraas.spiders.sources.openalex_spider.OpenAlexSpider", - "crossref": "uraas.spiders.sources.crossref_spider.CrossrefSpider", - "arxiv": "uraas.spiders.sources.arxiv_spider.ArxivSpider", - "orcid": "uraas.spiders.sources.orcid_spider.ORCIDSpider", - "oai": "uraas.spiders.sources.oai_spider.OAISpider", - "semantic_scholar":"uraas.spiders.sources.semantic_scholar_spider.SemanticScholarSpider", - "europepmc": "uraas.spiders.sources.europepmc_spider.EuropePMCSpider", - "core": "uraas.spiders.sources.core_spider.CORESpider", - "pubmed": "uraas.spiders.sources.pubmed_spider.PubMedSpider", - "openaire": "uraas.spiders.sources.openaire_spider.OpenAIRESpider", - "doaj": "uraas.spiders.sources.doaj_spider.DOAJSpider", - "ajol": "uraas.spiders.sources.ajol_spider.AJOLSpider", - } - - # "all" = every web-discovery spider (excludes "oai" which reads FROM the IR) - ALL_WEB_SPIDERS = [ - "openalex", "crossref", "semantic_scholar", "europepmc", - "core", "pubmed", "openaire", "doaj", "ajol", "arxiv", "orcid", - ] - - if args.spider == "all": - spider_names_to_run = ALL_WEB_SPIDERS - # Divide target across spiders so total ≈ requested target - per_spider_target = max(1, args.target // len(spider_names_to_run)) - else: - spider_names_to_run = [args.spider] - per_spider_target = args.target - - # Validate + import all spider classes up front so errors appear early - spider_classes = {} - for sname in spider_names_to_run: - path = spider_map.get(sname) - if not path: - print(f"\n[ERR] Spider '{sname}' not supported", flush=True) - return 1 - mod_path, cls_name = path.rsplit(".", 1) - mod = __import__(mod_path, fromlist=[cls_name]) - spider_classes[sname] = getattr(mod, cls_name) - - # Legacy single-spider variable (used below) - spider_class = spider_classes.get(spider_names_to_run[0]) - - for inst in valid_institutions: - config = registry.get(inst) - print(f" [VALID] {config.name} ({config.short_name})", flush=True) - print(f" ROR: {config.ror}", flush=True) - print(f" Staff: {len(config.staff_names)}", flush=True) - - if not valid_institutions: - print("\n[ERR] No valid institutions found. Exiting.", flush=True) - return 1 - - print(f"\n{len(valid_institutions)} institution(s) validated", flush=True) - print("=" * 60, flush=True) - - # Schedule crawls — ONE CrawlerProcess for ALL institutions - print(f"\nScheduling crawls...", flush=True) - settings = get_project_settings() - settings.set( - "ITEM_PIPELINES", - { - "uraas.pipelines.database.DatabaseStoragePipeline": 300, - }, - ) - settings.set("LOG_LEVEL", "INFO") - settings.set("LOG_SCRAPED_ITEMS", False) - settings.set("TELNETCONSOLE_ENABLED", False) - - process = CrawlerProcess(settings) - - print(f" Boost special collections: {args.boost_special}", flush=True) - print(f" SC-only mode: {args.sc_only}", flush=True) - print(f" Spiders: {', '.join(spider_names_to_run)}", flush=True) - - for inst in valid_institutions: - cfg = registry.get(inst) - print(f" -> {cfg.name}", flush=True) - for sname in spider_names_to_run: - scls = spider_classes[sname] - if sname == "oai": - process.crawl( - scls, - institution=inst, - target=per_spider_target, - from_date=args.from_date, - until_date=args.until_date, - ) - else: - process.crawl( - scls, - institution=inst, - target=per_spider_target, - boost_special=args.boost_special, - sc_only=args.sc_only, - ) - - print( - f"\nStarting crawl for {len(valid_institutions)} institution(s)...", flush=True - ) - print("=" * 60, flush=True) - sys.stdout.flush() - - # Start crawling - try: - process.start() - print("\n" + "=" * 60) - print("CRAWL COMPLETED") - print("=" * 60) - return 0 - - except KeyboardInterrupt: - print("\n\n[ERR] Crawl interrupted by user") - return 1 - - except Exception as e: - print(f"\n\n[ERR] Crawl failed: {e}") - import traceback - - traceback.print_exc() - return 1 - - -if __name__ == "__main__": - sys.exit(main()) +""" +Multi-Institution Crawler +Crawls papers for multiple Nigerian universities simultaneously +""" + +import argparse +import os +import subprocess +import sys + +from scrapy.crawler import CrawlerProcess +from scrapy.utils.project import get_project_settings + +# Force unbuffered output so terminal log is in correct order +sys.stdout.reconfigure(line_buffering=True) + +# Add project root to path (parent of scripts/) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from uraas.config.institutions import get_registry + + +def main(): + parser = argparse.ArgumentParser( + description="Multi-institution research paper crawler" + ) + parser.add_argument( + "--institutions", + type=str, + default="all", + help='Comma-separated list of institution short names, or "all" (default: all)', + ) + parser.add_argument( + "--target", type=int, default=20, help="Target number of papers per institution" + ) + parser.add_argument( + "--spider", + type=str, + default="openalex", + choices=["openalex", "crossref", "arxiv", "orcid", "oai", + "semantic_scholar", "europepmc", "core", "pubmed", + "openaire", "doaj", "ajol", "all"], + help=( + "Spider to use for crawling. " + "'all' fans out across every web source (openalex + crossref + " + "semantic_scholar + europepmc + arxiv + orcid) for maximum coverage." + ), + ) + parser.add_argument( + "--from-date", + dest="from_date", + type=str, + default=None, + help="OAI harvest lower bound YYYY-MM-DD (oai spider only; " + "defaults to a recent look-back window)", + ) + parser.add_argument( + "--until-date", + dest="until_date", + type=str, + default=None, + help="OAI harvest upper bound YYYY-MM-DD (oai spider only; optional)", + ) + parser.add_argument( + "--clean", + action="store_true", + help="Run database cleanup script before crawling", + ) + parser.add_argument( + "--no-boost-special", + dest="boost_special", + action="store_false", + help="Disable Special Collections boost waves (default: boost ON)", + ) + parser.add_argument( + "--sc-only", + action="store_true", + help="Crawl ONLY Special Collections seed waves (skip generic ROR pass)", + ) + parser.set_defaults(boost_special=True) + + args = parser.parse_args() + + if args.clean: + print("\n" + "=" * 60) + print("RUNNING DATABASE CLEANUP") + print("=" * 60) + try: + subprocess.run([sys.executable, "scripts/clean_database.py"], check=True) + print("Cleanup completed successfully.") + except subprocess.CalledProcessError as e: + print(f"Cleanup failed: {e}") + return 1 + + registry = get_registry() + + if args.institutions.lower() == "all": + valid_institutions = [inst.short_name.lower() for inst in registry.list_all()] + else: + # Parse institutions + institution_list = [inst.strip() for inst in args.institutions.split(",")] + valid_institutions = [] + for inst in institution_list: + config = registry.get(inst) + if config: + valid_institutions.append(config.short_name.lower()) + else: + print(f" [NOT FOUND] '{inst}' not found in registry") + + print("\n" + "=" * 60, flush=True) + print("MULTI-INSTITUTION CRAWLER", flush=True) + print("=" * 60, flush=True) + print(f"\nTarget: {args.target} papers total per institution", flush=True) + print(f"Spider: {args.spider}", flush=True) + print(f"\nValidating institutions...", flush=True) + + # Map spider names to classes (defined early so we can validate) + spider_map = { + "openalex": "uraas.spiders.sources.openalex_spider.OpenAlexSpider", + "crossref": "uraas.spiders.sources.crossref_spider.CrossrefSpider", + "arxiv": "uraas.spiders.sources.arxiv_spider.ArxivSpider", + "orcid": "uraas.spiders.sources.orcid_spider.ORCIDSpider", + "oai": "uraas.spiders.sources.oai_spider.OAISpider", + "semantic_scholar":"uraas.spiders.sources.semantic_scholar_spider.SemanticScholarSpider", + "europepmc": "uraas.spiders.sources.europepmc_spider.EuropePMCSpider", + "core": "uraas.spiders.sources.core_spider.CORESpider", + "pubmed": "uraas.spiders.sources.pubmed_spider.PubMedSpider", + "openaire": "uraas.spiders.sources.openaire_spider.OpenAIRESpider", + "doaj": "uraas.spiders.sources.doaj_spider.DOAJSpider", + "ajol": "uraas.spiders.sources.ajol_spider.AJOLSpider", + } + + # "all" = every web-discovery spider (excludes "oai" which reads FROM the IR) + ALL_WEB_SPIDERS = [ + "openalex", "crossref", "semantic_scholar", "europepmc", + "core", "pubmed", "openaire", "doaj", "ajol", "arxiv", "orcid", + ] + + if args.spider == "all": + spider_names_to_run = ALL_WEB_SPIDERS + # Divide target across spiders so total ≈ requested target + per_spider_target = max(1, args.target // len(spider_names_to_run)) + else: + spider_names_to_run = [args.spider] + per_spider_target = args.target + + # Validate + import all spider classes up front so errors appear early + spider_classes = {} + for sname in spider_names_to_run: + path = spider_map.get(sname) + if not path: + print(f"\n[ERR] Spider '{sname}' not supported", flush=True) + return 1 + mod_path, cls_name = path.rsplit(".", 1) + mod = __import__(mod_path, fromlist=[cls_name]) + spider_classes[sname] = getattr(mod, cls_name) + + # Legacy single-spider variable (used below) + spider_class = spider_classes.get(spider_names_to_run[0]) + + for inst in valid_institutions: + config = registry.get(inst) + print(f" [VALID] {config.name} ({config.short_name})", flush=True) + print(f" ROR: {config.ror}", flush=True) + print(f" Staff: {len(config.staff_names)}", flush=True) + + if not valid_institutions: + print("\n[ERR] No valid institutions found. Exiting.", flush=True) + return 1 + + print(f"\n{len(valid_institutions)} institution(s) validated", flush=True) + print("=" * 60, flush=True) + + # Schedule crawls — ONE CrawlerProcess for ALL institutions + print(f"\nScheduling crawls...", flush=True) + settings = get_project_settings() + settings.set( + "ITEM_PIPELINES", + { + "uraas.pipelines.database.DatabaseStoragePipeline": 300, + }, + ) + settings.set("LOG_LEVEL", "INFO") + settings.set("LOG_SCRAPED_ITEMS", False) + settings.set("TELNETCONSOLE_ENABLED", False) + + process = CrawlerProcess(settings) + + print(f" Boost special collections: {args.boost_special}", flush=True) + print(f" SC-only mode: {args.sc_only}", flush=True) + print(f" Spiders: {', '.join(spider_names_to_run)}", flush=True) + + for inst in valid_institutions: + cfg = registry.get(inst) + print(f" -> {cfg.name}", flush=True) + for sname in spider_names_to_run: + scls = spider_classes[sname] + if sname == "oai": + process.crawl( + scls, + institution=inst, + target=per_spider_target, + from_date=args.from_date, + until_date=args.until_date, + ) + else: + process.crawl( + scls, + institution=inst, + target=per_spider_target, + boost_special=args.boost_special, + sc_only=args.sc_only, + ) + + print( + f"\nStarting crawl for {len(valid_institutions)} institution(s)...", flush=True + ) + print("=" * 60, flush=True) + sys.stdout.flush() + + # Start crawling + try: + process.start() + print("\n" + "=" * 60) + print("CRAWL COMPLETED") + print("=" * 60) + return 0 + + except KeyboardInterrupt: + print("\n\n[ERR] Crawl interrupted by user") + return 1 + + except Exception as e: + print(f"\n\n[ERR] Crawl failed: {e}") + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 3a916ddfd7d98cbe264f1fdb7bcb3a602c99d885..a801ca358802e59153ad7fcac9de693f09d98dc7 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,110 +1,110 @@ -#!/usr/bin/env bash -# ────────────────────────────────────────────────────────────────────────────── -# URAAS — One-shot deployment script for Ubuntu 22.04 / Debian 12 -# -# Run on a fresh VPS as root or a sudo user: -# curl -sSL https://raw.githubusercontent.com/YOUR/repo/main/scripts/deploy.sh | bash -# OR after cloning: -# bash scripts/deploy.sh -# -# What it does: -# 1. Install Docker + Docker Compose plugin -# 2. Generate password hashes interactively -# 3. Build and start all containers (postgres, redis, app, nginx) -# 4. Print the URL to reach the dashboard -# ────────────────────────────────────────────────────────────────────────────── -set -euo pipefail - -REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$REPO_DIR" - -echo "" -echo "═══════════════════════════════════════════════════" -echo " URAAS Deployment — $(date +%Y-%m-%d)" -echo "═══════════════════════════════════════════════════" - -# ── 1. Docker ───────────────────────────────────────────────────────────────── -if ! command -v docker &>/dev/null; then - echo "" - echo "▶ Installing Docker..." - curl -fsSL https://get.docker.com | sh - usermod -aG docker "$USER" || true - echo " Docker installed. You may need to log out and back in." -fi - -if ! docker compose version &>/dev/null 2>&1; then - echo "" - echo "▶ Installing Docker Compose plugin..." - apt-get install -y docker-compose-plugin 2>/dev/null || \ - pip install docker-compose 2>/dev/null || \ - echo " Install docker-compose manually from docs.docker.com/compose/install/" -fi - -# ── 2. .env.prod ────────────────────────────────────────────────────────────── -if [ ! -f .env.prod ]; then - echo "" - echo "▶ Creating .env.prod from example..." - cp .env.prod.example .env.prod - - # Get server IP - SERVER_IP=$(curl -s https://ifconfig.me || curl -s https://api.ipify.org || echo "YOUR_SERVER_IP") - sed -i "s/YOUR_SERVER_IP/$SERVER_IP/g" .env.prod - - # Generate secret key - SECRET_KEY=$(python3 -c "import secrets; print(secrets.token_hex(32))" 2>/dev/null || \ - openssl rand -hex 32) - sed -i "s/REPLACE_WITH_STRONG_RANDOM_KEY/$SECRET_KEY/" .env.prod - - # Generate password hashes - echo "" - echo "Enter the ADMIN password (for dashboard login):" - read -rs ADMIN_PASS - ADMIN_HASH=$(python3 -c "from werkzeug.security import generate_password_hash as g; print(g('$ADMIN_PASS'))" 2>/dev/null || \ - python3 -c "import hashlib, os; print('pbkdf2:sha256:' + hashlib.pbkdf2_hmac('sha256', b'$ADMIN_PASS', os.urandom(16), 150000).hex())") - sed -i "s|REPLACE_WITH_WERKZEUG_HASH|$ADMIN_HASH|g" .env.prod - - echo "" - echo " .env.prod created. Edit it to add SMTP_PASSWORD and API keys before running." - echo "" - echo " IMPORTANT: Set these in .env.prod before the demo:" - echo " SMTP_PASSWORD=" - echo " S2_API_KEY=" - echo " CORE_API_KEY=" -fi - -# ── 3. Required directories ─────────────────────────────────────────────────── -mkdir -p storage/pdfs data logs backups nginx/ssl - -# ── 4. Build + Start ────────────────────────────────────────────────────────── -echo "" -echo "▶ Building and starting containers (this takes ~3 min first time)..." -echo "" -# docker-compose.demo.yml = HTTP-only, works on bare IP (no SSL cert needed). -# Switch to docker-compose.prod.yml once you have a domain + SSL certificate. -docker compose --env-file .env.prod -f docker-compose.demo.yml up --build -d - -# ── 5. Wait for health ──────────────────────────────────────────────────────── -echo "" -echo "▶ Waiting for app to become healthy..." -for i in $(seq 1 20); do - STATUS=$(docker inspect --format='{{.State.Health.Status}}' uraas-app 2>/dev/null || echo "starting") - if [ "$STATUS" = "healthy" ]; then - echo " App is healthy!" - break - fi - echo " [$i/20] Status: $STATUS — waiting 5s..." - sleep 5 -done - -# ── 6. Done ─────────────────────────────────────────────────────────────────── -SERVER_IP=$(curl -s https://ifconfig.me 2>/dev/null || echo "YOUR_SERVER_IP") -echo "" -echo "═══════════════════════════════════════════════════" -echo " URAAS is running!" -echo "" -echo " Dashboard (direct): http://$SERVER_IP:8080" -echo " Dashboard (nginx): http://$SERVER_IP" -echo "" -echo " Logs: docker compose -f docker-compose.prod.yml logs -f app" -echo " Stop: docker compose -f docker-compose.prod.yml down" -echo "═══════════════════════════════════════════════════" +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────────────── +# URAAS — One-shot deployment script for Ubuntu 22.04 / Debian 12 +# +# Run on a fresh VPS as root or a sudo user: +# curl -sSL https://raw.githubusercontent.com/YOUR/repo/main/scripts/deploy.sh | bash +# OR after cloning: +# bash scripts/deploy.sh +# +# What it does: +# 1. Install Docker + Docker Compose plugin +# 2. Generate password hashes interactively +# 3. Build and start all containers (postgres, redis, app, nginx) +# 4. Print the URL to reach the dashboard +# ────────────────────────────────────────────────────────────────────────────── +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_DIR" + +echo "" +echo "═══════════════════════════════════════════════════" +echo " URAAS Deployment — $(date +%Y-%m-%d)" +echo "═══════════════════════════════════════════════════" + +# ── 1. Docker ───────────────────────────────────────────────────────────────── +if ! command -v docker &>/dev/null; then + echo "" + echo "▶ Installing Docker..." + curl -fsSL https://get.docker.com | sh + usermod -aG docker "$USER" || true + echo " Docker installed. You may need to log out and back in." +fi + +if ! docker compose version &>/dev/null 2>&1; then + echo "" + echo "▶ Installing Docker Compose plugin..." + apt-get install -y docker-compose-plugin 2>/dev/null || \ + pip install docker-compose 2>/dev/null || \ + echo " Install docker-compose manually from docs.docker.com/compose/install/" +fi + +# ── 2. .env.prod ────────────────────────────────────────────────────────────── +if [ ! -f .env.prod ]; then + echo "" + echo "▶ Creating .env.prod from example..." + cp .env.prod.example .env.prod + + # Get server IP + SERVER_IP=$(curl -s https://ifconfig.me || curl -s https://api.ipify.org || echo "YOUR_SERVER_IP") + sed -i "s/YOUR_SERVER_IP/$SERVER_IP/g" .env.prod + + # Generate secret key + SECRET_KEY=$(python3 -c "import secrets; print(secrets.token_hex(32))" 2>/dev/null || \ + openssl rand -hex 32) + sed -i "s/REPLACE_WITH_STRONG_RANDOM_KEY/$SECRET_KEY/" .env.prod + + # Generate password hashes + echo "" + echo "Enter the ADMIN password (for dashboard login):" + read -rs ADMIN_PASS + ADMIN_HASH=$(python3 -c "from werkzeug.security import generate_password_hash as g; print(g('$ADMIN_PASS'))" 2>/dev/null || \ + python3 -c "import hashlib, os; print('pbkdf2:sha256:' + hashlib.pbkdf2_hmac('sha256', b'$ADMIN_PASS', os.urandom(16), 150000).hex())") + sed -i "s|REPLACE_WITH_WERKZEUG_HASH|$ADMIN_HASH|g" .env.prod + + echo "" + echo " .env.prod created. Edit it to add SMTP_PASSWORD and API keys before running." + echo "" + echo " IMPORTANT: Set these in .env.prod before the demo:" + echo " SMTP_PASSWORD=" + echo " S2_API_KEY=" + echo " CORE_API_KEY=" +fi + +# ── 3. Required directories ─────────────────────────────────────────────────── +mkdir -p storage/pdfs data logs backups nginx/ssl + +# ── 4. Build + Start ────────────────────────────────────────────────────────── +echo "" +echo "▶ Building and starting containers (this takes ~3 min first time)..." +echo "" +# docker-compose.demo.yml = HTTP-only, works on bare IP (no SSL cert needed). +# Switch to docker-compose.prod.yml once you have a domain + SSL certificate. +docker compose --env-file .env.prod -f docker-compose.demo.yml up --build -d + +# ── 5. Wait for health ──────────────────────────────────────────────────────── +echo "" +echo "▶ Waiting for app to become healthy..." +for i in $(seq 1 20); do + STATUS=$(docker inspect --format='{{.State.Health.Status}}' uraas-app 2>/dev/null || echo "starting") + if [ "$STATUS" = "healthy" ]; then + echo " App is healthy!" + break + fi + echo " [$i/20] Status: $STATUS — waiting 5s..." + sleep 5 +done + +# ── 6. Done ─────────────────────────────────────────────────────────────────── +SERVER_IP=$(curl -s https://ifconfig.me 2>/dev/null || echo "YOUR_SERVER_IP") +echo "" +echo "═══════════════════════════════════════════════════" +echo " URAAS is running!" +echo "" +echo " Dashboard (direct): http://$SERVER_IP:8080" +echo " Dashboard (nginx): http://$SERVER_IP" +echo "" +echo " Logs: docker compose -f docker-compose.prod.yml logs -f app" +echo " Stop: docker compose -f docker-compose.prod.yml down" +echo "═══════════════════════════════════════════════════" diff --git a/scripts/fix_rors.py b/scripts/fix_rors.py index dabcc43de9ad7af023b98893a89a0c6034dd0a82..15c0a03e5fb57ea39547a63ec8761ce8e42b1176 100644 --- a/scripts/fix_rors.py +++ b/scripts/fix_rors.py @@ -1,26 +1,26 @@ -import glob -import json -import os -import urllib.parse -import urllib.request - -files = glob.glob("config/institutions/*.json") - -for fpath in files: - with open(fpath, "r", encoding="utf-8") as f: - data = json.load(f) - name = data.get("name") - if not name: - continue - - url = f"https://api.openalex.org/institutions?search={urllib.parse.quote(name)}&per-page=1" - try: - res = json.loads(urllib.request.urlopen(url).read().decode())["results"][0] - correct_ror = res.get("ror") - if correct_ror and correct_ror != data.get("ror"): - print(f"Updating {name}: {data.get('ror')} -> {correct_ror}") - data["ror"] = correct_ror - with open(fpath, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2) - except Exception as e: - print(f"Error for {name}: {e}") +import glob +import json +import os +import urllib.parse +import urllib.request + +files = glob.glob("config/institutions/*.json") + +for fpath in files: + with open(fpath, "r", encoding="utf-8") as f: + data = json.load(f) + name = data.get("name") + if not name: + continue + + url = f"https://api.openalex.org/institutions?search={urllib.parse.quote(name)}&per-page=1" + try: + res = json.loads(urllib.request.urlopen(url).read().decode())["results"][0] + correct_ror = res.get("ror") + if correct_ror and correct_ror != data.get("ror"): + print(f"Updating {name}: {data.get('ror')} -> {correct_ror}") + data["ror"] = correct_ror + with open(fpath, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + except Exception as e: + print(f"Error for {name}: {e}") diff --git a/scripts/generate_registry.py b/scripts/generate_registry.py index 8604c091b69ee04617afd171fc6e901b794d5434..8ccc746a5c657a473effebad436ac0bfbd7bf484 100644 --- a/scripts/generate_registry.py +++ b/scripts/generate_registry.py @@ -1,444 +1,444 @@ -import json -import os - -# Sub-regions and countries -subregions = { - "North Africa": ["Egypt", "Morocco", "Algeria", "Tunisia", "Libya", "Sudan"], - "West Africa": [ - "Nigeria", - "Ghana", - "Senegal", - "Cote d'Ivoire", - "Benin", - "Burkina Faso", - "Cape Verde", - "Gambia", - "Guinea", - "Guinea-Bissau", - "Liberia", - "Mali", - "Mauritania", - "Niger", - "Sierra Leone", - "Togo", - ], - "East Africa": [ - "Kenya", - "Uganda", - "Tanzania", - "Ethiopia", - "Rwanda", - "Burundi", - "Djibouti", - "Eritrea", - "Somalia", - "South Sudan", - "Madagascar", - "Mauritius", - "Seychelles", - "Comoros", - ], - "Southern Africa": [ - "South Africa", - "Zimbabwe", - "Zambia", - "Namibia", - "Botswana", - "Lesotho", - "Eswatini", - "Malawi", - "Mozambique", - ], - "Central Africa": [ - "Cameroon", - "DR Congo", - "Angola", - "Gabon", - "Republic of the Congo", - "Central African Republic", - "Chad", - "Equatorial Guinea", - "Sao Tome and Principe", - ], -} - -# Major cities for generation if needed -capitals = { - "Egypt": "Cairo", - "Morocco": "Rabat", - "Algeria": "Algiers", - "Tunisia": "Tunis", - "Libya": "Tripoli", - "Sudan": "Khartoum", - "Nigeria": "Abuja", - "Ghana": "Accra", - "Senegal": "Dakar", - "Cote d'Ivoire": "Yamoussoukro", - "Benin": "Porto-Novo", - "Burkina Faso": "Ouagadougou", - "Cape Verde": "Praia", - "Gambia": "Banjul", - "Guinea": "Conakry", - "Guinea-Bissau": "Bissau", - "Liberia": "Monrovia", - "Mali": "Bamako", - "Mauritania": "Nouakchott", - "Niger": "Niamey", - "Sierra Leone": "Freetown", - "Togo": "Lome", - "Kenya": "Nairobi", - "Uganda": "Kampala", - "Tanzania": "Dodoma", - "Ethiopia": "Addis Ababa", - "Rwanda": "Kigali", - "Burundi": "Gitega", - "Djibouti": "Djibouti", - "Eritrea": "Asmara", - "Somalia": "Mogadishu", - "South Sudan": "Juba", - "Madagascar": "Antananarivo", - "Mauritius": "Port Louis", - "Seychelles": "Victoria", - "Comoros": "Moroni", - "South Africa": "Pretoria", - "Zimbabwe": "Harare", - "Zambia": "Lusaka", - "Namibia": "Windhoek", - "Botswana": "Gaborone", - "Lesotho": "Maseru", - "Eswatini": "Mbabane", - "Malawi": "Lilongwe", - "Mozambique": "Maputo", - "Cameroon": "Yaounde", - "DR Congo": "Kinshasa", - "Angola": "Luanda", - "Gabon": "Libreville", - "Republic of the Congo": "Brazzaville", - "Central African Republic": "Bangui", - "Chad": "N'Djamena", - "Equatorial Guinea": "Malabo", - "Sao Tome and Principe": "Sao Tome", -} - -# Hand-curated top universities to include (demo universities) -curated_universities = { - # North Africa - "Egypt": [ - {"name": "Cairo University", "ror": "https://ror.org/03c4mpy73"}, - {"name": "Ain Shams University", "ror": "https://ror.org/034x7p097"}, - {"name": "Alexandria University", "ror": "https://ror.org/02078r490"}, - {"name": "Mansoura University", "ror": "https://ror.org/032p18087"}, - {"name": "Assiut University", "ror": "https://ror.org/047fpp722"}, - ], - "Morocco": [ - {"name": "Université Mohammed V de Rabat", "ror": "https://ror.org/03vpy3v17"}, - {"name": "Université Cadi Ayyad", "ror": "https://ror.org/0154pcf71"}, - { - "name": "Université Hassan II de Casablanca", - "ror": "https://ror.org/013y27r38", - }, - ], - "Tunisia": [ - {"name": "Université de Tunis El Manar", "ror": "https://ror.org/050j3a172"}, - {"name": "Université de Sfax", "ror": "https://ror.org/02157p641"}, - {"name": "Université de Carthage", "ror": "https://ror.org/011yvpy71"}, - ], - # Central Africa - "Cameroon": [ - {"name": "Université de Yaoundé I", "ror": "https://ror.org/04h7g6177"}, - {"name": "Université de Dschang", "ror": "https://ror.org/012y7p041"}, - {"name": "Université de Douala", "ror": "https://ror.org/041y27r28"}, - ], - "DR Congo": [ - {"name": "Université de Kinshasa", "ror": "https://ror.org/05vzwad88"}, - {"name": "Université de Lubumbashi", "ror": "https://ror.org/02rry3m21"}, - ], - "Angola": [ - {"name": "Université Agostinho Neto", "ror": "https://ror.org/00z2bpt98"} - ], - "Gabon": [ - { - "name": "Université des Sciences et Techniques de Masuku", - "ror": "https://ror.org/059gqse72", - }, - {"name": "Université Omar Bongo", "ror": "https://ror.org/041yp1812"}, - ], - "Republic of the Congo": [ - {"name": "Université Marien Ngouabi", "ror": "https://ror.org/02y1sra05"} - ], - # West Africa - "Nigeria": [ - {"name": "University of Lagos", "ror": "https://ror.org/05rk03822"}, - {"name": "University of Ibadan", "ror": "https://ror.org/01es5me90"}, - {"name": "Covenant University", "ror": "https://ror.org/02n05rk12"}, - {"name": "Obafemi Awolowo University", "ror": "https://ror.org/013pcr241"}, - {"name": "University of Nigeria Nsukka", "ror": "https://ror.org/02kpy5732"}, - ], - "Ghana": [ - {"name": "University of Ghana", "ror": "https://ror.org/00zpy3v12"}, - { - "name": "Kwame Nkrumah University of Science and Technology", - "ror": "https://ror.org/00x4mpy73", - }, - {"name": "University of Cape Coast", "ror": "https://ror.org/01es3v123"}, - ], - # Southern Africa - "South Africa": [ - {"name": "University of Cape Town", "ror": "https://ror.org/017620319"}, - {"name": "Stellenbosch University", "ror": "https://ror.org/05777p686"}, - {"name": "University of the Witwatersrand", "ror": "https://ror.org/039482g93"}, - {"name": "University of Pretoria", "ror": "https://ror.org/047fpp722"}, - {"name": "University of KwaZulu-Natal", "ror": "https://ror.org/01267r312"}, - ], - "Zimbabwe": [ - {"name": "University of Zimbabwe", "ror": "https://ror.org/03w489125"}, - { - "name": "National University of Science and Technology", - "ror": "https://ror.org/01y6mpy73", - }, - ], - # East Africa - "Kenya": [ - {"name": "University of Nairobi", "ror": "https://ror.org/01078r490"}, - {"name": "Kenyatta University", "ror": "https://ror.org/01py3v171"}, - { - "name": "Jomo Kenyatta University of Agriculture and Technology", - "ror": "https://ror.org/03pyvpy71", - }, - ], - "Uganda": [ - {"name": "Makerere University", "ror": "https://ror.org/05vzwad88"}, - { - "name": "Mbarara University of Science and Technology", - "ror": "https://ror.org/0155pcf71", - }, - ], - "Tanzania": [ - {"name": "University of Dar es Salaam", "ror": "https://ror.org/0199e1957"}, - { - "name": "Sokoine University of Agriculture", - "ror": "https://ror.org/011y27r38", - }, - ], - "Ethiopia": [ - {"name": "Addis Ababa University", "ror": "https://ror.org/01py3v171"} - ], - "Rwanda": [{"name": "University of Rwanda", "ror": "https://ror.org/02yr01r27"}], -} - -# Generate 15-20 universities for each country -registry_data = {} -for subregion, countries in subregions.items(): - registry_data[subregion] = {} - for country in countries: - cap = capitals.get(country, "City") - # Start with curated list or empty - unis = curated_universities.get(country, []).copy() - - # Add generated universities to reach 15 - existing_names = {u["name"] for u in unis} - templates = [ - f"University of {cap}", - f"National University of {country}", - f"{country} University of Science and Technology", - f"{cap} Institute of Technology", - f"State University of {cap}", - f"{country} International University", - f"Pan-African University, {cap} Campus", - f"Catholic University of {country}", - f"Technical University of {cap}", - ( - f"Ahmadu Bello University of {cap}" - if country == "Nigeria" - else f"Federal University of {cap}" - ), - f"Metropolitan University of {cap}", - f"Central University of {country}", - f"Presbyterian University of {country}", - f"Adventist University of {country}", - f"Islamic University of {country}", - f"Methodist University of {country}", - f"Covenant University of {cap}", - f"{cap} College of Medicine and Health Sciences", - f"Regional Institute of Information Technology, {cap}", - f"Greenfield University, {cap}", - ] - - idx = 0 - while len(unis) < 18: - name = templates[idx % len(templates)] - # ensure uniqueness - if name not in existing_names: - unis.append({"name": name, "ror": ""}) - existing_names.add(name) - idx += 1 - - registry_data[subregion][country] = unis - -# Write university_registry.json -os.makedirs("data", exist_ok=True) -with open("data/university_registry.json", "w", encoding="utf-8") as f: - json.dump(registry_data, f, indent=2, ensure_ascii=False) -print( - "Generated data/university_registry.json with 52 countries and 18 universities each." -) - -# Configurations for the 15 new universities to make a total of 25 demo universities -new_universities = [ - # North (5) - { - "file": "cairo.json", - "ror": "https://ror.org/03c4mpy73", - "name": "Cairo University", - "short_name": "Cairo Univ", - "country": "Egypt", - "sub_region": "North Africa", - }, - { - "file": "ainshams.json", - "ror": "https://ror.org/034x7p097", - "name": "Ain Shams University", - "short_name": "Ain Shams", - "country": "Egypt", - "sub_region": "North Africa", - }, - { - "file": "alexandria.json", - "ror": "https://ror.org/02078r490", - "name": "Alexandria University", - "short_name": "Alexandria", - "country": "Egypt", - "sub_region": "North Africa", - }, - { - "file": "tunis.json", - "ror": "https://ror.org/050j3a172", - "name": "Université de Tunis El Manar", - "short_name": "Tunis El Manar", - "country": "Tunisia", - "sub_region": "North Africa", - }, - { - "file": "mohammedv.json", - "ror": "https://ror.org/03vpy3v17", - "name": "Université Mohammed V de Rabat", - "short_name": "Mohammed V", - "country": "Morocco", - "sub_region": "North Africa", - }, - # Central (5) - { - "file": "yaoundei.json", - "ror": "https://ror.org/04h7g6177", - "name": "Université de Yaoundé I", - "short_name": "Yaoundé I", - "country": "Cameroon", - "sub_region": "Central Africa", - }, - { - "file": "kinshasa.json", - "ror": "https://ror.org/05vzwad88", - "name": "Université de Kinshasa", - "short_name": "UNIKIN", - "country": "DR Congo", - "sub_region": "Central Africa", - }, - { - "file": "agostinhoneto.json", - "ror": "https://ror.org/00z2bpt98", - "name": "Université Agostinho Neto", - "short_name": "Agostinho Neto", - "country": "Angola", - "sub_region": "Central Africa", - }, - { - "file": "marienngouabi.json", - "ror": "https://ror.org/02y1sra05", - "name": "Université Marien Ngouabi", - "short_name": "Marien Ngouabi", - "country": "Republic of the Congo", - "sub_region": "Central Africa", - }, - { - "file": "masuku.json", - "ror": "https://ror.org/059gqse72", - "name": "Université des Sciences et Techniques de Masuku", - "short_name": "USTM Masuku", - "country": "Gabon", - "sub_region": "Central Africa", - }, - # Southern (+3 new ones) - { - "file": "wits.json", - "ror": "https://ror.org/039482g93", - "name": "University of the Witwatersrand", - "short_name": "Wits", - "country": "South Africa", - "sub_region": "Southern Africa", - }, - { - "file": "pretoria.json", - "ror": "https://ror.org/047fpp722", - "name": "University of Pretoria", - "short_name": "UP", - "country": "South Africa", - "sub_region": "Southern Africa", - }, - { - "file": "zimbabwe.json", - "ror": "https://ror.org/03w489125", - "name": "University of Zimbabwe", - "short_name": "UZ", - "country": "Zimbabwe", - "sub_region": "Southern Africa", - }, - # East (+2 new ones) - { - "file": "daressalaam.json", - "ror": "https://ror.org/0199e1957", - "name": "University of Dar es Salaam", - "short_name": "UDSM", - "country": "Tanzania", - "sub_region": "East Africa", - }, - { - "file": "rwanda.json", - "ror": "https://ror.org/02yr01r27", - "name": "University of Rwanda", - "short_name": "UR", - "country": "Rwanda", - "sub_region": "East Africa", - }, -] - -# Write institutional configs -os.makedirs("config/institutions", exist_ok=True) -for u in new_universities: - cfg = { - "ror": u["ror"], - "name": u["name"], - "short_name": u["short_name"], - "country": u["country"], - "sub_region": u["sub_region"], # explicitly add sub-region to config files - "staff_file": f"data/{u['short_name'].lower().replace(' ', '_')}_staff.json", - "affiliation_patterns": [u["name"], u["short_name"], f"{u['name']} Department"], - "faculties": [ - "Science", - "Humanities", - "Engineering", - "Medicine", - "Social Sciences", - "Arts", - "Law", - ], - "crawler_settings": { - "rate_limit": 2.0, - "concurrent_requests": 8, - "retry_times": 3, - "download_delay": 2.0, - }, - } - with open(f"config/institutions/{u['file']}", "w", encoding="utf-8") as f: - json.dump(cfg, f, indent=2, ensure_ascii=False) - -print("Generated 15 new institutional config files.") +import json +import os + +# Sub-regions and countries +subregions = { + "North Africa": ["Egypt", "Morocco", "Algeria", "Tunisia", "Libya", "Sudan"], + "West Africa": [ + "Nigeria", + "Ghana", + "Senegal", + "Cote d'Ivoire", + "Benin", + "Burkina Faso", + "Cape Verde", + "Gambia", + "Guinea", + "Guinea-Bissau", + "Liberia", + "Mali", + "Mauritania", + "Niger", + "Sierra Leone", + "Togo", + ], + "East Africa": [ + "Kenya", + "Uganda", + "Tanzania", + "Ethiopia", + "Rwanda", + "Burundi", + "Djibouti", + "Eritrea", + "Somalia", + "South Sudan", + "Madagascar", + "Mauritius", + "Seychelles", + "Comoros", + ], + "Southern Africa": [ + "South Africa", + "Zimbabwe", + "Zambia", + "Namibia", + "Botswana", + "Lesotho", + "Eswatini", + "Malawi", + "Mozambique", + ], + "Central Africa": [ + "Cameroon", + "DR Congo", + "Angola", + "Gabon", + "Republic of the Congo", + "Central African Republic", + "Chad", + "Equatorial Guinea", + "Sao Tome and Principe", + ], +} + +# Major cities for generation if needed +capitals = { + "Egypt": "Cairo", + "Morocco": "Rabat", + "Algeria": "Algiers", + "Tunisia": "Tunis", + "Libya": "Tripoli", + "Sudan": "Khartoum", + "Nigeria": "Abuja", + "Ghana": "Accra", + "Senegal": "Dakar", + "Cote d'Ivoire": "Yamoussoukro", + "Benin": "Porto-Novo", + "Burkina Faso": "Ouagadougou", + "Cape Verde": "Praia", + "Gambia": "Banjul", + "Guinea": "Conakry", + "Guinea-Bissau": "Bissau", + "Liberia": "Monrovia", + "Mali": "Bamako", + "Mauritania": "Nouakchott", + "Niger": "Niamey", + "Sierra Leone": "Freetown", + "Togo": "Lome", + "Kenya": "Nairobi", + "Uganda": "Kampala", + "Tanzania": "Dodoma", + "Ethiopia": "Addis Ababa", + "Rwanda": "Kigali", + "Burundi": "Gitega", + "Djibouti": "Djibouti", + "Eritrea": "Asmara", + "Somalia": "Mogadishu", + "South Sudan": "Juba", + "Madagascar": "Antananarivo", + "Mauritius": "Port Louis", + "Seychelles": "Victoria", + "Comoros": "Moroni", + "South Africa": "Pretoria", + "Zimbabwe": "Harare", + "Zambia": "Lusaka", + "Namibia": "Windhoek", + "Botswana": "Gaborone", + "Lesotho": "Maseru", + "Eswatini": "Mbabane", + "Malawi": "Lilongwe", + "Mozambique": "Maputo", + "Cameroon": "Yaounde", + "DR Congo": "Kinshasa", + "Angola": "Luanda", + "Gabon": "Libreville", + "Republic of the Congo": "Brazzaville", + "Central African Republic": "Bangui", + "Chad": "N'Djamena", + "Equatorial Guinea": "Malabo", + "Sao Tome and Principe": "Sao Tome", +} + +# Hand-curated top universities to include (demo universities) +curated_universities = { + # North Africa + "Egypt": [ + {"name": "Cairo University", "ror": "https://ror.org/03c4mpy73"}, + {"name": "Ain Shams University", "ror": "https://ror.org/034x7p097"}, + {"name": "Alexandria University", "ror": "https://ror.org/02078r490"}, + {"name": "Mansoura University", "ror": "https://ror.org/032p18087"}, + {"name": "Assiut University", "ror": "https://ror.org/047fpp722"}, + ], + "Morocco": [ + {"name": "Université Mohammed V de Rabat", "ror": "https://ror.org/03vpy3v17"}, + {"name": "Université Cadi Ayyad", "ror": "https://ror.org/0154pcf71"}, + { + "name": "Université Hassan II de Casablanca", + "ror": "https://ror.org/013y27r38", + }, + ], + "Tunisia": [ + {"name": "Université de Tunis El Manar", "ror": "https://ror.org/050j3a172"}, + {"name": "Université de Sfax", "ror": "https://ror.org/02157p641"}, + {"name": "Université de Carthage", "ror": "https://ror.org/011yvpy71"}, + ], + # Central Africa + "Cameroon": [ + {"name": "Université de Yaoundé I", "ror": "https://ror.org/04h7g6177"}, + {"name": "Université de Dschang", "ror": "https://ror.org/012y7p041"}, + {"name": "Université de Douala", "ror": "https://ror.org/041y27r28"}, + ], + "DR Congo": [ + {"name": "Université de Kinshasa", "ror": "https://ror.org/05vzwad88"}, + {"name": "Université de Lubumbashi", "ror": "https://ror.org/02rry3m21"}, + ], + "Angola": [ + {"name": "Université Agostinho Neto", "ror": "https://ror.org/00z2bpt98"} + ], + "Gabon": [ + { + "name": "Université des Sciences et Techniques de Masuku", + "ror": "https://ror.org/059gqse72", + }, + {"name": "Université Omar Bongo", "ror": "https://ror.org/041yp1812"}, + ], + "Republic of the Congo": [ + {"name": "Université Marien Ngouabi", "ror": "https://ror.org/02y1sra05"} + ], + # West Africa + "Nigeria": [ + {"name": "University of Lagos", "ror": "https://ror.org/05rk03822"}, + {"name": "University of Ibadan", "ror": "https://ror.org/01es5me90"}, + {"name": "Covenant University", "ror": "https://ror.org/02n05rk12"}, + {"name": "Obafemi Awolowo University", "ror": "https://ror.org/013pcr241"}, + {"name": "University of Nigeria Nsukka", "ror": "https://ror.org/02kpy5732"}, + ], + "Ghana": [ + {"name": "University of Ghana", "ror": "https://ror.org/00zpy3v12"}, + { + "name": "Kwame Nkrumah University of Science and Technology", + "ror": "https://ror.org/00x4mpy73", + }, + {"name": "University of Cape Coast", "ror": "https://ror.org/01es3v123"}, + ], + # Southern Africa + "South Africa": [ + {"name": "University of Cape Town", "ror": "https://ror.org/017620319"}, + {"name": "Stellenbosch University", "ror": "https://ror.org/05777p686"}, + {"name": "University of the Witwatersrand", "ror": "https://ror.org/039482g93"}, + {"name": "University of Pretoria", "ror": "https://ror.org/047fpp722"}, + {"name": "University of KwaZulu-Natal", "ror": "https://ror.org/01267r312"}, + ], + "Zimbabwe": [ + {"name": "University of Zimbabwe", "ror": "https://ror.org/03w489125"}, + { + "name": "National University of Science and Technology", + "ror": "https://ror.org/01y6mpy73", + }, + ], + # East Africa + "Kenya": [ + {"name": "University of Nairobi", "ror": "https://ror.org/01078r490"}, + {"name": "Kenyatta University", "ror": "https://ror.org/01py3v171"}, + { + "name": "Jomo Kenyatta University of Agriculture and Technology", + "ror": "https://ror.org/03pyvpy71", + }, + ], + "Uganda": [ + {"name": "Makerere University", "ror": "https://ror.org/05vzwad88"}, + { + "name": "Mbarara University of Science and Technology", + "ror": "https://ror.org/0155pcf71", + }, + ], + "Tanzania": [ + {"name": "University of Dar es Salaam", "ror": "https://ror.org/0199e1957"}, + { + "name": "Sokoine University of Agriculture", + "ror": "https://ror.org/011y27r38", + }, + ], + "Ethiopia": [ + {"name": "Addis Ababa University", "ror": "https://ror.org/01py3v171"} + ], + "Rwanda": [{"name": "University of Rwanda", "ror": "https://ror.org/02yr01r27"}], +} + +# Generate 15-20 universities for each country +registry_data = {} +for subregion, countries in subregions.items(): + registry_data[subregion] = {} + for country in countries: + cap = capitals.get(country, "City") + # Start with curated list or empty + unis = curated_universities.get(country, []).copy() + + # Add generated universities to reach 15 + existing_names = {u["name"] for u in unis} + templates = [ + f"University of {cap}", + f"National University of {country}", + f"{country} University of Science and Technology", + f"{cap} Institute of Technology", + f"State University of {cap}", + f"{country} International University", + f"Pan-African University, {cap} Campus", + f"Catholic University of {country}", + f"Technical University of {cap}", + ( + f"Ahmadu Bello University of {cap}" + if country == "Nigeria" + else f"Federal University of {cap}" + ), + f"Metropolitan University of {cap}", + f"Central University of {country}", + f"Presbyterian University of {country}", + f"Adventist University of {country}", + f"Islamic University of {country}", + f"Methodist University of {country}", + f"Covenant University of {cap}", + f"{cap} College of Medicine and Health Sciences", + f"Regional Institute of Information Technology, {cap}", + f"Greenfield University, {cap}", + ] + + idx = 0 + while len(unis) < 18: + name = templates[idx % len(templates)] + # ensure uniqueness + if name not in existing_names: + unis.append({"name": name, "ror": ""}) + existing_names.add(name) + idx += 1 + + registry_data[subregion][country] = unis + +# Write university_registry.json +os.makedirs("data", exist_ok=True) +with open("data/university_registry.json", "w", encoding="utf-8") as f: + json.dump(registry_data, f, indent=2, ensure_ascii=False) +print( + "Generated data/university_registry.json with 52 countries and 18 universities each." +) + +# Configurations for the 15 new universities to make a total of 25 demo universities +new_universities = [ + # North (5) + { + "file": "cairo.json", + "ror": "https://ror.org/03c4mpy73", + "name": "Cairo University", + "short_name": "Cairo Univ", + "country": "Egypt", + "sub_region": "North Africa", + }, + { + "file": "ainshams.json", + "ror": "https://ror.org/034x7p097", + "name": "Ain Shams University", + "short_name": "Ain Shams", + "country": "Egypt", + "sub_region": "North Africa", + }, + { + "file": "alexandria.json", + "ror": "https://ror.org/02078r490", + "name": "Alexandria University", + "short_name": "Alexandria", + "country": "Egypt", + "sub_region": "North Africa", + }, + { + "file": "tunis.json", + "ror": "https://ror.org/050j3a172", + "name": "Université de Tunis El Manar", + "short_name": "Tunis El Manar", + "country": "Tunisia", + "sub_region": "North Africa", + }, + { + "file": "mohammedv.json", + "ror": "https://ror.org/03vpy3v17", + "name": "Université Mohammed V de Rabat", + "short_name": "Mohammed V", + "country": "Morocco", + "sub_region": "North Africa", + }, + # Central (5) + { + "file": "yaoundei.json", + "ror": "https://ror.org/04h7g6177", + "name": "Université de Yaoundé I", + "short_name": "Yaoundé I", + "country": "Cameroon", + "sub_region": "Central Africa", + }, + { + "file": "kinshasa.json", + "ror": "https://ror.org/05vzwad88", + "name": "Université de Kinshasa", + "short_name": "UNIKIN", + "country": "DR Congo", + "sub_region": "Central Africa", + }, + { + "file": "agostinhoneto.json", + "ror": "https://ror.org/00z2bpt98", + "name": "Université Agostinho Neto", + "short_name": "Agostinho Neto", + "country": "Angola", + "sub_region": "Central Africa", + }, + { + "file": "marienngouabi.json", + "ror": "https://ror.org/02y1sra05", + "name": "Université Marien Ngouabi", + "short_name": "Marien Ngouabi", + "country": "Republic of the Congo", + "sub_region": "Central Africa", + }, + { + "file": "masuku.json", + "ror": "https://ror.org/059gqse72", + "name": "Université des Sciences et Techniques de Masuku", + "short_name": "USTM Masuku", + "country": "Gabon", + "sub_region": "Central Africa", + }, + # Southern (+3 new ones) + { + "file": "wits.json", + "ror": "https://ror.org/039482g93", + "name": "University of the Witwatersrand", + "short_name": "Wits", + "country": "South Africa", + "sub_region": "Southern Africa", + }, + { + "file": "pretoria.json", + "ror": "https://ror.org/047fpp722", + "name": "University of Pretoria", + "short_name": "UP", + "country": "South Africa", + "sub_region": "Southern Africa", + }, + { + "file": "zimbabwe.json", + "ror": "https://ror.org/03w489125", + "name": "University of Zimbabwe", + "short_name": "UZ", + "country": "Zimbabwe", + "sub_region": "Southern Africa", + }, + # East (+2 new ones) + { + "file": "daressalaam.json", + "ror": "https://ror.org/0199e1957", + "name": "University of Dar es Salaam", + "short_name": "UDSM", + "country": "Tanzania", + "sub_region": "East Africa", + }, + { + "file": "rwanda.json", + "ror": "https://ror.org/02yr01r27", + "name": "University of Rwanda", + "short_name": "UR", + "country": "Rwanda", + "sub_region": "East Africa", + }, +] + +# Write institutional configs +os.makedirs("config/institutions", exist_ok=True) +for u in new_universities: + cfg = { + "ror": u["ror"], + "name": u["name"], + "short_name": u["short_name"], + "country": u["country"], + "sub_region": u["sub_region"], # explicitly add sub-region to config files + "staff_file": f"data/{u['short_name'].lower().replace(' ', '_')}_staff.json", + "affiliation_patterns": [u["name"], u["short_name"], f"{u['name']} Department"], + "faculties": [ + "Science", + "Humanities", + "Engineering", + "Medicine", + "Social Sciences", + "Arts", + "Law", + ], + "crawler_settings": { + "rate_limit": 2.0, + "concurrent_requests": 8, + "retry_times": 3, + "download_delay": 2.0, + }, + } + with open(f"config/institutions/{u['file']}", "w", encoding="utf-8") as f: + json.dump(cfg, f, indent=2, ensure_ascii=False) + +print("Generated 15 new institutional config files.") diff --git a/scripts/harvest_staff_openalex.py b/scripts/harvest_staff_openalex.py index bae1875375272511922c422501e3123f54edec79..19ef9b7a3fddf8afd29e99486eab898f72adbb7a 100644 --- a/scripts/harvest_staff_openalex.py +++ b/scripts/harvest_staff_openalex.py @@ -1,315 +1,315 @@ -""" -Staff Harvester — fetches real staff names, ORCIDs, departments from OpenAlex -for every configured institution. Saves enriched JSON to data/{inst}_staff.json. - -Usage: - python scripts/harvest_staff_openalex.py # all institutions - python scripts/harvest_staff_openalex.py --institution unilag - python scripts/harvest_staff_openalex.py --dry-run # just print counts -""" - -import argparse -import json -import logging -import os -import sys -import time -import urllib.error -import urllib.parse -import urllib.request - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from uraas.config.institutions import get_registry, reset_registry - -logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) - -OPENALEX_BASE = "https://api.openalex.org" -MAILTO = "uraas-bot@research.edu.ng" -MAX_AUTHORS = 500 # cap per institution to avoid very long runs -DELAY = 0.5 # seconds between requests (polite) - - -def _get(url: str, retries: int = 3) -> dict: - """Simple urllib GET with retries.""" - for attempt in range(retries): - try: - req = urllib.request.Request( - url, headers={"User-Agent": f"URAAS/1.0 (mailto:{MAILTO})"} - ) - with urllib.request.urlopen(req, timeout=20) as resp: - return json.loads(resp.read().decode()) - except urllib.error.HTTPError as e: - if e.code == 429: - wait = 5 * (attempt + 1) - log.warning(f"Rate limited, waiting {wait}s …") - time.sleep(wait) - else: - log.error(f"HTTP {e.code} for {url}") - break - except Exception as e: - log.error(f"Request error ({attempt+1}/{retries}): {e}") - time.sleep(2) - return {} - - -def harvest_institution(inst_config, dry_run: bool = False) -> list: - """ - Harvest staff by looking at recent works from the institution on OpenAlex. - Extracts unique authors from the authorships array. - Returns list of rich staff dicts: {name, orcid, department, faculty, openalex_id, paper_count} - """ - ror_url = inst_config.ror - inst_name = inst_config.name - log.info(f"Harvesting staff for {inst_name} (ROR: {ror_url}) …") - - unique_staff = {} - cursor = "*" - page = 0 - max_pages = 50 # Limit to 50 pages (10k works max) to avoid running forever - - while len(unique_staff) < MAX_AUTHORS and page < max_pages: - # We query the works endpoint using the exact ROR url - url = ( - f"{OPENALEX_BASE}/works" - f"?filter=institutions.ror:{urllib.parse.quote(ror_url)}" - f"&select=authorships" - f"&per-page=200" - f"&cursor={urllib.parse.quote(cursor)}" - f"&mailto={MAILTO}" - ) - data = _get(url) - if not data: - break - - results = data.get("results", []) - if not results: - break - - for work in results: - for authorship in work.get("authorships", []): - # Ensure the author is affiliated with our target institution for this work - is_affiliated = False - for inst in authorship.get("institutions", []): - if inst.get("ror") == ror_url: - is_affiliated = True - break - - if not is_affiliated: - continue - - author = authorship.get("author", {}) - aid = author.get("id") - if not aid or aid in unique_staff: - if aid in unique_staff: - unique_staff[aid]["paper_count"] += 1 - continue - - name = author.get("display_name", "").strip() - if not name: - continue - - orcid_url = author.get("orcid", "") - orcid = ( - orcid_url.replace("https://orcid.org/", "") if orcid_url else None - ) - - # We can't get concepts easily from works authorships without extra queries, - # so we will leave faculty and department empty for now. - - unique_staff[aid] = { - "name": name, - "orcid": orcid, - "department": None, - "faculty": None, - "openalex_id": aid.replace("https://openalex.org/", ""), - "paper_count": 1, - } - - if len(unique_staff) >= MAX_AUTHORS: - break - - if len(unique_staff) >= MAX_AUTHORS: - break - - log.info( - f" Page {page+1}: Processed {len(results)} works | Unique staff so far: {len(unique_staff)}" - ) - page += 1 - time.sleep(DELAY) - - meta = data.get("meta", {}) - cursor = meta.get("next_cursor") - if not cursor: - break - - staff_list = list(unique_staff.values()) - log.info(f" Harvested {len(staff_list)} staff for {inst_name}") - return staff_list - - -def _map_concept_to_faculty(concept: str, faculties: list) -> str: - """Rough concept→faculty mapping via keyword overlap.""" - concept_lower = concept.lower() - faculty_map = { - "medicine": ["health", "medicine", "clinical", "nursing", "pharmacy", "dental"], - "engineering": [ - "engineering", - "technology", - "mechanical", - "electrical", - "civil", - "chemical", - ], - "science": [ - "biology", - "chemistry", - "physics", - "mathematics", - "statistics", - "computer", - ], - "arts": [ - "literature", - "linguistics", - "language", - "history", - "philosophy", - "arts", - ], - "social": [ - "sociology", - "economics", - "political", - "psychology", - "anthropology", - "social", - ], - "law": ["law", "legal", "jurisprudence", "criminology"], - "education": ["education", "pedagogy", "teaching", "curriculum"], - "agriculture": ["agriculture", "botany", "zoology", "ecology", "forestry"], - "management": ["business", "management", "accounting", "finance", "marketing"], - "environmental": [ - "environment", - "urban", - "planning", - "geography", - "architecture", - ], - } - for fac_key, keywords in faculty_map.items(): - if any(kw in concept_lower for kw in keywords): - # Try to match to actual faculty names - for f in faculties: - if fac_key in f.lower() or any(kw in f.lower() for kw in keywords): - return f - return None - - -def get_orcid_details(orcid: str) -> dict: - """Fetch name and affiliation details from ORCID public API.""" - url = f"https://pub.orcid.org/v3.0/{orcid}/person" - try: - req = urllib.request.Request( - url, - headers={ - "Accept": "application/json", - "User-Agent": f"URAAS/1.0 (mailto:{MAILTO})", - }, - ) - with urllib.request.urlopen(req, timeout=15) as resp: - data = json.loads(resp.read().decode()) - affiliations = data.get("activities-summary", {}) - return {"orcid": orcid} - except Exception: - return {} - - -def save_staff(inst_config, staff: list, dry_run: bool = False): - """Save staff list to data/{short_name_lower}_staff.json""" - short = inst_config.short_name.lower() - # Resolve base directory - base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - out_path = os.path.join(base_dir, "data", f"{short}_staff.json") - - if dry_run: - log.info(f"[DRY-RUN] Would save {len(staff)} staff records to {out_path}") - return - - os.makedirs(os.path.dirname(out_path), exist_ok=True) - with open(out_path, "w", encoding="utf-8") as f: - json.dump(staff, f, indent=2, ensure_ascii=False) - log.info(f"Saved {len(staff)} staff records → {out_path}") - - -def main(): - parser = argparse.ArgumentParser( - description="Harvest staff from OpenAlex for URAAS institutions" - ) - parser.add_argument( - "--institution", - type=str, - default=None, - help="Single institution short name (default: all)", - ) - parser.add_argument( - "--dry-run", action="store_true", help="Print counts without saving" - ) - args = parser.parse_args() - - reset_registry() - registry = get_registry() - all_insts = registry.list_all() - - if args.institution: - inst = registry.get(args.institution) - if not inst: - print(f"ERROR: Institution '{args.institution}' not found") - sys.exit(1) - target_insts = [inst] - else: - target_insts = all_insts - - print(f"\n{'='*60}") - print(f"URAAS Staff Harvester — OpenAlex") - print(f"Institutions: {len(target_insts)}") - print(f"{'='*60}\n") - - summary = [] - for inst in target_insts: - try: - staff = harvest_institution(inst, dry_run=args.dry_run) - orcid_count = sum(1 for s in staff if s.get("orcid")) - save_staff(inst, staff, dry_run=args.dry_run) - summary.append( - { - "institution": inst.name, - "staff_total": len(staff), - "with_orcid": orcid_count, - } - ) - except Exception as e: - log.error(f"Failed harvesting {inst.name}: {e}") - summary.append( - {"institution": inst.name, "staff_total": 0, "with_orcid": 0} - ) - time.sleep(1) - - print(f"\n{'='*60}") - print("HARVEST SUMMARY") - print(f"{'='*60}") - total_staff = 0 - total_orcid = 0 - for s in summary: - print( - f" {s['institution']:<45} {s['staff_total']:>5} staff {s['with_orcid']:>4} ORCID" - ) - total_staff += s["staff_total"] - total_orcid += s["with_orcid"] - print(f"{'-'*60}") - print(f" {'TOTAL':<45} {total_staff:>5} staff {total_orcid:>4} ORCID") - print(f"{'='*60}\n") - - -if __name__ == "__main__": - main() +""" +Staff Harvester — fetches real staff names, ORCIDs, departments from OpenAlex +for every configured institution. Saves enriched JSON to data/{inst}_staff.json. + +Usage: + python scripts/harvest_staff_openalex.py # all institutions + python scripts/harvest_staff_openalex.py --institution unilag + python scripts/harvest_staff_openalex.py --dry-run # just print counts +""" + +import argparse +import json +import logging +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from uraas.config.institutions import get_registry, reset_registry + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) + +OPENALEX_BASE = "https://api.openalex.org" +MAILTO = "uraas-bot@research.edu.ng" +MAX_AUTHORS = 500 # cap per institution to avoid very long runs +DELAY = 0.5 # seconds between requests (polite) + + +def _get(url: str, retries: int = 3) -> dict: + """Simple urllib GET with retries.""" + for attempt in range(retries): + try: + req = urllib.request.Request( + url, headers={"User-Agent": f"URAAS/1.0 (mailto:{MAILTO})"} + ) + with urllib.request.urlopen(req, timeout=20) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + if e.code == 429: + wait = 5 * (attempt + 1) + log.warning(f"Rate limited, waiting {wait}s …") + time.sleep(wait) + else: + log.error(f"HTTP {e.code} for {url}") + break + except Exception as e: + log.error(f"Request error ({attempt+1}/{retries}): {e}") + time.sleep(2) + return {} + + +def harvest_institution(inst_config, dry_run: bool = False) -> list: + """ + Harvest staff by looking at recent works from the institution on OpenAlex. + Extracts unique authors from the authorships array. + Returns list of rich staff dicts: {name, orcid, department, faculty, openalex_id, paper_count} + """ + ror_url = inst_config.ror + inst_name = inst_config.name + log.info(f"Harvesting staff for {inst_name} (ROR: {ror_url}) …") + + unique_staff = {} + cursor = "*" + page = 0 + max_pages = 50 # Limit to 50 pages (10k works max) to avoid running forever + + while len(unique_staff) < MAX_AUTHORS and page < max_pages: + # We query the works endpoint using the exact ROR url + url = ( + f"{OPENALEX_BASE}/works" + f"?filter=institutions.ror:{urllib.parse.quote(ror_url)}" + f"&select=authorships" + f"&per-page=200" + f"&cursor={urllib.parse.quote(cursor)}" + f"&mailto={MAILTO}" + ) + data = _get(url) + if not data: + break + + results = data.get("results", []) + if not results: + break + + for work in results: + for authorship in work.get("authorships", []): + # Ensure the author is affiliated with our target institution for this work + is_affiliated = False + for inst in authorship.get("institutions", []): + if inst.get("ror") == ror_url: + is_affiliated = True + break + + if not is_affiliated: + continue + + author = authorship.get("author", {}) + aid = author.get("id") + if not aid or aid in unique_staff: + if aid in unique_staff: + unique_staff[aid]["paper_count"] += 1 + continue + + name = author.get("display_name", "").strip() + if not name: + continue + + orcid_url = author.get("orcid", "") + orcid = ( + orcid_url.replace("https://orcid.org/", "") if orcid_url else None + ) + + # We can't get concepts easily from works authorships without extra queries, + # so we will leave faculty and department empty for now. + + unique_staff[aid] = { + "name": name, + "orcid": orcid, + "department": None, + "faculty": None, + "openalex_id": aid.replace("https://openalex.org/", ""), + "paper_count": 1, + } + + if len(unique_staff) >= MAX_AUTHORS: + break + + if len(unique_staff) >= MAX_AUTHORS: + break + + log.info( + f" Page {page+1}: Processed {len(results)} works | Unique staff so far: {len(unique_staff)}" + ) + page += 1 + time.sleep(DELAY) + + meta = data.get("meta", {}) + cursor = meta.get("next_cursor") + if not cursor: + break + + staff_list = list(unique_staff.values()) + log.info(f" Harvested {len(staff_list)} staff for {inst_name}") + return staff_list + + +def _map_concept_to_faculty(concept: str, faculties: list) -> str: + """Rough concept→faculty mapping via keyword overlap.""" + concept_lower = concept.lower() + faculty_map = { + "medicine": ["health", "medicine", "clinical", "nursing", "pharmacy", "dental"], + "engineering": [ + "engineering", + "technology", + "mechanical", + "electrical", + "civil", + "chemical", + ], + "science": [ + "biology", + "chemistry", + "physics", + "mathematics", + "statistics", + "computer", + ], + "arts": [ + "literature", + "linguistics", + "language", + "history", + "philosophy", + "arts", + ], + "social": [ + "sociology", + "economics", + "political", + "psychology", + "anthropology", + "social", + ], + "law": ["law", "legal", "jurisprudence", "criminology"], + "education": ["education", "pedagogy", "teaching", "curriculum"], + "agriculture": ["agriculture", "botany", "zoology", "ecology", "forestry"], + "management": ["business", "management", "accounting", "finance", "marketing"], + "environmental": [ + "environment", + "urban", + "planning", + "geography", + "architecture", + ], + } + for fac_key, keywords in faculty_map.items(): + if any(kw in concept_lower for kw in keywords): + # Try to match to actual faculty names + for f in faculties: + if fac_key in f.lower() or any(kw in f.lower() for kw in keywords): + return f + return None + + +def get_orcid_details(orcid: str) -> dict: + """Fetch name and affiliation details from ORCID public API.""" + url = f"https://pub.orcid.org/v3.0/{orcid}/person" + try: + req = urllib.request.Request( + url, + headers={ + "Accept": "application/json", + "User-Agent": f"URAAS/1.0 (mailto:{MAILTO})", + }, + ) + with urllib.request.urlopen(req, timeout=15) as resp: + data = json.loads(resp.read().decode()) + affiliations = data.get("activities-summary", {}) + return {"orcid": orcid} + except Exception: + return {} + + +def save_staff(inst_config, staff: list, dry_run: bool = False): + """Save staff list to data/{short_name_lower}_staff.json""" + short = inst_config.short_name.lower() + # Resolve base directory + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + out_path = os.path.join(base_dir, "data", f"{short}_staff.json") + + if dry_run: + log.info(f"[DRY-RUN] Would save {len(staff)} staff records to {out_path}") + return + + os.makedirs(os.path.dirname(out_path), exist_ok=True) + with open(out_path, "w", encoding="utf-8") as f: + json.dump(staff, f, indent=2, ensure_ascii=False) + log.info(f"Saved {len(staff)} staff records → {out_path}") + + +def main(): + parser = argparse.ArgumentParser( + description="Harvest staff from OpenAlex for URAAS institutions" + ) + parser.add_argument( + "--institution", + type=str, + default=None, + help="Single institution short name (default: all)", + ) + parser.add_argument( + "--dry-run", action="store_true", help="Print counts without saving" + ) + args = parser.parse_args() + + reset_registry() + registry = get_registry() + all_insts = registry.list_all() + + if args.institution: + inst = registry.get(args.institution) + if not inst: + print(f"ERROR: Institution '{args.institution}' not found") + sys.exit(1) + target_insts = [inst] + else: + target_insts = all_insts + + print(f"\n{'='*60}") + print(f"URAAS Staff Harvester — OpenAlex") + print(f"Institutions: {len(target_insts)}") + print(f"{'='*60}\n") + + summary = [] + for inst in target_insts: + try: + staff = harvest_institution(inst, dry_run=args.dry_run) + orcid_count = sum(1 for s in staff if s.get("orcid")) + save_staff(inst, staff, dry_run=args.dry_run) + summary.append( + { + "institution": inst.name, + "staff_total": len(staff), + "with_orcid": orcid_count, + } + ) + except Exception as e: + log.error(f"Failed harvesting {inst.name}: {e}") + summary.append( + {"institution": inst.name, "staff_total": 0, "with_orcid": 0} + ) + time.sleep(1) + + print(f"\n{'='*60}") + print("HARVEST SUMMARY") + print(f"{'='*60}") + total_staff = 0 + total_orcid = 0 + for s in summary: + print( + f" {s['institution']:<45} {s['staff_total']:>5} staff {s['with_orcid']:>4} ORCID" + ) + total_staff += s["staff_total"] + total_orcid += s["with_orcid"] + print(f"{'-'*60}") + print(f" {'TOTAL':<45} {total_staff:>5} staff {total_orcid:>4} ORCID") + print(f"{'='*60}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/init_db.py b/scripts/init_db.py index daf2c8eed2570678620b0569ee800e82313a8379..b9a902f7df9ec15d1ae72791eaa1d1d55de6a5a1 100644 --- a/scripts/init_db.py +++ b/scripts/init_db.py @@ -1,78 +1,78 @@ -""" -Database Initialization Script -Creates all tables and seeds Communities and Collections based on UNILAG structure. -""" - -import os -import sys - -# Add project root to path (parent of scripts/) -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from uraas.database import Collection, Community, SessionLocal, init_db -from uraas.utils.unilag_classifier import UNILAG_STRUCTURE - - -def seed_communities_and_collections(): - """Seed the database with UNILAG faculty and department structure.""" - session = SessionLocal() - - try: - print("Seeding Communities (Faculties) and Collections (Departments)...") - - for faculty_name, departments in UNILAG_STRUCTURE.items(): - # Check if community exists - community = session.query(Community).filter_by(name=faculty_name).first() - if not community: - community = Community(name=faculty_name) - session.add(community) - session.flush() - print(f" Created Community: {faculty_name}") - - # Create collections (departments) under this community - for dept_name, keywords in departments.items(): - collection = session.query(Collection).filter_by(name=dept_name).first() - if not collection: - collection = Collection( - community_id=community.id, - name=dept_name, - keywords=", ".join(keywords), - ) - session.add(collection) - print(f" Created Collection: {dept_name}") - - session.commit() - print("\n[OK] Database seeding completed successfully!") - print(f" Total Communities: {session.query(Community).count()}") - print(f" Total Collections: {session.query(Collection).count()}") - - except Exception as e: - print(f"\n[ERR] Error seeding database: {e}") - session.rollback() - raise - finally: - session.close() - - -def main(): - print("=" * 60) - print("URAAS Database Initialization") - print("=" * 60) - print() - - # Create all tables - print("Creating database tables...") - init_db() - print("[OK] Tables created successfully!") - print() - - # Seed communities and collections - seed_communities_and_collections() - print() - print("=" * 60) - print("Database is ready for use!") - print("=" * 60) - - -if __name__ == "__main__": - main() +""" +Database Initialization Script +Creates all tables and seeds Communities and Collections based on UNILAG structure. +""" + +import os +import sys + +# Add project root to path (parent of scripts/) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from uraas.database import Collection, Community, SessionLocal, init_db +from uraas.utils.unilag_classifier import UNILAG_STRUCTURE + + +def seed_communities_and_collections(): + """Seed the database with UNILAG faculty and department structure.""" + session = SessionLocal() + + try: + print("Seeding Communities (Faculties) and Collections (Departments)...") + + for faculty_name, departments in UNILAG_STRUCTURE.items(): + # Check if community exists + community = session.query(Community).filter_by(name=faculty_name).first() + if not community: + community = Community(name=faculty_name) + session.add(community) + session.flush() + print(f" Created Community: {faculty_name}") + + # Create collections (departments) under this community + for dept_name, keywords in departments.items(): + collection = session.query(Collection).filter_by(name=dept_name).first() + if not collection: + collection = Collection( + community_id=community.id, + name=dept_name, + keywords=", ".join(keywords), + ) + session.add(collection) + print(f" Created Collection: {dept_name}") + + session.commit() + print("\n[OK] Database seeding completed successfully!") + print(f" Total Communities: {session.query(Community).count()}") + print(f" Total Collections: {session.query(Collection).count()}") + + except Exception as e: + print(f"\n[ERR] Error seeding database: {e}") + session.rollback() + raise + finally: + session.close() + + +def main(): + print("=" * 60) + print("URAAS Database Initialization") + print("=" * 60) + print() + + # Create all tables + print("Creating database tables...") + init_db() + print("[OK] Tables created successfully!") + print() + + # Seed communities and collections + seed_communities_and_collections() + print() + print("=" * 60) + print("Database is ready for use!") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/scripts/migrate.py b/scripts/migrate.py index e34e284dca45079c5a205fc9399f3f21f8dbc927..73128e58693b591966f54034cf8dd7be077a455f 100644 --- a/scripts/migrate.py +++ b/scripts/migrate.py @@ -1,44 +1,44 @@ -"""Run once to add new APA columns to the existing SQLite database.""" - -import sys - -sys.path.insert(0, ".") -from sqlalchemy import inspect, text - -from uraas.database import engine - -NEW_COLS = [ - ("items", "dc_type", "TEXT"), - ("items", "dc_language", "TEXT"), - ("items", "dc_subject", "TEXT"), - ("items", "docid", "TEXT UNIQUE"), - ("items", "docid_assigned_at", "DATETIME"), - ("items", "content_type", 'TEXT DEFAULT "research_paper"'), - ("items", "tk_label", "TEXT"), - ("items", "tk_community", "TEXT"), - ("items", "patent_id", "TEXT"), - ("items", "patent_date", "DATETIME"), - ("items", "language_code", "TEXT"), - ("items", "is_african_language", "INTEGER DEFAULT 0"), - ("items", "sdg_tags", "TEXT"), - ("items", "ai_keywords", "TEXT"), - ("authors", "orcid", "TEXT"), - ("authors", "ror", "TEXT"), - ("communities", "ror_id", "TEXT"), - ("communities", "institution", "TEXT"), -] - -inspector = inspect(engine) -with engine.connect() as conn: - for table, col, col_type in NEW_COLS: - existing = [c["name"] for c in inspector.get_columns(table)] - if col not in existing: - # SQLite doesn't support UNIQUE in ALTER TABLE — skip constraint - safe_type = col_type.replace(" UNIQUE", "") - conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {col} {safe_type}")) - print(f" + {table}.{col}") - else: - print(f" . {table}.{col} (exists)") - conn.commit() - -print("Migration complete.") +"""Run once to add new APA columns to the existing SQLite database.""" + +import sys + +sys.path.insert(0, ".") +from sqlalchemy import inspect, text + +from uraas.database import engine + +NEW_COLS = [ + ("items", "dc_type", "TEXT"), + ("items", "dc_language", "TEXT"), + ("items", "dc_subject", "TEXT"), + ("items", "docid", "TEXT UNIQUE"), + ("items", "docid_assigned_at", "DATETIME"), + ("items", "content_type", 'TEXT DEFAULT "research_paper"'), + ("items", "tk_label", "TEXT"), + ("items", "tk_community", "TEXT"), + ("items", "patent_id", "TEXT"), + ("items", "patent_date", "DATETIME"), + ("items", "language_code", "TEXT"), + ("items", "is_african_language", "INTEGER DEFAULT 0"), + ("items", "sdg_tags", "TEXT"), + ("items", "ai_keywords", "TEXT"), + ("authors", "orcid", "TEXT"), + ("authors", "ror", "TEXT"), + ("communities", "ror_id", "TEXT"), + ("communities", "institution", "TEXT"), +] + +inspector = inspect(engine) +with engine.connect() as conn: + for table, col, col_type in NEW_COLS: + existing = [c["name"] for c in inspector.get_columns(table)] + if col not in existing: + # SQLite doesn't support UNIQUE in ALTER TABLE — skip constraint + safe_type = col_type.replace(" UNIQUE", "") + conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {col} {safe_type}")) + print(f" + {table}.{col}") + else: + print(f" . {table}.{col} (exists)") + conn.commit() + +print("Migration complete.") diff --git a/scripts/migrate_2026_upgrade.py b/scripts/migrate_2026_upgrade.py index 69a5a466a9c37bf0d10d81eb41fefec5bb5da71e..63640254dc4f9416fc2d484d3c1513baa2ba8275 100644 --- a/scripts/migrate_2026_upgrade.py +++ b/scripts/migrate_2026_upgrade.py @@ -1,86 +1,86 @@ -""" -Schema migration: 2026 UNESCO upgrade. Idempotent; safe on SQLite + Postgres. - -Adds to items: - - alignment_scores (JSON TEXT) + alignment_version (framework alignment) - - coauthor_countries / african_country_count / is_intra_african (collaboration) - - openalex_id / counts_by_year / cited_by_count / african_citation_share (citations) - - ark / ark_assigned_at (ARK persistent identifiers) - -New tables (item_affiliations, alignment_aggregates) are created by -scripts/init_db.py via Base.metadata.create_all — run init_db.py first. -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from sqlalchemy import inspect, text - -from uraas.database import engine - -# (column, DDL type clause) — types chosen to work on both SQLite and Postgres. -ITEMS_COLUMNS = [ - ("alignment_scores", "TEXT"), - ("alignment_version", "INTEGER DEFAULT 0"), - ("coauthor_countries", "TEXT"), - ("african_country_count", "INTEGER DEFAULT 0"), - ("is_intra_african", "BOOLEAN DEFAULT FALSE"), - ("openalex_id", "VARCHAR(64)"), - ("counts_by_year", "TEXT"), - ("cited_by_count", "INTEGER DEFAULT 0"), - ("african_citation_share", "FLOAT"), - ("ark", "VARCHAR(128)"), - ("ark_assigned_at", "TIMESTAMP"), -] - -INDEXES = [ - "CREATE INDEX IF NOT EXISTS ix_items_is_intra_african ON items (is_intra_african)", - "CREATE UNIQUE INDEX IF NOT EXISTS ux_items_ark ON items (ark)", -] - - -def column_exists(insp, table: str, column: str) -> bool: - return column in {c["name"] for c in insp.get_columns(table)} - - -def main() -> int: - print("Migration: 2026 upgrade (alignment / collaboration / citations / ARK)") - - dialect = engine.dialect.name - print(f"Dialect: {dialect}") - - insp = inspect(engine) - statements = [] - for column, ddl_type in ITEMS_COLUMNS: - if column_exists(insp, "items", column): - print(f" {column} already present, skipping") - continue - if dialect == "sqlite" and "BOOLEAN" in ddl_type: - # SQLite stores booleans as integers - ddl_type = ddl_type.replace("BOOLEAN", "INTEGER").replace("FALSE", "0") - statements.append(f"ALTER TABLE items ADD COLUMN {column} {ddl_type}") - - if statements: - with engine.begin() as conn: - for stmt in statements: - print(f" -> {stmt}") - conn.execute(text(stmt)) - else: - print(" All columns already present.") - - for stmt in INDEXES: - try: - with engine.begin() as conn: - conn.execute(text(stmt)) - print(f" -> {stmt.split(' ON ')[0].replace('CREATE ', '').strip()} ensured") - except Exception as e: - print(f" (index creation skipped: {e})") - - print("Done.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +""" +Schema migration: 2026 UNESCO upgrade. Idempotent; safe on SQLite + Postgres. + +Adds to items: + - alignment_scores (JSON TEXT) + alignment_version (framework alignment) + - coauthor_countries / african_country_count / is_intra_african (collaboration) + - openalex_id / counts_by_year / cited_by_count / african_citation_share (citations) + - ark / ark_assigned_at (ARK persistent identifiers) + +New tables (item_affiliations, alignment_aggregates) are created by +scripts/init_db.py via Base.metadata.create_all — run init_db.py first. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from sqlalchemy import inspect, text + +from uraas.database import engine + +# (column, DDL type clause) — types chosen to work on both SQLite and Postgres. +ITEMS_COLUMNS = [ + ("alignment_scores", "TEXT"), + ("alignment_version", "INTEGER DEFAULT 0"), + ("coauthor_countries", "TEXT"), + ("african_country_count", "INTEGER DEFAULT 0"), + ("is_intra_african", "BOOLEAN DEFAULT FALSE"), + ("openalex_id", "VARCHAR(64)"), + ("counts_by_year", "TEXT"), + ("cited_by_count", "INTEGER DEFAULT 0"), + ("african_citation_share", "FLOAT"), + ("ark", "VARCHAR(128)"), + ("ark_assigned_at", "TIMESTAMP"), +] + +INDEXES = [ + "CREATE INDEX IF NOT EXISTS ix_items_is_intra_african ON items (is_intra_african)", + "CREATE UNIQUE INDEX IF NOT EXISTS ux_items_ark ON items (ark)", +] + + +def column_exists(insp, table: str, column: str) -> bool: + return column in {c["name"] for c in insp.get_columns(table)} + + +def main() -> int: + print("Migration: 2026 upgrade (alignment / collaboration / citations / ARK)") + + dialect = engine.dialect.name + print(f"Dialect: {dialect}") + + insp = inspect(engine) + statements = [] + for column, ddl_type in ITEMS_COLUMNS: + if column_exists(insp, "items", column): + print(f" {column} already present, skipping") + continue + if dialect == "sqlite" and "BOOLEAN" in ddl_type: + # SQLite stores booleans as integers + ddl_type = ddl_type.replace("BOOLEAN", "INTEGER").replace("FALSE", "0") + statements.append(f"ALTER TABLE items ADD COLUMN {column} {ddl_type}") + + if statements: + with engine.begin() as conn: + for stmt in statements: + print(f" -> {stmt}") + conn.execute(text(stmt)) + else: + print(" All columns already present.") + + for stmt in INDEXES: + try: + with engine.begin() as conn: + conn.execute(text(stmt)) + print(f" -> {stmt.split(' ON ')[0].replace('CREATE ', '').strip()} ensured") + except Exception as e: + print(f" (index creation skipped: {e})") + + print("Done.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/migrate_add_ror.py b/scripts/migrate_add_ror.py index aa76a8ccbedd7306357784914f98fa6ac188b204..233c5d19e4f4aec506403fb034df9caac20f4188 100644 --- a/scripts/migrate_add_ror.py +++ b/scripts/migrate_add_ror.py @@ -1,69 +1,69 @@ -""" -Database migration: Add ROR support for multi-institution comparison -""" - -from sqlalchemy import text - -from uraas.database import Item, SessionLocal, engine - - -def migrate(): - print("Adding ROR columns to items table...") - - with engine.connect() as conn: - try: - # Add ror column - conn.execute(text("ALTER TABLE items ADD COLUMN ror VARCHAR(128)")) - conn.execute(text("CREATE INDEX ix_items_ror ON items(ror)")) - print("✓ Added ror column and index") - except Exception as e: - if ( - "duplicate column" in str(e).lower() - or "already exists" in str(e).lower() - ): - print("✓ ROR column already exists") - else: - print(f"Error: {e}") - - try: - # Add institution column if not exists - conn.execute(text("ALTER TABLE items ADD COLUMN institution VARCHAR(255)")) - print("✓ Added institution column") - except Exception as e: - if ( - "duplicate column" in str(e).lower() - or "already exists" in str(e).lower() - ): - print("✓ Institution column already exists") - else: - print(f"Error: {e}") - - conn.commit() - - # Set default ROR for UNILAG papers - print("\nSetting default ROR for existing UNILAG papers...") - session = SessionLocal() - try: - unilag_ror = "https://ror.org/03qcnxw14" - count = ( - session.query(Item) - .filter(Item.ror.is_(None)) - .update( - {Item.ror: unilag_ror, Item.institution: "University of Lagos"}, - synchronize_session=False, - ) - ) - session.commit() - print(f"✓ Updated {count} papers with UNILAG ROR") - finally: - session.close() - - print("\nMigration complete!") - print("\nNext steps:") - print("1. Add Comparator tab to dashboard") - print("2. Test multi-institution comparison") - print("3. Add more institutions to database") - - -if __name__ == "__main__": - migrate() +""" +Database migration: Add ROR support for multi-institution comparison +""" + +from sqlalchemy import text + +from uraas.database import Item, SessionLocal, engine + + +def migrate(): + print("Adding ROR columns to items table...") + + with engine.connect() as conn: + try: + # Add ror column + conn.execute(text("ALTER TABLE items ADD COLUMN ror VARCHAR(128)")) + conn.execute(text("CREATE INDEX ix_items_ror ON items(ror)")) + print("✓ Added ror column and index") + except Exception as e: + if ( + "duplicate column" in str(e).lower() + or "already exists" in str(e).lower() + ): + print("✓ ROR column already exists") + else: + print(f"Error: {e}") + + try: + # Add institution column if not exists + conn.execute(text("ALTER TABLE items ADD COLUMN institution VARCHAR(255)")) + print("✓ Added institution column") + except Exception as e: + if ( + "duplicate column" in str(e).lower() + or "already exists" in str(e).lower() + ): + print("✓ Institution column already exists") + else: + print(f"Error: {e}") + + conn.commit() + + # Set default ROR for UNILAG papers + print("\nSetting default ROR for existing UNILAG papers...") + session = SessionLocal() + try: + unilag_ror = "https://ror.org/03qcnxw14" + count = ( + session.query(Item) + .filter(Item.ror.is_(None)) + .update( + {Item.ror: unilag_ror, Item.institution: "University of Lagos"}, + synchronize_session=False, + ) + ) + session.commit() + print(f"✓ Updated {count} papers with UNILAG ROR") + finally: + session.close() + + print("\nMigration complete!") + print("\nNext steps:") + print("1. Add Comparator tab to dashboard") + print("2. Test multi-institution comparison") + print("3. Add more institutions to database") + + +if __name__ == "__main__": + migrate() diff --git a/scripts/migrate_add_sc_columns.py b/scripts/migrate_add_sc_columns.py index 4eb58838c7f15dbbeca787768abd0e2013aefb78..6bb7b1bb7fe6ebcb9684eb2f52b1e99fe3d67650 100644 --- a/scripts/migrate_add_sc_columns.py +++ b/scripts/migrate_add_sc_columns.py @@ -1,72 +1,72 @@ -""" -Schema migration: add special_collection_score + special_collection_categories -columns to items table. Idempotent. -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from sqlalchemy import inspect, text - -from uraas.database import engine - - -def column_exists(table: str, column: str) -> bool: - insp = inspect(engine) - return column in {c["name"] for c in insp.get_columns(table)} - - -def main() -> int: - print( - "Migration: adding special_collection_score + special_collection_categories to items" - ) - - dialect = engine.dialect.name - print(f"Dialect: {dialect}") - - statements = [] - if not column_exists("items", "special_collection_score"): - statements.append( - "ALTER TABLE items ADD COLUMN special_collection_score FLOAT DEFAULT 0.0" - ) - else: - print(" special_collection_score already present, skipping") - - if not column_exists("items", "special_collection_categories"): - # TEXT for both sqlite + postgres - statements.append( - "ALTER TABLE items ADD COLUMN special_collection_categories TEXT" - ) - else: - print(" special_collection_categories already present, skipping") - - if not statements: - print("Nothing to do.") - return 0 - - with engine.begin() as conn: - for stmt in statements: - print(f" -> {stmt}") - conn.execute(text(stmt)) - - # Index on score so ORDER BY score DESC is fast - try: - with engine.begin() as conn: - conn.execute( - text( - "CREATE INDEX IF NOT EXISTS ix_items_special_collection_score " - "ON items (special_collection_score)" - ) - ) - print(" -> index ix_items_special_collection_score ensured") - except Exception as e: - print(f" (index creation skipped: {e})") - - print("Done.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +""" +Schema migration: add special_collection_score + special_collection_categories +columns to items table. Idempotent. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from sqlalchemy import inspect, text + +from uraas.database import engine + + +def column_exists(table: str, column: str) -> bool: + insp = inspect(engine) + return column in {c["name"] for c in insp.get_columns(table)} + + +def main() -> int: + print( + "Migration: adding special_collection_score + special_collection_categories to items" + ) + + dialect = engine.dialect.name + print(f"Dialect: {dialect}") + + statements = [] + if not column_exists("items", "special_collection_score"): + statements.append( + "ALTER TABLE items ADD COLUMN special_collection_score FLOAT DEFAULT 0.0" + ) + else: + print(" special_collection_score already present, skipping") + + if not column_exists("items", "special_collection_categories"): + # TEXT for both sqlite + postgres + statements.append( + "ALTER TABLE items ADD COLUMN special_collection_categories TEXT" + ) + else: + print(" special_collection_categories already present, skipping") + + if not statements: + print("Nothing to do.") + return 0 + + with engine.begin() as conn: + for stmt in statements: + print(f" -> {stmt}") + conn.execute(text(stmt)) + + # Index on score so ORDER BY score DESC is fast + try: + with engine.begin() as conn: + conn.execute( + text( + "CREATE INDEX IF NOT EXISTS ix_items_special_collection_score " + "ON items (special_collection_score)" + ) + ) + print(" -> index ix_items_special_collection_score ensured") + except Exception as e: + print(f" (index creation skipped: {e})") + + print("Done.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/migrate_sqlite_to_postgres.py b/scripts/migrate_sqlite_to_postgres.py index 83d0b64a815f2e2386fc9d1aec65e65c16fe8de5..d49e075b0f7b24672c4d00577167cd82a1dc7954 100644 --- a/scripts/migrate_sqlite_to_postgres.py +++ b/scripts/migrate_sqlite_to_postgres.py @@ -1,202 +1,202 @@ -""" -Migration script: Copy all data from local SQLite database (uraas.db) -to the production PostgreSQL database. -""" - -import os -import sys -from sqlalchemy import create_engine, MetaData, text -from sqlalchemy.orm import sessionmaker - -# Add project root to path -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from uraas.database import Base, Community, Collection, Author, Item, File, item_authors, item_collections - -def migrate(): - # SQLite URL - sqlite_url = "sqlite:///uraas.db" - - # Postgres URL (get from environment variable) - postgres_url = os.getenv("DATABASE_URL") - if not postgres_url: - print("[ERR] DATABASE_URL environment variable is not set!") - print("Please run this command with DATABASE_URL set, for example:") - print("DATABASE_URL=postgresql://user:pass@host:port/dbname python scripts/migrate_sqlite_to_postgres.py") - sys.exit(1) - - # Standardize Render's postgres:// prefix to postgresql:// if needed - if postgres_url.startswith("postgres://"): - postgres_url = postgres_url.replace("postgres://", "postgresql://", 1) - - print(f"Source SQLite database: {sqlite_url}") - print(f"Destination PostgreSQL database: {postgres_url.split('@')[-1] if '@' in postgres_url else postgres_url}") - print("\nInitializing connections...") - - sqlite_engine = create_engine(sqlite_url) - postgres_engine = create_engine(postgres_url) - - SqliteSession = sessionmaker(bind=sqlite_engine) - PostgresSession = sessionmaker(bind=postgres_engine) - - sqlite_session = SqliteSession() - postgres_session = PostgresSession() - - try: - print("Recreating destination database tables if they do not exist...") - Base.metadata.create_all(bind=postgres_engine) - - print("Clearing existing data in PostgreSQL tables to prevent collisions...") - # Order matters for foreign key constraints - postgres_session.execute(text("TRUNCATE TABLE files, item_authors, item_collections, items, authors, collections, communities CASCADE")) - postgres_session.commit() - - # 1. Migrate Communities - print("Migrating Communities...") - communities = sqlite_session.query(Community).all() - for comm in communities: - new_comm = Community( - id=comm.id, - name=comm.name, - ror_id=comm.ror_id, - institution=comm.institution, - ror=comm.ror - ) - postgres_session.add(new_comm) - postgres_session.flush() - print(f" Migrated {len(communities)} communities.") - - # 2. Migrate Collections - print("Migrating Collections...") - collections = sqlite_session.query(Collection).all() - for coll in collections: - new_coll = Collection( - id=coll.id, - community_id=coll.community_id, - name=coll.name, - email_domains=coll.email_domains, - keywords=coll.keywords - ) - postgres_session.add(new_coll) - postgres_session.flush() - print(f" Migrated {len(collections)} collections.") - - # 3. Migrate Authors - print("Migrating Authors...") - authors = sqlite_session.query(Author).all() - for auth in authors: - new_auth = Author( - id=auth.id, - name=auth.name, - normalized_name=auth.normalized_name, - profile_url=auth.profile_url, - orcid=auth.orcid, - ror=auth.ror - ) - postgres_session.add(new_auth) - postgres_session.flush() - print(f" Migrated {len(authors)} authors.") - - # 4. Migrate Items - print("Migrating Items...") - items = sqlite_session.query(Item).all() - for item in items: - new_item = Item( - id=item.id, - title=item.title, - abstract=item.abstract, - doi=item.doi, - publication_date=item.publication_date, - url=item.url, - source_repository=item.source_repository, - pdf_url=item.pdf_url, - dc_title=item.dc_title, - dc_date_issued=item.dc_date_issued, - dc_identifier_uri=item.dc_identifier_uri, - dc_identifier_doi=item.dc_identifier_doi, - dc_description_provenance=item.dc_description_provenance, - dc_rights=item.dc_rights, - dc_type=item.dc_type, - dc_language=item.dc_language, - dc_subject=item.dc_subject, - docid=item.docid, - docid_assigned_at=item.docid_assigned_at, - ror=item.ror, - institution=item.institution, - content_type=item.content_type, - tk_label=item.tk_label, - tk_community=item.tk_community, - patent_id=item.patent_id, - patent_date=item.patent_date, - language_code=item.language_code, - is_african_language=item.is_african_language, - sdg_tags=item.sdg_tags, - ai_keywords=item.ai_keywords, - special_collection_score=item.special_collection_score, - special_collection_categories=item.special_collection_categories, - created_at=item.created_at - ) - postgres_session.add(new_item) - postgres_session.flush() - print(f" Migrated {len(items)} items.") - - # 5. Migrate Files - print("Migrating Files...") - files = sqlite_session.query(File).all() - for file in files: - new_file = File( - id=file.id, - item_id=file.item_id, - file_path=file.file_path, - sha256_hash=file.sha256_hash, - access_policy=file.access_policy, - downloaded_at=file.downloaded_at - ) - postgres_session.add(new_file) - postgres_session.flush() - print(f" Migrated {len(files)} files.") - - # 6. Migrate association tables (item_authors and item_collections) - print("Migrating Item-Author associations...") - item_author_rows = sqlite_session.execute(item_authors.select()).all() - for row in item_author_rows: - postgres_session.execute( - item_authors.insert().values(item_id=row.item_id, author_id=row.author_id) - ) - print(f" Migrated {len(item_author_rows)} item-author mappings.") - - print("Migrating Item-Collection associations...") - item_coll_rows = sqlite_session.execute(item_collections.select()).all() - for row in item_coll_rows: - postgres_session.execute( - item_collections.insert().values( - item_id=row.item_id, - collection_id=row.collection_id, - confidence_score=row.confidence_score - ) - ) - print(f" Migrated {len(item_coll_rows)} item-collection mappings.") - - postgres_session.commit() - print("[SUCCESS] Data migrated to PostgreSQL successfully!") - - # Reset sequences in Postgres so future inserts don't collide - print("Resetting PostgreSQL primary key sequences...") - tables = ["communities", "collections", "authors", "items", "files"] - for table in tables: - postgres_session.execute(text( - f"SELECT setval(pg_get_serial_sequence('{table}', 'id'), COALESCE(MAX(id), 1) + 1) FROM {table}" - )) - postgres_session.commit() - print("[SUCCESS] Sequences advanced.") - - except Exception as e: - print(f"[ERR] Migration failed: {e}") - postgres_session.rollback() - raise - finally: - sqlite_session.close() - postgres_session.close() - -if __name__ == "__main__": - migrate() +""" +Migration script: Copy all data from local SQLite database (uraas.db) +to the production PostgreSQL database. +""" + +import os +import sys +from sqlalchemy import create_engine, MetaData, text +from sqlalchemy.orm import sessionmaker + +# Add project root to path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from uraas.database import Base, Community, Collection, Author, Item, File, item_authors, item_collections + +def migrate(): + # SQLite URL + sqlite_url = "sqlite:///uraas.db" + + # Postgres URL (get from environment variable) + postgres_url = os.getenv("DATABASE_URL") + if not postgres_url: + print("[ERR] DATABASE_URL environment variable is not set!") + print("Please run this command with DATABASE_URL set, for example:") + print("DATABASE_URL=postgresql://user:pass@host:port/dbname python scripts/migrate_sqlite_to_postgres.py") + sys.exit(1) + + # Standardize Render's postgres:// prefix to postgresql:// if needed + if postgres_url.startswith("postgres://"): + postgres_url = postgres_url.replace("postgres://", "postgresql://", 1) + + print(f"Source SQLite database: {sqlite_url}") + print(f"Destination PostgreSQL database: {postgres_url.split('@')[-1] if '@' in postgres_url else postgres_url}") + print("\nInitializing connections...") + + sqlite_engine = create_engine(sqlite_url) + postgres_engine = create_engine(postgres_url) + + SqliteSession = sessionmaker(bind=sqlite_engine) + PostgresSession = sessionmaker(bind=postgres_engine) + + sqlite_session = SqliteSession() + postgres_session = PostgresSession() + + try: + print("Recreating destination database tables if they do not exist...") + Base.metadata.create_all(bind=postgres_engine) + + print("Clearing existing data in PostgreSQL tables to prevent collisions...") + # Order matters for foreign key constraints + postgres_session.execute(text("TRUNCATE TABLE files, item_authors, item_collections, items, authors, collections, communities CASCADE")) + postgres_session.commit() + + # 1. Migrate Communities + print("Migrating Communities...") + communities = sqlite_session.query(Community).all() + for comm in communities: + new_comm = Community( + id=comm.id, + name=comm.name, + ror_id=comm.ror_id, + institution=comm.institution, + ror=comm.ror + ) + postgres_session.add(new_comm) + postgres_session.flush() + print(f" Migrated {len(communities)} communities.") + + # 2. Migrate Collections + print("Migrating Collections...") + collections = sqlite_session.query(Collection).all() + for coll in collections: + new_coll = Collection( + id=coll.id, + community_id=coll.community_id, + name=coll.name, + email_domains=coll.email_domains, + keywords=coll.keywords + ) + postgres_session.add(new_coll) + postgres_session.flush() + print(f" Migrated {len(collections)} collections.") + + # 3. Migrate Authors + print("Migrating Authors...") + authors = sqlite_session.query(Author).all() + for auth in authors: + new_auth = Author( + id=auth.id, + name=auth.name, + normalized_name=auth.normalized_name, + profile_url=auth.profile_url, + orcid=auth.orcid, + ror=auth.ror + ) + postgres_session.add(new_auth) + postgres_session.flush() + print(f" Migrated {len(authors)} authors.") + + # 4. Migrate Items + print("Migrating Items...") + items = sqlite_session.query(Item).all() + for item in items: + new_item = Item( + id=item.id, + title=item.title, + abstract=item.abstract, + doi=item.doi, + publication_date=item.publication_date, + url=item.url, + source_repository=item.source_repository, + pdf_url=item.pdf_url, + dc_title=item.dc_title, + dc_date_issued=item.dc_date_issued, + dc_identifier_uri=item.dc_identifier_uri, + dc_identifier_doi=item.dc_identifier_doi, + dc_description_provenance=item.dc_description_provenance, + dc_rights=item.dc_rights, + dc_type=item.dc_type, + dc_language=item.dc_language, + dc_subject=item.dc_subject, + docid=item.docid, + docid_assigned_at=item.docid_assigned_at, + ror=item.ror, + institution=item.institution, + content_type=item.content_type, + tk_label=item.tk_label, + tk_community=item.tk_community, + patent_id=item.patent_id, + patent_date=item.patent_date, + language_code=item.language_code, + is_african_language=item.is_african_language, + sdg_tags=item.sdg_tags, + ai_keywords=item.ai_keywords, + special_collection_score=item.special_collection_score, + special_collection_categories=item.special_collection_categories, + created_at=item.created_at + ) + postgres_session.add(new_item) + postgres_session.flush() + print(f" Migrated {len(items)} items.") + + # 5. Migrate Files + print("Migrating Files...") + files = sqlite_session.query(File).all() + for file in files: + new_file = File( + id=file.id, + item_id=file.item_id, + file_path=file.file_path, + sha256_hash=file.sha256_hash, + access_policy=file.access_policy, + downloaded_at=file.downloaded_at + ) + postgres_session.add(new_file) + postgres_session.flush() + print(f" Migrated {len(files)} files.") + + # 6. Migrate association tables (item_authors and item_collections) + print("Migrating Item-Author associations...") + item_author_rows = sqlite_session.execute(item_authors.select()).all() + for row in item_author_rows: + postgres_session.execute( + item_authors.insert().values(item_id=row.item_id, author_id=row.author_id) + ) + print(f" Migrated {len(item_author_rows)} item-author mappings.") + + print("Migrating Item-Collection associations...") + item_coll_rows = sqlite_session.execute(item_collections.select()).all() + for row in item_coll_rows: + postgres_session.execute( + item_collections.insert().values( + item_id=row.item_id, + collection_id=row.collection_id, + confidence_score=row.confidence_score + ) + ) + print(f" Migrated {len(item_coll_rows)} item-collection mappings.") + + postgres_session.commit() + print("[SUCCESS] Data migrated to PostgreSQL successfully!") + + # Reset sequences in Postgres so future inserts don't collide + print("Resetting PostgreSQL primary key sequences...") + tables = ["communities", "collections", "authors", "items", "files"] + for table in tables: + postgres_session.execute(text( + f"SELECT setval(pg_get_serial_sequence('{table}', 'id'), COALESCE(MAX(id), 1) + 1) FROM {table}" + )) + postgres_session.commit() + print("[SUCCESS] Sequences advanced.") + + except Exception as e: + print(f"[ERR] Migration failed: {e}") + postgres_session.rollback() + raise + finally: + sqlite_session.close() + postgres_session.close() + +if __name__ == "__main__": + migrate() diff --git a/scripts/migrate_unilag_ror.py b/scripts/migrate_unilag_ror.py index 199221101165f3f37c27462d76b06142ca51f524..42e60189fd0ccdc6cdfd76991f398d59f3b22829 100644 --- a/scripts/migrate_unilag_ror.py +++ b/scripts/migrate_unilag_ror.py @@ -1,55 +1,55 @@ -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from uraas.database import Author, Community, Item, SessionLocal - - -def migrate(): - session = SessionLocal() - try: - old_ror = "https://ror.org/03qcnxw14" - new_ror = "https://ror.org/05rk03822" - - print("Migrating UNILAG RORs in database...") - - # 1. Update items - item_count = ( - session.query(Item) - .filter(Item.ror == old_ror) - .update({Item.ror: new_ror}, synchronize_session=False) - ) - print(f"[OK] Updated {item_count} items") - - # 2. Update communities - comm_count = ( - session.query(Community) - .filter((Community.ror == old_ror) | (Community.ror_id == old_ror)) - .update( - {Community.ror: new_ror, Community.ror_id: new_ror}, - synchronize_session=False, - ) - ) - print(f"[OK] Updated {comm_count} communities") - - # 3. Update authors - author_count = ( - session.query(Author) - .filter(Author.ror == old_ror) - .update({Author.ror: new_ror}, synchronize_session=False) - ) - print(f"[OK] Updated {author_count} authors") - - session.commit() - print("Migration complete successfully!") - - except Exception as e: - session.rollback() - print(f"Error during migration: {e}") - finally: - session.close() - - -if __name__ == "__main__": - migrate() +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from uraas.database import Author, Community, Item, SessionLocal + + +def migrate(): + session = SessionLocal() + try: + old_ror = "https://ror.org/03qcnxw14" + new_ror = "https://ror.org/05rk03822" + + print("Migrating UNILAG RORs in database...") + + # 1. Update items + item_count = ( + session.query(Item) + .filter(Item.ror == old_ror) + .update({Item.ror: new_ror}, synchronize_session=False) + ) + print(f"[OK] Updated {item_count} items") + + # 2. Update communities + comm_count = ( + session.query(Community) + .filter((Community.ror == old_ror) | (Community.ror_id == old_ror)) + .update( + {Community.ror: new_ror, Community.ror_id: new_ror}, + synchronize_session=False, + ) + ) + print(f"[OK] Updated {comm_count} communities") + + # 3. Update authors + author_count = ( + session.query(Author) + .filter(Author.ror == old_ror) + .update({Author.ror: new_ror}, synchronize_session=False) + ) + print(f"[OK] Updated {author_count} authors") + + session.commit() + print("Migration complete successfully!") + + except Exception as e: + session.rollback() + print(f"Error during migration: {e}") + finally: + session.close() + + +if __name__ == "__main__": + migrate() diff --git a/scripts/patch_html.py b/scripts/patch_html.py index c31b0d71e1a76dce1627ed4b2d55d871bd74ecef..87ad7a89b9ae9020f7693dcfa934c1453184c778 100644 --- a/scripts/patch_html.py +++ b/scripts/patch_html.py @@ -1,74 +1,74 @@ -import re - -with open("uraas/dashboard/templates/index.html", "r", encoding="utf-8") as f: - content = f.read() - -# Find the special collections section and replace it -pattern = r"( \s*)" -replacement = """ - - - - - - - """ - -new_content = re.sub(pattern, replacement, content, flags=re.DOTALL) -if new_content == content: - print("ERROR: Pattern not matched. Trying direct string replace...") - # Try to find where special collections starts - idx = content.find("") - print(f"Found at index: {idx}") - if idx >= 0: - end_marker = " " - end_idx = content.find(end_marker, idx) - print(f"End marker at: {end_idx}") - segment = content[idx : end_idx + len(end_marker)] - print(f"Segment length: {len(segment)}") - print("First 200 chars:", repr(segment[:200])) -else: - with open("uraas/dashboard/templates/index.html", "w", encoding="utf-8") as f: - f.write(new_content) - print("SUCCESS") +import re + +with open("uraas/dashboard/templates/index.html", "r", encoding="utf-8") as f: + content = f.read() + +# Find the special collections section and replace it +pattern = r"( \s*)" +replacement = """ + + + + + + + """ + +new_content = re.sub(pattern, replacement, content, flags=re.DOTALL) +if new_content == content: + print("ERROR: Pattern not matched. Trying direct string replace...") + # Try to find where special collections starts + idx = content.find("") + print(f"Found at index: {idx}") + if idx >= 0: + end_marker = " " + end_idx = content.find(end_marker, idx) + print(f"End marker at: {end_idx}") + segment = content[idx : end_idx + len(end_marker)] + print(f"Segment length: {len(segment)}") + print("First 200 chars:", repr(segment[:200])) +else: + with open("uraas/dashboard/templates/index.html", "w", encoding="utf-8") as f: + f.write(new_content) + print("SUCCESS") diff --git a/scripts/push_to_hf.py b/scripts/push_to_hf.py index b1238d5f488d084a3579775f8e7fc74ee6dcc648..ebfd3970b83caffe9635e2fda029c3f3e7f9ea7c 100644 --- a/scripts/push_to_hf.py +++ b/scripts/push_to_hf.py @@ -1,195 +1,195 @@ -""" -Push URAAS to Hugging Face Spaces — Lordkiki/APA-URAAS - -Usage: - python scripts/push_to_hf.py - -What it does: - 1. Logs you into HF (paste your write token when prompted) - 2. Stages a clean copy of the project: - Dockerfile.hf → Dockerfile (HF Spaces Dockerfile, not the prod one) - README.hf.md → README.md (has the HF Space frontmatter) - 3. Uploads everything to the Space via huggingface_hub.upload_folder - 4. HF triggers an auto-build — app is live in ~5 minutes -""" - -import os -import shutil -import sys -import tempfile - -# ── Config ──────────────────────────────────────────────────────────────────── -REPO_ID = "Lordkiki/APA-URAAS" -REPO_TYPE = "space" -REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# Directories/files to never push -IGNORE_DIRS = { - ".git", ".claude", "__pycache__", ".pytest_cache", ".mypy_cache", - "storage", "logs", "data", "backups", "node_modules", - ".venv", "venv", "env", -} -IGNORE_FILES = { - ".env", ".env.prod", ".env.prod.example", - "uraas.db", # DB lives on /data in HF, not in image - "Dockerfile", # replaced by Dockerfile.hf - "README.md", # replaced by README.hf.md (has HF frontmatter) - "docker-compose.yml", - "docker-compose.prod.yml", - "docker-compose.replica.yml", - "docker-compose.demo.yml", -} -IGNORE_EXTS = {".pyc", ".pyo", ".pyd"} - - -def should_skip(rel_path: str, is_dir: bool) -> bool: - parts = rel_path.replace("\\", "/").split("/") - name = parts[-1] - if is_dir: - return name in IGNORE_DIRS - return ( - name in IGNORE_FILES - or os.path.splitext(name)[1] in IGNORE_EXTS - ) - - -def stage_project(src: str, dst: str) -> int: - """Copy src → dst with HF-specific renames and exclusions.""" - count = 0 - for root, dirs, files in os.walk(src): - rel_root = os.path.relpath(root, src) - - # Prune excluded directories in-place so os.walk doesn't recurse - dirs[:] = [ - d for d in dirs - if not should_skip( - os.path.join(rel_root, d) if rel_root != "." else d, - is_dir=True, - ) - ] - - for fname in files: - rel = os.path.join(rel_root, fname) if rel_root != "." else fname - if should_skip(rel, is_dir=False): - continue - - src_file = os.path.join(root, fname) - - # HF-specific renames - if fname == "Dockerfile.hf": - dest_rel = os.path.join(os.path.dirname(rel), "Dockerfile") if os.path.dirname(rel) else "Dockerfile" - elif fname == "README.hf.md": - dest_rel = "README.md" - else: - dest_rel = rel - - dst_file = os.path.join(dst, dest_rel) - os.makedirs(os.path.dirname(dst_file), exist_ok=True) - shutil.copy2(src_file, dst_file) - count += 1 - return count - - -def main(): - # ── Ensure huggingface_hub is available ──────────────────────────────── - try: - from huggingface_hub import HfApi, login - except ImportError: - print("Installing huggingface_hub…") - os.system(f"{sys.executable} -m pip install huggingface_hub -q") - from huggingface_hub import HfApi, login # type: ignore - - print() - print("═" * 55) - print(" URAAS → Hugging Face Spaces") - print(f" Space: {REPO_ID}") - print("═" * 55) - - # ── Auth ─────────────────────────────────────────────────────────────── - token = os.getenv("HF_TOKEN") - if token: - login(token=token, add_to_git_credential=True) - print(" Logged in via HF_TOKEN env var.") - else: - print() - print(" Paste your HF write token below.") - print(" (Get one at: https://huggingface.co/settings/tokens)") - print() - login(add_to_git_credential=True) - - api = HfApi() - - # ── Stage files ──────────────────────────────────────────────────────── - print() - print("Staging project files…") - with tempfile.TemporaryDirectory() as staging: - n = stage_project(REPO_ROOT, staging) - staged_names = os.listdir(staging) - print(f" {n} files staged across {len(staged_names)} top-level items") - - # Sanity checks - has_dockerfile = "Dockerfile" in staged_names - has_readme = "README.md" in staged_names - has_start_sh = os.path.exists(os.path.join(staging, "scripts", "start_hf.sh")) - - print(f" Dockerfile : {'✓' if has_dockerfile else '✗ MISSING — check Dockerfile.hf exists'}") - print(f" README.md : {'✓' if has_readme else '✗ MISSING — check README.hf.md exists'}") - print(f" start_hf.sh : {'✓' if has_start_sh else '✗ MISSING — check scripts/start_hf.sh'}") - - if not has_dockerfile: - print() - print("ERROR: Dockerfile missing from staging. Aborting.") - sys.exit(1) - - # ── Upload ───────────────────────────────────────────────────────── - print() - print(f"Uploading to {REPO_ID}…") - api.upload_folder( - folder_path=staging, - repo_id=REPO_ID, - repo_type=REPO_TYPE, - commit_message="Deploy URAAS — African Research Archival & Analytics System", - ) - - # ── Done ─────────────────────────────────────────────────────────────── - print() - print("═" * 55) - print(" Upload complete! Build starting on HF (~5 min).") - print() - print(" Watch build: https://huggingface.co/spaces/Lordkiki/APA-URAAS") - print(" App URL : https://lordkiki-apa-uraas.hf.space") - print() - print(" ─── Secrets to set in Space Settings → Variables & Secrets ───") - secrets = [ - ("URAAS_ENV", "production"), - ("DASHBOARD_SECRET_KEY", "307790fc5aff3fe1e766303f6b94e2fc28c831582bfba5b34802e2c9cbbac0ce"), - ("ADMIN_USERNAME", "admin"), - ("ADMIN_PASSWORD_HASH", "scrypt:32768:8:1$r2KZFX32rJ2twbfV$16f394a253c2b505a215ff2747f7dafbb098eb0eb8b4e6bb9fb521f0ea38af8ce71f33d2354245d68cc383678257367bf410aa45f835b5e848bff95a746878c8"), - ("VIEWER_USERNAME", "viewer"), - ("VIEWER_PASSWORD_HASH", "scrypt:32768:8:1$M8OjWxX64B38akos$2803ad5b29508c4d69df115579630b2a4cbdf2c8406157598c088d5511a7b3d79f91399035040660705413c63fdc41aa0d020cbd7b45038c8ed51d20092ea609"), - ("SMTP_HOST", "smtp.gmail.com"), - ("SMTP_PORT", "587"), - ("SMTP_USE_TLS", "true"), - ("SMTP_USER", "lawalgiyath200716@gmail.com"), - ("SMTP_PASSWORD", "ufwqbdrecpfrzppn"), - ("SMTP_FROM", "URAAS UNILAG "), - ("DASHBOARD_BASE_URL", "https://lordkiki-apa-uraas.hf.space"), - ("DASHBOARD_CORS_ORIGINS", "https://lordkiki-apa-uraas.hf.space"), - ("ARK_NAAN", "99999"), - ("ARK_SHOULDER", "z1"), - ("OPENALEX_MAILTO", "lawalgiyath200716@gmail.com"), - ("DSPACE_API_URL", "https://api-ir.unilag.edu.ng/server"), - ("DSPACE_USERNAME", ""), - ("DSPACE_PASSWORD", ""), - ] - max_k = max(len(k) for k, _ in secrets) - for k, v in secrets: - print(f" {k:<{max_k}} = {v}") - print() - print(" Admin login : admin / URAAS2024demo") - print(" Viewer login: viewer / view2024") - print("═" * 55) - - -if __name__ == "__main__": - main() +""" +Push URAAS to Hugging Face Spaces — Lordkiki/APA-URAAS + +Usage: + python scripts/push_to_hf.py + +What it does: + 1. Logs you into HF (paste your write token when prompted) + 2. Stages a clean copy of the project: + Dockerfile.hf → Dockerfile (HF Spaces Dockerfile, not the prod one) + README.hf.md → README.md (has the HF Space frontmatter) + 3. Uploads everything to the Space via huggingface_hub.upload_folder + 4. HF triggers an auto-build — app is live in ~5 minutes +""" + +import os +import shutil +import sys +import tempfile + +# ── Config ──────────────────────────────────────────────────────────────────── +REPO_ID = "Lordkiki/APA-URAAS" +REPO_TYPE = "space" +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Directories/files to never push +IGNORE_DIRS = { + ".git", ".claude", "__pycache__", ".pytest_cache", ".mypy_cache", + "storage", "logs", "data", "backups", "node_modules", + ".venv", "venv", "env", +} +IGNORE_FILES = { + ".env", ".env.prod", ".env.prod.example", + "uraas.db", # DB lives on /data in HF, not in image + "Dockerfile", # replaced by Dockerfile.hf + "README.md", # replaced by README.hf.md (has HF frontmatter) + "docker-compose.yml", + "docker-compose.prod.yml", + "docker-compose.replica.yml", + "docker-compose.demo.yml", +} +IGNORE_EXTS = {".pyc", ".pyo", ".pyd"} + + +def should_skip(rel_path: str, is_dir: bool) -> bool: + parts = rel_path.replace("\\", "/").split("/") + name = parts[-1] + if is_dir: + return name in IGNORE_DIRS + return ( + name in IGNORE_FILES + or os.path.splitext(name)[1] in IGNORE_EXTS + ) + + +def stage_project(src: str, dst: str) -> int: + """Copy src → dst with HF-specific renames and exclusions.""" + count = 0 + for root, dirs, files in os.walk(src): + rel_root = os.path.relpath(root, src) + + # Prune excluded directories in-place so os.walk doesn't recurse + dirs[:] = [ + d for d in dirs + if not should_skip( + os.path.join(rel_root, d) if rel_root != "." else d, + is_dir=True, + ) + ] + + for fname in files: + rel = os.path.join(rel_root, fname) if rel_root != "." else fname + if should_skip(rel, is_dir=False): + continue + + src_file = os.path.join(root, fname) + + # HF-specific renames + if fname == "Dockerfile.hf": + dest_rel = os.path.join(os.path.dirname(rel), "Dockerfile") if os.path.dirname(rel) else "Dockerfile" + elif fname == "README.hf.md": + dest_rel = "README.md" + else: + dest_rel = rel + + dst_file = os.path.join(dst, dest_rel) + os.makedirs(os.path.dirname(dst_file), exist_ok=True) + shutil.copy2(src_file, dst_file) + count += 1 + return count + + +def main(): + # ── Ensure huggingface_hub is available ──────────────────────────────── + try: + from huggingface_hub import HfApi, login + except ImportError: + print("Installing huggingface_hub…") + os.system(f"{sys.executable} -m pip install huggingface_hub -q") + from huggingface_hub import HfApi, login # type: ignore + + print() + print("═" * 55) + print(" URAAS → Hugging Face Spaces") + print(f" Space: {REPO_ID}") + print("═" * 55) + + # ── Auth ─────────────────────────────────────────────────────────────── + token = os.getenv("HF_TOKEN") + if token: + login(token=token, add_to_git_credential=True) + print(" Logged in via HF_TOKEN env var.") + else: + print() + print(" Paste your HF write token below.") + print(" (Get one at: https://huggingface.co/settings/tokens)") + print() + login(add_to_git_credential=True) + + api = HfApi() + + # ── Stage files ──────────────────────────────────────────────────────── + print() + print("Staging project files…") + with tempfile.TemporaryDirectory() as staging: + n = stage_project(REPO_ROOT, staging) + staged_names = os.listdir(staging) + print(f" {n} files staged across {len(staged_names)} top-level items") + + # Sanity checks + has_dockerfile = "Dockerfile" in staged_names + has_readme = "README.md" in staged_names + has_start_sh = os.path.exists(os.path.join(staging, "scripts", "start_hf.sh")) + + print(f" Dockerfile : {'✓' if has_dockerfile else '✗ MISSING — check Dockerfile.hf exists'}") + print(f" README.md : {'✓' if has_readme else '✗ MISSING — check README.hf.md exists'}") + print(f" start_hf.sh : {'✓' if has_start_sh else '✗ MISSING — check scripts/start_hf.sh'}") + + if not has_dockerfile: + print() + print("ERROR: Dockerfile missing from staging. Aborting.") + sys.exit(1) + + # ── Upload ───────────────────────────────────────────────────────── + print() + print(f"Uploading to {REPO_ID}…") + api.upload_folder( + folder_path=staging, + repo_id=REPO_ID, + repo_type=REPO_TYPE, + commit_message="Deploy URAAS — African Research Archival & Analytics System", + ) + + # ── Done ─────────────────────────────────────────────────────────────── + print() + print("═" * 55) + print(" Upload complete! Build starting on HF (~5 min).") + print() + print(" Watch build: https://huggingface.co/spaces/Lordkiki/APA-URAAS") + print(" App URL : https://lordkiki-apa-uraas.hf.space") + print() + print(" ─── Secrets to set in Space Settings → Variables & Secrets ───") + secrets = [ + ("URAAS_ENV", "production"), + ("DASHBOARD_SECRET_KEY", "307790fc5aff3fe1e766303f6b94e2fc28c831582bfba5b34802e2c9cbbac0ce"), + ("ADMIN_USERNAME", "admin"), + ("ADMIN_PASSWORD_HASH", "scrypt:32768:8:1$r2KZFX32rJ2twbfV$16f394a253c2b505a215ff2747f7dafbb098eb0eb8b4e6bb9fb521f0ea38af8ce71f33d2354245d68cc383678257367bf410aa45f835b5e848bff95a746878c8"), + ("VIEWER_USERNAME", "viewer"), + ("VIEWER_PASSWORD_HASH", "scrypt:32768:8:1$M8OjWxX64B38akos$2803ad5b29508c4d69df115579630b2a4cbdf2c8406157598c088d5511a7b3d79f91399035040660705413c63fdc41aa0d020cbd7b45038c8ed51d20092ea609"), + ("SMTP_HOST", "smtp.gmail.com"), + ("SMTP_PORT", "587"), + ("SMTP_USE_TLS", "true"), + ("SMTP_USER", "lawalgiyath200716@gmail.com"), + ("SMTP_PASSWORD", "ufwqbdrecpfrzppn"), + ("SMTP_FROM", "URAAS UNILAG "), + ("DASHBOARD_BASE_URL", "https://lordkiki-apa-uraas.hf.space"), + ("DASHBOARD_CORS_ORIGINS", "https://lordkiki-apa-uraas.hf.space"), + ("ARK_NAAN", "99999"), + ("ARK_SHOULDER", "z1"), + ("OPENALEX_MAILTO", "lawalgiyath200716@gmail.com"), + ("DSPACE_API_URL", "https://api-ir.unilag.edu.ng/server"), + ("DSPACE_USERNAME", ""), + ("DSPACE_PASSWORD", ""), + ] + max_k = max(len(k) for k, _ in secrets) + for k, v in secrets: + print(f" {k:<{max_k}} = {v}") + print() + print(" Admin login : admin / URAAS2024demo") + print(" Viewer login: viewer / view2024") + print("═" * 55) + + +if __name__ == "__main__": + main() diff --git a/scripts/reclassify_and_prune_sc.py b/scripts/reclassify_and_prune_sc.py index cdf1bb640e67fe66a59fd05ad7ef720e5a016999..7ea2f976e1f76c663c7e5354aa6fcae40badbfe9 100644 --- a/scripts/reclassify_and_prune_sc.py +++ b/scripts/reclassify_and_prune_sc.py @@ -1,199 +1,199 @@ -""" -Re-classify every Item with the Special Collections decision engine and prune -everything that is not a genuine special collection. - -The platform is Special-Collections-only: papers that the engine scores 0 are -research noise (STEM/medical/jargon) and must be removed. - -Usage: - python scripts/reclassify_and_prune_sc.py # DRY RUN (default) — no writes - python scripts/reclassify_and_prune_sc.py --apply # re-score + delete non-SC - -The --apply pass: - 1. Backs up uraas.db -> uraas.db.bak (SQLite only). - 2. Re-scores all items, writing special_collection_score / _categories. - 3. Deletes items with score == 0 (ORM delete so association/file rows cascade), - then removes orphan authors / empty collections / empty communities. - 4. Flushes the analytics cache. - -Run with the dashboard and any crawler STOPPED to avoid SQLite write locks. -""" - -import argparse -import os -import shutil -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from sqlalchemy import text - -from uraas.config import config -from uraas.database import ( - Author, - Collection, - Community, - Item, - SessionLocal, - engine, - item_authors, -) -from uraas.services.sc_engine import is_special_collection -from uraas.utils.analytics_cache import analytics_cache - - -def backup_sqlite(): - url = (config.DATABASE_URL or "").lower() - if not url.startswith("sqlite"): - print(f"[backup] Non-SQLite DB ({url[:30]}...) — skipping file backup.") - return - db_path = config.DATABASE_URL.split("///")[-1] - if not os.path.exists(db_path): - print(f"[backup] DB file not found at {db_path}; nothing to back up.") - return - bak = db_path + ".bak" - shutil.copy2(db_path, bak) - print(f"[backup] {db_path} -> {bak}") - - -def rescore(session, apply: bool): - """Re-score every item. Returns (keep_ids, drop_ids).""" - items = session.query(Item).all() - keep_ids, drop_ids = [], [] - for it in items: - is_sc, score, cats = is_special_collection( - it.title or "", it.abstract or "", it.dc_subject or "" - ) - if apply: - it.special_collection_score = float(score) - it.special_collection_categories = ",".join(cats) if is_sc else "" - (keep_ids if is_sc else drop_ids).append(it.id) - if apply: - session.commit() - return keep_ids, drop_ids - - -def prune(session, drop_ids): - """Delete non-SC items + orphan authors/collections/communities.""" - # Enforce FK cascade for this SQLite connection (default is OFF). - session.execute(text("PRAGMA foreign_keys=ON")) - - deleted = 0 - for chunk_start in range(0, len(drop_ids), 500): - chunk = drop_ids[chunk_start : chunk_start + 500] - for it in session.query(Item).filter(Item.id.in_(chunk)).all(): - session.delete(it) # ORM delete -> association + file rows cascade - deleted += 1 - session.commit() - print(f"[prune] deleted {deleted} non-SC items") - - # Sweep stray association rows that referenced deleted items (SQLite FK - # cascade is unreliable for raw association tables across chunked deletes). - session.execute( - text( - "DELETE FROM item_authors WHERE item_id NOT IN (SELECT id FROM items) " - "OR author_id NOT IN (SELECT id FROM authors)" - ) - ) - session.execute( - text( - "DELETE FROM item_collections WHERE item_id NOT IN (SELECT id FROM items) " - "OR collection_id NOT IN (SELECT id FROM collections)" - ) - ) - session.commit() - - # Orphan authors: no remaining item associations. - orphan_authors = ( - session.query(Author) - .filter(~Author.id.in_(session.query(item_authors.c.author_id))) - .all() - ) - for a in orphan_authors: - session.delete(a) - print(f"[prune] deleted {len(orphan_authors)} orphan authors") - - # Empty collections (no items) and then empty communities (no collections). - empty_colls = [c for c in session.query(Collection).all() if not c.items] - for c in empty_colls: - session.delete(c) - session.commit() - print(f"[prune] deleted {len(empty_colls)} empty collections") - - empty_comms = [c for c in session.query(Community).all() if not c.collections] - for c in empty_comms: - session.delete(c) - session.commit() - print(f"[prune] deleted {len(empty_comms)} empty communities") - - -def main(): - parser = argparse.ArgumentParser(description="Re-classify & prune non-SC papers") - parser.add_argument( - "--apply", action="store_true", help="Actually re-score and delete (default: dry run)" - ) - parser.add_argument( - "--samples", type=int, default=20, help="How many borderline drops to print" - ) - args = parser.parse_args() - - session = SessionLocal() - try: - total = session.query(Item).count() - old_sc = session.query(Item).filter(Item.special_collection_score > 0).count() - print("=" * 64) - print(f"Total items: {total} (old score>0: {old_sc})") - print("=" * 64) - - if args.apply: - backup_sqlite() - - keep_ids, drop_ids = rescore(session, apply=args.apply) - print(f"\nKEEP (special collections): {len(keep_ids)}") - print(f"DROP (not special collections): {len(drop_ids)}") - - # Show a sample of what would be / was dropped that previously scored > 0 - # (these are the meaningful changes to eyeball). - prev_sc = { - i for (i,) in session.query(Item.id).filter(Item.special_collection_score >= 0).all() - } if not args.apply else set() - sample = ( - session.query(Item.title) - .filter(Item.id.in_(drop_ids[: args.samples])) - .all() - ) - print(f"\n--- sample of dropped titles (first {args.samples}) ---") - for (t,) in sample: - safe = (t or "").encode("ascii", "replace").decode() - print(" DROP:", safe[:90]) - - if not args.apply: - print("\n[DRY RUN] No changes written. Re-run with --apply to prune.") - return 0 - - prune(session, drop_ids) - analytics_cache.invalidate_all() - - remaining = session.query(Item).count() - sc_remaining = ( - session.query(Item).filter(Item.special_collection_score > 0).count() - ) - orphan_left = ( - session.query(Author) - .filter(~Author.id.in_(session.query(item_authors.c.author_id))) - .count() - ) - print("\n" + "=" * 64) - print(f"DONE. Items remaining: {remaining} (score>0: {sc_remaining})") - print(f"Orphan authors remaining: {orphan_left}") - assert remaining == sc_remaining, "Mismatch: non-SC rows survived!" - assert orphan_left == 0, "Orphan authors survived!" - print("Invariants OK. Restart the dashboard to serve fresh data.") - print("=" * 64) - return 0 - finally: - session.close() - - -if __name__ == "__main__": - sys.exit(main()) +""" +Re-classify every Item with the Special Collections decision engine and prune +everything that is not a genuine special collection. + +The platform is Special-Collections-only: papers that the engine scores 0 are +research noise (STEM/medical/jargon) and must be removed. + +Usage: + python scripts/reclassify_and_prune_sc.py # DRY RUN (default) — no writes + python scripts/reclassify_and_prune_sc.py --apply # re-score + delete non-SC + +The --apply pass: + 1. Backs up uraas.db -> uraas.db.bak (SQLite only). + 2. Re-scores all items, writing special_collection_score / _categories. + 3. Deletes items with score == 0 (ORM delete so association/file rows cascade), + then removes orphan authors / empty collections / empty communities. + 4. Flushes the analytics cache. + +Run with the dashboard and any crawler STOPPED to avoid SQLite write locks. +""" + +import argparse +import os +import shutil +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from sqlalchemy import text + +from uraas.config import config +from uraas.database import ( + Author, + Collection, + Community, + Item, + SessionLocal, + engine, + item_authors, +) +from uraas.services.sc_engine import is_special_collection +from uraas.utils.analytics_cache import analytics_cache + + +def backup_sqlite(): + url = (config.DATABASE_URL or "").lower() + if not url.startswith("sqlite"): + print(f"[backup] Non-SQLite DB ({url[:30]}...) — skipping file backup.") + return + db_path = config.DATABASE_URL.split("///")[-1] + if not os.path.exists(db_path): + print(f"[backup] DB file not found at {db_path}; nothing to back up.") + return + bak = db_path + ".bak" + shutil.copy2(db_path, bak) + print(f"[backup] {db_path} -> {bak}") + + +def rescore(session, apply: bool): + """Re-score every item. Returns (keep_ids, drop_ids).""" + items = session.query(Item).all() + keep_ids, drop_ids = [], [] + for it in items: + is_sc, score, cats = is_special_collection( + it.title or "", it.abstract or "", it.dc_subject or "" + ) + if apply: + it.special_collection_score = float(score) + it.special_collection_categories = ",".join(cats) if is_sc else "" + (keep_ids if is_sc else drop_ids).append(it.id) + if apply: + session.commit() + return keep_ids, drop_ids + + +def prune(session, drop_ids): + """Delete non-SC items + orphan authors/collections/communities.""" + # Enforce FK cascade for this SQLite connection (default is OFF). + session.execute(text("PRAGMA foreign_keys=ON")) + + deleted = 0 + for chunk_start in range(0, len(drop_ids), 500): + chunk = drop_ids[chunk_start : chunk_start + 500] + for it in session.query(Item).filter(Item.id.in_(chunk)).all(): + session.delete(it) # ORM delete -> association + file rows cascade + deleted += 1 + session.commit() + print(f"[prune] deleted {deleted} non-SC items") + + # Sweep stray association rows that referenced deleted items (SQLite FK + # cascade is unreliable for raw association tables across chunked deletes). + session.execute( + text( + "DELETE FROM item_authors WHERE item_id NOT IN (SELECT id FROM items) " + "OR author_id NOT IN (SELECT id FROM authors)" + ) + ) + session.execute( + text( + "DELETE FROM item_collections WHERE item_id NOT IN (SELECT id FROM items) " + "OR collection_id NOT IN (SELECT id FROM collections)" + ) + ) + session.commit() + + # Orphan authors: no remaining item associations. + orphan_authors = ( + session.query(Author) + .filter(~Author.id.in_(session.query(item_authors.c.author_id))) + .all() + ) + for a in orphan_authors: + session.delete(a) + print(f"[prune] deleted {len(orphan_authors)} orphan authors") + + # Empty collections (no items) and then empty communities (no collections). + empty_colls = [c for c in session.query(Collection).all() if not c.items] + for c in empty_colls: + session.delete(c) + session.commit() + print(f"[prune] deleted {len(empty_colls)} empty collections") + + empty_comms = [c for c in session.query(Community).all() if not c.collections] + for c in empty_comms: + session.delete(c) + session.commit() + print(f"[prune] deleted {len(empty_comms)} empty communities") + + +def main(): + parser = argparse.ArgumentParser(description="Re-classify & prune non-SC papers") + parser.add_argument( + "--apply", action="store_true", help="Actually re-score and delete (default: dry run)" + ) + parser.add_argument( + "--samples", type=int, default=20, help="How many borderline drops to print" + ) + args = parser.parse_args() + + session = SessionLocal() + try: + total = session.query(Item).count() + old_sc = session.query(Item).filter(Item.special_collection_score > 0).count() + print("=" * 64) + print(f"Total items: {total} (old score>0: {old_sc})") + print("=" * 64) + + if args.apply: + backup_sqlite() + + keep_ids, drop_ids = rescore(session, apply=args.apply) + print(f"\nKEEP (special collections): {len(keep_ids)}") + print(f"DROP (not special collections): {len(drop_ids)}") + + # Show a sample of what would be / was dropped that previously scored > 0 + # (these are the meaningful changes to eyeball). + prev_sc = { + i for (i,) in session.query(Item.id).filter(Item.special_collection_score >= 0).all() + } if not args.apply else set() + sample = ( + session.query(Item.title) + .filter(Item.id.in_(drop_ids[: args.samples])) + .all() + ) + print(f"\n--- sample of dropped titles (first {args.samples}) ---") + for (t,) in sample: + safe = (t or "").encode("ascii", "replace").decode() + print(" DROP:", safe[:90]) + + if not args.apply: + print("\n[DRY RUN] No changes written. Re-run with --apply to prune.") + return 0 + + prune(session, drop_ids) + analytics_cache.invalidate_all() + + remaining = session.query(Item).count() + sc_remaining = ( + session.query(Item).filter(Item.special_collection_score > 0).count() + ) + orphan_left = ( + session.query(Author) + .filter(~Author.id.in_(session.query(item_authors.c.author_id))) + .count() + ) + print("\n" + "=" * 64) + print(f"DONE. Items remaining: {remaining} (score>0: {sc_remaining})") + print(f"Orphan authors remaining: {orphan_left}") + assert remaining == sc_remaining, "Mismatch: non-SC rows survived!" + assert orphan_left == 0, "Orphan authors survived!" + print("Invariants OK. Restart the dashboard to serve fresh data.") + print("=" * 64) + return 0 + finally: + session.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/scrape_nigerian_universities.py b/scripts/scrape_nigerian_universities.py index 091df7cccfd1f69d4bc7144f9996de7b6e0fb628..de4d53f371f76f7dc2dfe5fab3cb00be49e459ec 100644 --- a/scripts/scrape_nigerian_universities.py +++ b/scripts/scrape_nigerian_universities.py @@ -1,618 +1,618 @@ -""" -Comprehensive scraper for Nigerian university faculty directories -Collects full staff names with high accuracy -""" - -import json -import re -import time -from typing import Dict, List, Set -from urllib.parse import urljoin, urlparse - -import requests -from bs4 import BeautifulSoup - - -class UniversityStaffScraper: - """Base class for university staff scraping""" - - def __init__(self, institution_name: str, base_url: str): - self.institution_name = institution_name - self.base_url = base_url - self.session = requests.Session() - self.session.headers.update( - { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" - } - ) - self.staff_data = [] - self.staff_names = set() - - def clean_name(self, name: str) -> str: - """Clean and standardize name""" - if not name: - return "" - - # Remove extra whitespace - name = re.sub(r"\s+", " ", name).strip() - - # Remove common artifacts - name = re.sub(r"\s*\([^)]*\)\s*", " ", name) # Remove parentheses content - name = re.sub(r"\s*\[[^\]]*\]\s*", " ", name) # Remove brackets content - name = re.sub(r"\s+", " ", name).strip() - - # Ensure proper capitalization - if name.isupper() or name.islower(): - name = name.title() - - return name - - def is_valid_name(self, name: str) -> bool: - """Validate if string is a proper name""" - if not name or len(name) < 5: - return False - - # Must have at least 2 words - words = name.split() - if len(words) < 2: - return False - - # Must contain letters - if not re.search(r"[a-zA-Z]", name): - return False - - # Reject if too many numbers - if len(re.findall(r"\d", name)) > 3: - return False - - # Reject common non-name patterns - reject_patterns = [ - r"^(page|home|about|contact|staff|faculty|department)", - r"(\.pdf|\.doc|\.jpg|\.png)$", - r"^(dr|prof|mr|mrs|ms)\.?$", - r"^\d+$", - ] - - for pattern in reject_patterns: - if re.search(pattern, name.lower()): - return False - - return True - - def save_to_json(self, filename: str): - """Save collected staff data to JSON""" - output = { - "institution": self.institution_name, - "total_staff": len(self.staff_names), - "collection_date": time.strftime("%Y-%m-%d"), - "staff": sorted(list(self.staff_names)), - "detailed_records": self.staff_data, - } - - with open(filename, "w", encoding="utf-8") as f: - json.dump(output, f, indent=2, ensure_ascii=False) - - print(f"\n✓ Saved {len(self.staff_names)} staff members to {filename}") - - def scrape(self): - """Override in subclass""" - raise NotImplementedError - - -class UIStaffScraper(UniversityStaffScraper): - """University of Ibadan staff scraper""" - - def __init__(self): - super().__init__("University of Ibadan", "https://www.ui.edu.ng") - - def scrape(self): - """Scrape UI faculty directory""" - print(f"\n{'='*60}") - print(f"Scraping: {self.institution_name}") - print(f"{'='*60}") - - # UI faculty pages - faculty_urls = [ - "/faculties/arts", - "/faculties/science", - "/faculties/technology", - "/faculties/agriculture-and-forestry", - "/faculties/veterinary-medicine", - "/faculties/medicine", - "/faculties/dentistry", - "/faculties/pharmacy", - "/faculties/public-health", - "/faculties/social-sciences", - "/faculties/law", - "/faculties/education", - "/faculties/environmental-design-and-management", - ] - - # Try to scrape from staff directory if available - try: - response = self.session.get(f"{self.base_url}/staff-directory", timeout=10) - if response.status_code == 200: - self._parse_staff_page(response.text, "Staff Directory") - except Exception as e: - print(f" Note: Staff directory not accessible: {e}") - - # Try faculty pages - for faculty_url in faculty_urls: - try: - url = urljoin(self.base_url, faculty_url) - print(f" Checking: {url}") - response = self.session.get(url, timeout=10) - - if response.status_code == 200: - self._parse_staff_page(response.text, faculty_url) - time.sleep(1) # Be polite - - except Exception as e: - print(f" Error accessing {faculty_url}: {e}") - - print(f"\n Total staff collected: {len(self.staff_names)}") - - def _parse_staff_page(self, html: str, source: str): - """Parse HTML page for staff names""" - soup = BeautifulSoup(html, "html.parser") - - # Look for common patterns - patterns = [ - ("div", {"class": re.compile(r"staff|faculty|member|person")}), - ("li", {"class": re.compile(r"staff|faculty|member")}), - ("h3", {}), - ("h4", {}), - ("p", {"class": re.compile(r"name|staff")}), - ] - - for tag, attrs in patterns: - elements = soup.find_all(tag, attrs) - for elem in elements: - text = elem.get_text(strip=True) - name = self.clean_name(text) - - if self.is_valid_name(name) and name not in self.staff_names: - self.staff_names.add(name) - self.staff_data.append( - {"name": name, "source": source, "faculty": "Unknown"} - ) - - -class OAUStaffScraper(UniversityStaffScraper): - """Obafemi Awolowo University staff scraper""" - - def __init__(self): - super().__init__("Obafemi Awolowo University", "https://oauife.edu.ng") - - def scrape(self): - """Scrape OAU faculty directory""" - print(f"\n{'='*60}") - print(f"Scraping: {self.institution_name}") - print(f"{'='*60}") - - # OAU faculty pages - faculty_urls = [ - "/faculties/arts", - "/faculties/science", - "/faculties/technology", - "/faculties/agriculture", - "/faculties/basic-medical-sciences", - "/faculties/clinical-sciences", - "/faculties/dentistry", - "/faculties/pharmacy", - "/faculties/social-sciences", - "/faculties/law", - "/faculties/education", - "/faculties/environmental-design", - ] - - # Try staff directory - try: - response = self.session.get(f"{self.base_url}/staff", timeout=10) - if response.status_code == 200: - self._parse_staff_page(response.text, "Staff Directory") - except Exception as e: - print(f" Note: Staff directory not accessible: {e}") - - # Try faculty pages - for faculty_url in faculty_urls: - try: - url = urljoin(self.base_url, faculty_url) - print(f" Checking: {url}") - response = self.session.get(url, timeout=10) - - if response.status_code == 200: - self._parse_staff_page(response.text, faculty_url) - time.sleep(1) - - except Exception as e: - print(f" Error accessing {faculty_url}: {e}") - - print(f"\n Total staff collected: {len(self.staff_names)}") - - def _parse_staff_page(self, html: str, source: str): - """Parse HTML page for staff names""" - soup = BeautifulSoup(html, "html.parser") - - # Look for staff names - patterns = [ - ("div", {"class": re.compile(r"staff|faculty|member|person")}), - ("li", {"class": re.compile(r"staff|faculty|member")}), - ("h3", {}), - ("h4", {}), - ("span", {"class": re.compile(r"name")}), - ] - - for tag, attrs in patterns: - elements = soup.find_all(tag, attrs) - for elem in elements: - text = elem.get_text(strip=True) - name = self.clean_name(text) - - if self.is_valid_name(name) and name not in self.staff_names: - self.staff_names.add(name) - self.staff_data.append( - {"name": name, "source": source, "faculty": "Unknown"} - ) - - -class UNNStaffScraper(UniversityStaffScraper): - """University of Nigeria, Nsukka staff scraper""" - - def __init__(self): - super().__init__("University of Nigeria, Nsukka", "https://www.unn.edu.ng") - - def scrape(self): - """Scrape UNN faculty directory""" - print(f"\n{'='*60}") - print(f"Scraping: {self.institution_name}") - print(f"{'='*60}") - - # UNN faculty pages - faculty_urls = [ - "/faculties/arts", - "/faculties/biological-sciences", - "/faculties/physical-sciences", - "/faculties/engineering", - "/faculties/agriculture", - "/faculties/veterinary-medicine", - "/faculties/medical-sciences", - "/faculties/dentistry", - "/faculties/pharmaceutical-sciences", - "/faculties/health-sciences", - "/faculties/social-sciences", - "/faculties/law", - "/faculties/education", - "/faculties/environmental-studies", - "/faculties/business-administration", - ] - - # Try staff directory - try: - response = self.session.get(f"{self.base_url}/staff-directory", timeout=10) - if response.status_code == 200: - self._parse_staff_page(response.text, "Staff Directory") - except Exception as e: - print(f" Note: Staff directory not accessible: {e}") - - # Try faculty pages - for faculty_url in faculty_urls: - try: - url = urljoin(self.base_url, faculty_url) - print(f" Checking: {url}") - response = self.session.get(url, timeout=10) - - if response.status_code == 200: - self._parse_staff_page(response.text, faculty_url) - time.sleep(1) - - except Exception as e: - print(f" Error accessing {faculty_url}: {e}") - - print(f"\n Total staff collected: {len(self.staff_names)}") - - def _parse_staff_page(self, html: str, source: str): - """Parse HTML page for staff names""" - soup = BeautifulSoup(html, "html.parser") - - # Look for staff names - patterns = [ - ("div", {"class": re.compile(r"staff|faculty|member|person")}), - ("li", {"class": re.compile(r"staff|faculty|member")}), - ("h3", {}), - ("h4", {}), - ("td", {}), - ] - - for tag, attrs in patterns: - elements = soup.find_all(tag, attrs) - for elem in elements: - text = elem.get_text(strip=True) - name = self.clean_name(text) - - if self.is_valid_name(name) and name not in self.staff_names: - self.staff_names.add(name) - self.staff_data.append( - {"name": name, "source": source, "faculty": "Unknown"} - ) - - -class ABUStaffScraper(UniversityStaffScraper): - """Ahmadu Bello University staff scraper""" - - def __init__(self): - super().__init__("Ahmadu Bello University", "https://www.abu.edu.ng") - - def scrape(self): - """Scrape ABU faculty directory""" - print(f"\n{'='*60}") - print(f"Scraping: {self.institution_name}") - print(f"{'='*60}") - - # ABU faculty pages - faculty_urls = [ - "/faculties/arts-and-islamic-studies", - "/faculties/science", - "/faculties/engineering", - "/faculties/agriculture", - "/faculties/veterinary-medicine", - "/faculties/medicine", - "/faculties/dentistry", - "/faculties/pharmaceutical-sciences", - "/faculties/allied-health-sciences", - "/faculties/social-sciences", - "/faculties/law", - "/faculties/education", - "/faculties/environmental-design", - "/faculties/administration", - ] - - # Try staff directory - try: - response = self.session.get(f"{self.base_url}/staff", timeout=10) - if response.status_code == 200: - self._parse_staff_page(response.text, "Staff Directory") - except Exception as e: - print(f" Note: Staff directory not accessible: {e}") - - # Try faculty pages - for faculty_url in faculty_urls: - try: - url = urljoin(self.base_url, faculty_url) - print(f" Checking: {url}") - response = self.session.get(url, timeout=10) - - if response.status_code == 200: - self._parse_staff_page(response.text, faculty_url) - time.sleep(1) - - except Exception as e: - print(f" Error accessing {faculty_url}: {e}") - - print(f"\n Total staff collected: {len(self.staff_names)}") - - def _parse_staff_page(self, html: str, source: str): - """Parse HTML page for staff names""" - soup = BeautifulSoup(html, "html.parser") - - # Look for staff names - patterns = [ - ("div", {"class": re.compile(r"staff|faculty|member|person")}), - ("li", {"class": re.compile(r"staff|faculty|member")}), - ("h3", {}), - ("h4", {}), - ("span", {"class": re.compile(r"name")}), - ] - - for tag, attrs in patterns: - elements = soup.find_all(tag, attrs) - for elem in elements: - text = elem.get_text(strip=True) - name = self.clean_name(text) - - if self.is_valid_name(name) and name not in self.staff_names: - self.staff_names.add(name) - self.staff_data.append( - {"name": name, "source": source, "faculty": "Unknown"} - ) - - -def generate_sample_names(institution: str, count: int) -> List[str]: - """ - Generate realistic Nigerian academic staff names as fallback - Uses common Nigerian naming patterns - """ - - # Common Nigerian surnames by region - yoruba_surnames = [ - "Adeyemi", - "Ogunlana", - "Oluwaseun", - "Babatunde", - "Adebayo", - "Oladipo", - "Adekunle", - "Olatunji", - "Adewale", - "Olaniyan", - "Afolabi", - "Ogunbiyi", - "Adeyinka", - "Oladele", - "Adebisi", - "Ogunleye", - "Adeola", - "Olayinka", - ] - - igbo_surnames = [ - "Okonkwo", - "Nwosu", - "Okeke", - "Eze", - "Okafor", - "Nwankwo", - "Chukwu", - "Onyeka", - "Ikechukwu", - "Obiora", - "Emeka", - "Chinedu", - "Ugochukwu", - "Nnamdi", - "Chibueze", - "Obinna", - "Kelechi", - "Chukwuemeka", - ] - - hausa_surnames = [ - "Ibrahim", - "Mohammed", - "Abdullahi", - "Usman", - "Ahmad", - "Hassan", - "Aliyu", - "Musa", - "Abubakar", - "Suleiman", - "Yusuf", - "Ismail", - "Bello", - "Garba", - "Sani", - "Umar", - "Tijjani", - "Kabir", - ] - - # Common first names - first_names = [ - "Oluwaseun", - "Chinedu", - "Abubakar", - "Ngozi", - "Fatima", - "Chiamaka", - "Tunde", - "Emeka", - "Musa", - "Adaeze", - "Zainab", - "Chioma", - "Segun", - "Obinna", - "Aliyu", - "Amaka", - "Aisha", - "Ifeoma", - ] - - # Academic titles - titles = ["Prof.", "Dr.", "Mr.", "Mrs.", "Ms."] - - # Middle initials - initials = list("ABCDEFGHIJKLMNOPQRSTUVWXYZ") - - import random - - random.seed(42) # For reproducibility - - all_surnames = yoruba_surnames + igbo_surnames + hausa_surnames - names = set() - - while len(names) < count: - title = random.choice(titles) - first = random.choice(first_names) - middle = random.choice(initials) - surname = random.choice(all_surnames) - - # Various name formats - formats = [ - f"{title} {first} {middle}. {surname}", - f"{title} {first} {surname}", - f"{first} {middle}. {surname}", - f"{surname}, {first} {middle}.", - ] - - name = random.choice(formats) - names.add(name) - - return sorted(list(names)) - - -def main(): - """Main scraping function""" - print("\n" + "=" * 60) - print("NIGERIAN UNIVERSITIES STAFF DATA COLLECTION") - print("=" * 60) - print("\nTarget: Collect full staff names from 4 universities") - print("Quality: Full names only, no abbreviations, no mistakes") - print("=" * 60) - - results = {} - - # Scrape each university - scrapers = [ - (UIStaffScraper(), "data/ui_staff.json"), - (OAUStaffScraper(), "data/oau_staff.json"), - (UNNStaffScraper(), "data/unn_staff.json"), - (ABUStaffScraper(), "data/abu_staff.json"), - ] - - for scraper, filename in scrapers: - try: - scraper.scrape() - - # If scraping didn't yield enough results, generate sample data - if len(scraper.staff_names) < 50: - print(f"\n ⚠ Warning: Only {len(scraper.staff_names)} names collected") - print(f" Generating sample Nigerian academic names for testing...") - - sample_names = generate_sample_names(scraper.institution_name, 300) - scraper.staff_names.update(sample_names) - - for name in sample_names: - scraper.staff_data.append( - { - "name": name, - "source": "Generated Sample", - "faculty": "Unknown", - } - ) - - print(f" ✓ Added {len(sample_names)} sample names") - - scraper.save_to_json(filename) - results[scraper.institution_name] = len(scraper.staff_names) - - except Exception as e: - print(f"\n✗ Error scraping {scraper.institution_name}: {e}") - import traceback - - traceback.print_exc() - - # Summary - print("\n" + "=" * 60) - print("COLLECTION SUMMARY") - print("=" * 60) - - total = 0 - for institution, count in results.items(): - print(f" {institution}: {count} staff members") - total += count - - print(f"\n TOTAL: {total} staff members across 4 universities") - print("=" * 60) - - print("\n✓ Data collection complete!") - print("\nNext steps:") - print(" 1. Review generated JSON files in data/ directory") - print(" 2. Manually verify sample of names") - print(" 3. Run test_multi_institution.py to verify") - print(" 4. Proceed to spider integration (Day 5-7)") - - -if __name__ == "__main__": - main() +""" +Comprehensive scraper for Nigerian university faculty directories +Collects full staff names with high accuracy +""" + +import json +import re +import time +from typing import Dict, List, Set +from urllib.parse import urljoin, urlparse + +import requests +from bs4 import BeautifulSoup + + +class UniversityStaffScraper: + """Base class for university staff scraping""" + + def __init__(self, institution_name: str, base_url: str): + self.institution_name = institution_name + self.base_url = base_url + self.session = requests.Session() + self.session.headers.update( + { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + } + ) + self.staff_data = [] + self.staff_names = set() + + def clean_name(self, name: str) -> str: + """Clean and standardize name""" + if not name: + return "" + + # Remove extra whitespace + name = re.sub(r"\s+", " ", name).strip() + + # Remove common artifacts + name = re.sub(r"\s*\([^)]*\)\s*", " ", name) # Remove parentheses content + name = re.sub(r"\s*\[[^\]]*\]\s*", " ", name) # Remove brackets content + name = re.sub(r"\s+", " ", name).strip() + + # Ensure proper capitalization + if name.isupper() or name.islower(): + name = name.title() + + return name + + def is_valid_name(self, name: str) -> bool: + """Validate if string is a proper name""" + if not name or len(name) < 5: + return False + + # Must have at least 2 words + words = name.split() + if len(words) < 2: + return False + + # Must contain letters + if not re.search(r"[a-zA-Z]", name): + return False + + # Reject if too many numbers + if len(re.findall(r"\d", name)) > 3: + return False + + # Reject common non-name patterns + reject_patterns = [ + r"^(page|home|about|contact|staff|faculty|department)", + r"(\.pdf|\.doc|\.jpg|\.png)$", + r"^(dr|prof|mr|mrs|ms)\.?$", + r"^\d+$", + ] + + for pattern in reject_patterns: + if re.search(pattern, name.lower()): + return False + + return True + + def save_to_json(self, filename: str): + """Save collected staff data to JSON""" + output = { + "institution": self.institution_name, + "total_staff": len(self.staff_names), + "collection_date": time.strftime("%Y-%m-%d"), + "staff": sorted(list(self.staff_names)), + "detailed_records": self.staff_data, + } + + with open(filename, "w", encoding="utf-8") as f: + json.dump(output, f, indent=2, ensure_ascii=False) + + print(f"\n✓ Saved {len(self.staff_names)} staff members to {filename}") + + def scrape(self): + """Override in subclass""" + raise NotImplementedError + + +class UIStaffScraper(UniversityStaffScraper): + """University of Ibadan staff scraper""" + + def __init__(self): + super().__init__("University of Ibadan", "https://www.ui.edu.ng") + + def scrape(self): + """Scrape UI faculty directory""" + print(f"\n{'='*60}") + print(f"Scraping: {self.institution_name}") + print(f"{'='*60}") + + # UI faculty pages + faculty_urls = [ + "/faculties/arts", + "/faculties/science", + "/faculties/technology", + "/faculties/agriculture-and-forestry", + "/faculties/veterinary-medicine", + "/faculties/medicine", + "/faculties/dentistry", + "/faculties/pharmacy", + "/faculties/public-health", + "/faculties/social-sciences", + "/faculties/law", + "/faculties/education", + "/faculties/environmental-design-and-management", + ] + + # Try to scrape from staff directory if available + try: + response = self.session.get(f"{self.base_url}/staff-directory", timeout=10) + if response.status_code == 200: + self._parse_staff_page(response.text, "Staff Directory") + except Exception as e: + print(f" Note: Staff directory not accessible: {e}") + + # Try faculty pages + for faculty_url in faculty_urls: + try: + url = urljoin(self.base_url, faculty_url) + print(f" Checking: {url}") + response = self.session.get(url, timeout=10) + + if response.status_code == 200: + self._parse_staff_page(response.text, faculty_url) + time.sleep(1) # Be polite + + except Exception as e: + print(f" Error accessing {faculty_url}: {e}") + + print(f"\n Total staff collected: {len(self.staff_names)}") + + def _parse_staff_page(self, html: str, source: str): + """Parse HTML page for staff names""" + soup = BeautifulSoup(html, "html.parser") + + # Look for common patterns + patterns = [ + ("div", {"class": re.compile(r"staff|faculty|member|person")}), + ("li", {"class": re.compile(r"staff|faculty|member")}), + ("h3", {}), + ("h4", {}), + ("p", {"class": re.compile(r"name|staff")}), + ] + + for tag, attrs in patterns: + elements = soup.find_all(tag, attrs) + for elem in elements: + text = elem.get_text(strip=True) + name = self.clean_name(text) + + if self.is_valid_name(name) and name not in self.staff_names: + self.staff_names.add(name) + self.staff_data.append( + {"name": name, "source": source, "faculty": "Unknown"} + ) + + +class OAUStaffScraper(UniversityStaffScraper): + """Obafemi Awolowo University staff scraper""" + + def __init__(self): + super().__init__("Obafemi Awolowo University", "https://oauife.edu.ng") + + def scrape(self): + """Scrape OAU faculty directory""" + print(f"\n{'='*60}") + print(f"Scraping: {self.institution_name}") + print(f"{'='*60}") + + # OAU faculty pages + faculty_urls = [ + "/faculties/arts", + "/faculties/science", + "/faculties/technology", + "/faculties/agriculture", + "/faculties/basic-medical-sciences", + "/faculties/clinical-sciences", + "/faculties/dentistry", + "/faculties/pharmacy", + "/faculties/social-sciences", + "/faculties/law", + "/faculties/education", + "/faculties/environmental-design", + ] + + # Try staff directory + try: + response = self.session.get(f"{self.base_url}/staff", timeout=10) + if response.status_code == 200: + self._parse_staff_page(response.text, "Staff Directory") + except Exception as e: + print(f" Note: Staff directory not accessible: {e}") + + # Try faculty pages + for faculty_url in faculty_urls: + try: + url = urljoin(self.base_url, faculty_url) + print(f" Checking: {url}") + response = self.session.get(url, timeout=10) + + if response.status_code == 200: + self._parse_staff_page(response.text, faculty_url) + time.sleep(1) + + except Exception as e: + print(f" Error accessing {faculty_url}: {e}") + + print(f"\n Total staff collected: {len(self.staff_names)}") + + def _parse_staff_page(self, html: str, source: str): + """Parse HTML page for staff names""" + soup = BeautifulSoup(html, "html.parser") + + # Look for staff names + patterns = [ + ("div", {"class": re.compile(r"staff|faculty|member|person")}), + ("li", {"class": re.compile(r"staff|faculty|member")}), + ("h3", {}), + ("h4", {}), + ("span", {"class": re.compile(r"name")}), + ] + + for tag, attrs in patterns: + elements = soup.find_all(tag, attrs) + for elem in elements: + text = elem.get_text(strip=True) + name = self.clean_name(text) + + if self.is_valid_name(name) and name not in self.staff_names: + self.staff_names.add(name) + self.staff_data.append( + {"name": name, "source": source, "faculty": "Unknown"} + ) + + +class UNNStaffScraper(UniversityStaffScraper): + """University of Nigeria, Nsukka staff scraper""" + + def __init__(self): + super().__init__("University of Nigeria, Nsukka", "https://www.unn.edu.ng") + + def scrape(self): + """Scrape UNN faculty directory""" + print(f"\n{'='*60}") + print(f"Scraping: {self.institution_name}") + print(f"{'='*60}") + + # UNN faculty pages + faculty_urls = [ + "/faculties/arts", + "/faculties/biological-sciences", + "/faculties/physical-sciences", + "/faculties/engineering", + "/faculties/agriculture", + "/faculties/veterinary-medicine", + "/faculties/medical-sciences", + "/faculties/dentistry", + "/faculties/pharmaceutical-sciences", + "/faculties/health-sciences", + "/faculties/social-sciences", + "/faculties/law", + "/faculties/education", + "/faculties/environmental-studies", + "/faculties/business-administration", + ] + + # Try staff directory + try: + response = self.session.get(f"{self.base_url}/staff-directory", timeout=10) + if response.status_code == 200: + self._parse_staff_page(response.text, "Staff Directory") + except Exception as e: + print(f" Note: Staff directory not accessible: {e}") + + # Try faculty pages + for faculty_url in faculty_urls: + try: + url = urljoin(self.base_url, faculty_url) + print(f" Checking: {url}") + response = self.session.get(url, timeout=10) + + if response.status_code == 200: + self._parse_staff_page(response.text, faculty_url) + time.sleep(1) + + except Exception as e: + print(f" Error accessing {faculty_url}: {e}") + + print(f"\n Total staff collected: {len(self.staff_names)}") + + def _parse_staff_page(self, html: str, source: str): + """Parse HTML page for staff names""" + soup = BeautifulSoup(html, "html.parser") + + # Look for staff names + patterns = [ + ("div", {"class": re.compile(r"staff|faculty|member|person")}), + ("li", {"class": re.compile(r"staff|faculty|member")}), + ("h3", {}), + ("h4", {}), + ("td", {}), + ] + + for tag, attrs in patterns: + elements = soup.find_all(tag, attrs) + for elem in elements: + text = elem.get_text(strip=True) + name = self.clean_name(text) + + if self.is_valid_name(name) and name not in self.staff_names: + self.staff_names.add(name) + self.staff_data.append( + {"name": name, "source": source, "faculty": "Unknown"} + ) + + +class ABUStaffScraper(UniversityStaffScraper): + """Ahmadu Bello University staff scraper""" + + def __init__(self): + super().__init__("Ahmadu Bello University", "https://www.abu.edu.ng") + + def scrape(self): + """Scrape ABU faculty directory""" + print(f"\n{'='*60}") + print(f"Scraping: {self.institution_name}") + print(f"{'='*60}") + + # ABU faculty pages + faculty_urls = [ + "/faculties/arts-and-islamic-studies", + "/faculties/science", + "/faculties/engineering", + "/faculties/agriculture", + "/faculties/veterinary-medicine", + "/faculties/medicine", + "/faculties/dentistry", + "/faculties/pharmaceutical-sciences", + "/faculties/allied-health-sciences", + "/faculties/social-sciences", + "/faculties/law", + "/faculties/education", + "/faculties/environmental-design", + "/faculties/administration", + ] + + # Try staff directory + try: + response = self.session.get(f"{self.base_url}/staff", timeout=10) + if response.status_code == 200: + self._parse_staff_page(response.text, "Staff Directory") + except Exception as e: + print(f" Note: Staff directory not accessible: {e}") + + # Try faculty pages + for faculty_url in faculty_urls: + try: + url = urljoin(self.base_url, faculty_url) + print(f" Checking: {url}") + response = self.session.get(url, timeout=10) + + if response.status_code == 200: + self._parse_staff_page(response.text, faculty_url) + time.sleep(1) + + except Exception as e: + print(f" Error accessing {faculty_url}: {e}") + + print(f"\n Total staff collected: {len(self.staff_names)}") + + def _parse_staff_page(self, html: str, source: str): + """Parse HTML page for staff names""" + soup = BeautifulSoup(html, "html.parser") + + # Look for staff names + patterns = [ + ("div", {"class": re.compile(r"staff|faculty|member|person")}), + ("li", {"class": re.compile(r"staff|faculty|member")}), + ("h3", {}), + ("h4", {}), + ("span", {"class": re.compile(r"name")}), + ] + + for tag, attrs in patterns: + elements = soup.find_all(tag, attrs) + for elem in elements: + text = elem.get_text(strip=True) + name = self.clean_name(text) + + if self.is_valid_name(name) and name not in self.staff_names: + self.staff_names.add(name) + self.staff_data.append( + {"name": name, "source": source, "faculty": "Unknown"} + ) + + +def generate_sample_names(institution: str, count: int) -> List[str]: + """ + Generate realistic Nigerian academic staff names as fallback + Uses common Nigerian naming patterns + """ + + # Common Nigerian surnames by region + yoruba_surnames = [ + "Adeyemi", + "Ogunlana", + "Oluwaseun", + "Babatunde", + "Adebayo", + "Oladipo", + "Adekunle", + "Olatunji", + "Adewale", + "Olaniyan", + "Afolabi", + "Ogunbiyi", + "Adeyinka", + "Oladele", + "Adebisi", + "Ogunleye", + "Adeola", + "Olayinka", + ] + + igbo_surnames = [ + "Okonkwo", + "Nwosu", + "Okeke", + "Eze", + "Okafor", + "Nwankwo", + "Chukwu", + "Onyeka", + "Ikechukwu", + "Obiora", + "Emeka", + "Chinedu", + "Ugochukwu", + "Nnamdi", + "Chibueze", + "Obinna", + "Kelechi", + "Chukwuemeka", + ] + + hausa_surnames = [ + "Ibrahim", + "Mohammed", + "Abdullahi", + "Usman", + "Ahmad", + "Hassan", + "Aliyu", + "Musa", + "Abubakar", + "Suleiman", + "Yusuf", + "Ismail", + "Bello", + "Garba", + "Sani", + "Umar", + "Tijjani", + "Kabir", + ] + + # Common first names + first_names = [ + "Oluwaseun", + "Chinedu", + "Abubakar", + "Ngozi", + "Fatima", + "Chiamaka", + "Tunde", + "Emeka", + "Musa", + "Adaeze", + "Zainab", + "Chioma", + "Segun", + "Obinna", + "Aliyu", + "Amaka", + "Aisha", + "Ifeoma", + ] + + # Academic titles + titles = ["Prof.", "Dr.", "Mr.", "Mrs.", "Ms."] + + # Middle initials + initials = list("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + import random + + random.seed(42) # For reproducibility + + all_surnames = yoruba_surnames + igbo_surnames + hausa_surnames + names = set() + + while len(names) < count: + title = random.choice(titles) + first = random.choice(first_names) + middle = random.choice(initials) + surname = random.choice(all_surnames) + + # Various name formats + formats = [ + f"{title} {first} {middle}. {surname}", + f"{title} {first} {surname}", + f"{first} {middle}. {surname}", + f"{surname}, {first} {middle}.", + ] + + name = random.choice(formats) + names.add(name) + + return sorted(list(names)) + + +def main(): + """Main scraping function""" + print("\n" + "=" * 60) + print("NIGERIAN UNIVERSITIES STAFF DATA COLLECTION") + print("=" * 60) + print("\nTarget: Collect full staff names from 4 universities") + print("Quality: Full names only, no abbreviations, no mistakes") + print("=" * 60) + + results = {} + + # Scrape each university + scrapers = [ + (UIStaffScraper(), "data/ui_staff.json"), + (OAUStaffScraper(), "data/oau_staff.json"), + (UNNStaffScraper(), "data/unn_staff.json"), + (ABUStaffScraper(), "data/abu_staff.json"), + ] + + for scraper, filename in scrapers: + try: + scraper.scrape() + + # If scraping didn't yield enough results, generate sample data + if len(scraper.staff_names) < 50: + print(f"\n ⚠ Warning: Only {len(scraper.staff_names)} names collected") + print(f" Generating sample Nigerian academic names for testing...") + + sample_names = generate_sample_names(scraper.institution_name, 300) + scraper.staff_names.update(sample_names) + + for name in sample_names: + scraper.staff_data.append( + { + "name": name, + "source": "Generated Sample", + "faculty": "Unknown", + } + ) + + print(f" ✓ Added {len(sample_names)} sample names") + + scraper.save_to_json(filename) + results[scraper.institution_name] = len(scraper.staff_names) + + except Exception as e: + print(f"\n✗ Error scraping {scraper.institution_name}: {e}") + import traceback + + traceback.print_exc() + + # Summary + print("\n" + "=" * 60) + print("COLLECTION SUMMARY") + print("=" * 60) + + total = 0 + for institution, count in results.items(): + print(f" {institution}: {count} staff members") + total += count + + print(f"\n TOTAL: {total} staff members across 4 universities") + print("=" * 60) + + print("\n✓ Data collection complete!") + print("\nNext steps:") + print(" 1. Review generated JSON files in data/ directory") + print(" 2. Manually verify sample of names") + print(" 3. Run test_multi_institution.py to verify") + print(" 4. Proceed to spider integration (Day 5-7)") + + +if __name__ == "__main__": + main() diff --git a/scripts/seed_demo_db.py b/scripts/seed_demo_db.py index e803439a532ae8e82560d31657d4ac2bfff77e41..9e3702932c8da7f110a06e3da581fe009aa1ea04 100644 --- a/scripts/seed_demo_db.py +++ b/scripts/seed_demo_db.py @@ -1,252 +1,252 @@ -""" -Seed a demo SQLite database with enough data for a compelling live demo. - -Run this ONCE on your local machine before deploying to HF Spaces: - python scripts/seed_demo_db.py - -This creates/populates uraas.db with: - - 30 realistic SC papers (from a cached harvest) - - ARK identifiers for each - - Author + collection associations - -The resulting uraas.db is then bundled into the Docker image (Dockerfile.hf -copies it in), so HF Spaces always starts with data even after a restart. -""" - -import os -import sys -from datetime import datetime, timedelta -import random - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from uraas.database import Author, Base, Collection, Community, Item, engine, SessionLocal -from uraas.utils.ark_generator import ark_generator - -DEMO_PAPERS = [ - { - "title": "Yoruba Oral Traditions and the Digital Archive: Preservation Challenges at the University of Lagos", - "abstract": "This paper examines the intersection of Yoruba oral traditions and digital preservation strategies. We document 847 oral narratives collected from the Lagos metropolitan area and propose a culturally sensitive framework for their archival representation.", - "authors": ["Adeyemi, O.A.", "Fashola, B.K.", "Okonkwo, C."], - "doi": "10.1234/uraas.2023.001", - "source": "AJOL", - "year": "2023", - "sc_score": 3.2, - "sc_cats": "indigenous_knowledge,oral_tradition,african_literature", - }, - { - "title": "Ethnobotanical Survey of Medicinal Plants Used by Traditional Healers in Lagos State", - "abstract": "A systematic ethnobotanical survey of 127 medicinal plant species used by traditional Yoruba healers in Lagos State. Interviews conducted with 89 traditional medical practitioners across 12 local government areas document indigenous pharmacological knowledge at risk of extinction.", - "authors": ["Okafor, N.N.", "Adewale, P.O."], - "doi": "10.1234/uraas.2023.002", - "source": "PubMed", - "year": "2023", - "sc_score": 2.8, - "sc_cats": "indigenous_knowledge,african_literature", - }, - { - "title": "Decolonising the Nigerian University Curriculum: A Case for Indigenous Epistemologies", - "abstract": "Critical examination of colonial legacies in Nigerian higher education curricula. Drawing on Ubuntu philosophy and Afrocentric scholarship, we propose a framework for recentring African knowledge systems within university pedagogy.", - "authors": ["Nwosu, E.C.", "Bamgbose, A.L.", "Eze, F.K."], - "doi": "10.1234/uraas.2023.003", - "source": "OpenAlex", - "year": "2022", - "sc_score": 2.5, - "sc_cats": "african_literature,postcolonial_studies", - }, - { - "title": "Cultural Heritage Documentation in Post-Colonial Nigeria: The Lagos Museum Collections", - "abstract": "Systematic documentation methodology for 3,400 artefacts in the Lagos State Museum. This work establishes provenance records, cultural context narratives, and digital metadata standards aligned with Dublin Core and the CIDOC-CRM ontology.", - "authors": ["Adewale, S.O.", "Obi, T.N."], - "doi": "10.1234/uraas.2023.004", - "source": "DOAJ", - "year": "2023", - "sc_score": 2.9, - "sc_cats": "cultural_heritage,indigenous_knowledge", - }, - { - "title": "Persistent Identifiers for African Institutional Repositories: The ARK Alliance Partnership", - "abstract": "Analysis of PID adoption patterns across 47 African institutional repositories. The Africa PID Alliance's partnership with the ARK Alliance (2025) provides a no-fee persistent identifier infrastructure appropriate for under-resourced institutions.", - "authors": ["Lawal, G.A.", "Ifeanyi, C.O."], - "doi": "10.1234/uraas.2024.001", - "source": "OpenAlex", - "year": "2024", - "sc_score": 1.8, - "sc_cats": "indigenous_knowledge", - }, - { - "title": "Igbo Proverb Literature and Collective Memory: A Computational Analysis", - "abstract": "Using NLP techniques, we analyse a corpus of 12,000 Igbo proverbs collected from 1952–2020. Semantic clustering reveals seven dominant thematic domains, and temporal analysis shows accelerating loss of proverbial usage in urban Igbo communities.", - "authors": ["Okonkwo, C.F.", "Nwosu, P.E.", "Adeyemi, R.A."], - "doi": "10.1234/uraas.2022.001", - "source": "Semantic Scholar", - "year": "2022", - "sc_score": 3.1, - "sc_cats": "oral_tradition,african_literature,indigenous_knowledge", - }, - { - "title": "Traditional Governance Systems and Modern State Formation in South-West Nigeria", - "abstract": "Comparative analysis of Yoruba traditional governance structures (obas, chiefs, age-grade systems) and their integration with post-independence Nigerian state institutions. Case studies from Oyo, Osun, and Lagos states.", - "authors": ["Fashola, K.T.", "Adewale, J.O."], - "doi": "10.1234/uraas.2021.001", - "source": "DOAJ", - "year": "2021", - "sc_score": 2.3, - "sc_cats": "cultural_heritage,indigenous_knowledge", - }, - { - "title": "Lagos Market Women's Oral Histories: Gender, Trade, and Urban Memory", - "abstract": "Oral history methodology applied to 234 interviews with Lagos market women aged 60–95. Documents the transformation of Yoruba women's economic practices from 1940 to present, preserving accounts unavailable in colonial archival records.", - "authors": ["Adeola, F.N.", "Okafor, B.C."], - "doi": "10.1234/uraas.2023.005", - "source": "AJOL", - "year": "2023", - "sc_score": 3.4, - "sc_cats": "oral_tradition,cultural_heritage,african_literature", - }, - { - "title": "Hausa Manuscript Collections in Northern Nigerian Libraries: A Conservation Survey", - "abstract": "Survey of 156 manuscript collections across 23 libraries in Kano, Sokoto, and Maiduguri. We identify 47,000 Hausa-language manuscripts at immediate conservation risk and propose a digitisation triage protocol.", - "authors": ["Musa, A.B.", "Ibrahim, K.S."], - "doi": "10.1234/uraas.2022.002", - "source": "CORE", - "year": "2022", - "sc_score": 2.7, - "sc_cats": "cultural_heritage,indigenous_knowledge", - }, - { - "title": "Postcolonial African Science Fiction: Imagining Futures Beyond Extractivism", - "abstract": "Literary analysis of 78 African science fiction works published 2010–2023. We argue that Afrofuturist fiction constitutes an emerging mode of indigenous knowledge production, encoding African cosmologies in speculative narrative form.", - "authors": ["Nwosu, C.I.", "Lawal, A.O.", "Eze, K.N."], - "doi": "10.1234/uraas.2023.006", - "source": "OpenAlex", - "year": "2023", - "sc_score": 2.2, - "sc_cats": "african_literature,postcolonial_studies", - }, -] - -# Pad to 30 papers -_extra_titles = [ - ("Ubuntu Philosophy and Collective Well-being in Contemporary African Ethics", "african_literature,indigenous_knowledge"), - ("Traditional Water Management Practices of the Niger Delta Communities", "indigenous_knowledge,cultural_heritage"), - ("Afrobeat as Cultural Heritage: Fela Kuti's Archive at University of Lagos", "cultural_heritage,african_literature"), - ("Endangered Languages of the Benue-Congo Region: A Documentation Framework", "indigenous_knowledge,oral_tradition"), - ("Sacred Groves as Living Cultural Heritage in Yorubaland", "cultural_heritage,indigenous_knowledge"), - ("Knowledge Repatriation: Returning Benin Bronzes and Digital Surrogates", "cultural_heritage,postcolonial_studies"), - ("Decolonising Cartography: Mapping Indigenous Territories in Nigeria", "indigenous_knowledge,postcolonial_studies"), - ("Nollywood and the Commodification of Yoruba Oral Narratives", "african_literature,oral_tradition"), - ("Traditional Ecological Knowledge and Biodiversity in Lagos Wetlands", "indigenous_knowledge,cultural_heritage"), - ("Precolonial Trans-Saharan Trade Networks: New Archaeological Evidence", "cultural_heritage,african_literature"), - ("African Proverbs in Contemporary Diplomatic Discourse", "oral_tradition,indigenous_knowledge"), - ("The Ogboni Society: Sacred Brotherhood and Political Power in Yorubaland", "indigenous_knowledge,cultural_heritage"), - ("Digital Humanities and African Archival Futures", "cultural_heritage,african_literature"), - ("Ancestral Veneration Practices in Urban Yoruba Communities", "indigenous_knowledge,oral_tradition"), - ("Linguistic Rights and African Language Policy in Nigerian Universities", "african_literature,indigenous_knowledge"), - ("Community Archives and the Decolonisation of Memory in West Africa", "cultural_heritage,postcolonial_studies"), - ("Trado-Medical Practitioners and the Nigerian Health System", "indigenous_knowledge,cultural_heritage"), - ("Ifa Divination Corpus: Computational Approaches to Sacred Oral Literature", "oral_tradition,indigenous_knowledge"), - ("Pan-African Student Movements and the Politics of Knowledge Production", "postcolonial_studies,african_literature"), - ("Nok Terracotta Figurines: New Dating Evidence from Northern Nigeria", "cultural_heritage,indigenous_knowledge"), -] - -for i, (title, cats) in enumerate(_extra_titles): - DEMO_PAPERS.append({ - "title": title, - "abstract": f"Research paper examining {title.lower()}. " - "This study contributes to the growing body of African Special Collections scholarship " - "accessible through URAAS at the University of Lagos.", - "authors": [f"Demo Author {chr(65 + i)}", f"Demo Author {chr(66 + i)}"], - "doi": f"10.1234/uraas.demo.{i+1:03d}", - "source": ["OpenAlex", "DOAJ", "AJOL", "Crossref"][i % 4], - "year": str(2019 + (i % 6)), - "sc_score": round(1.5 + (i % 10) * 0.2, 1), - "sc_cats": cats, - }) - - -def seed(): - Base.metadata.create_all(engine) - session = SessionLocal() - try: - if session.query(Item).count() >= 10: - print(f"Database already has {session.query(Item).count()} items — skipping seed.") - return - - print(f"Seeding {len(DEMO_PAPERS)} demo papers...") - - # Create a basic community/collection - community = session.query(Community).filter_by(name="Special Collections").first() - if not community: - community = Community( - name="Special Collections", - dc_title="Special Collections", - dc_description="African literature, indigenous knowledge, and cultural heritage.", - ) - session.add(community) - session.flush() - - collection = session.query(Collection).filter_by(name="Oral Traditions & Indigenous Knowledge").first() - if not collection: - collection = Collection( - name="Oral Traditions & Indigenous Knowledge", - community_id=community.id, - ) - session.add(collection) - session.flush() - - for i, p in enumerate(DEMO_PAPERS): - doi = p["doi"] - existing = session.query(Item).filter_by(doi=doi).first() - if existing: - continue - - pub_date = datetime(int(p["year"]), 1 + (i % 12), 1 + (i % 28)) - item = Item( - title=p["title"], - dc_title=p["title"], - abstract=p["abstract"], - doi=doi, - dc_identifier_doi=doi, - dc_identifier_uri=f"https://doi.org/{doi}", - url=f"https://doi.org/{doi}", - publication_date=pub_date, - dc_date_issued=p["year"], - source_repository=p["source"], - institution="University of Lagos", - ror="05rk03822", - special_collection_score=p["sc_score"], - special_collection_categories=p["sc_cats"], - dc_rights="info:eu-repo/semantics/openAccess", - dc_description_provenance=f"Seeded for demo — URAAS {datetime.utcnow().date()}", - is_african_language=False, - cited_by_count=random.randint(0, 45), - ) - # Mint ARK - item.ark = ark_generator.mint(doi) - item.ark_assigned_at = datetime.utcnow() - item.collections.append(collection) - - for a_name in p["authors"]: - author = session.query(Author).filter_by(normalized_name=a_name.lower()).first() - if not author: - author = Author( - name=a_name, - normalized_name=a_name.lower(), - orcid="", - ror="", - ) - session.add(author) - item.authors.append(author) - - session.add(item) - - session.commit() - count = session.query(Item).count() - print(f"Done. Database has {count} items with ARKs.") - finally: - session.close() - - -if __name__ == "__main__": - seed() +""" +Seed a demo SQLite database with enough data for a compelling live demo. + +Run this ONCE on your local machine before deploying to HF Spaces: + python scripts/seed_demo_db.py + +This creates/populates uraas.db with: + - 30 realistic SC papers (from a cached harvest) + - ARK identifiers for each + - Author + collection associations + +The resulting uraas.db is then bundled into the Docker image (Dockerfile.hf +copies it in), so HF Spaces always starts with data even after a restart. +""" + +import os +import sys +from datetime import datetime, timedelta +import random + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from uraas.database import Author, Base, Collection, Community, Item, engine, SessionLocal +from uraas.utils.ark_generator import ark_generator + +DEMO_PAPERS = [ + { + "title": "Yoruba Oral Traditions and the Digital Archive: Preservation Challenges at the University of Lagos", + "abstract": "This paper examines the intersection of Yoruba oral traditions and digital preservation strategies. We document 847 oral narratives collected from the Lagos metropolitan area and propose a culturally sensitive framework for their archival representation.", + "authors": ["Adeyemi, O.A.", "Fashola, B.K.", "Okonkwo, C."], + "doi": "10.1234/uraas.2023.001", + "source": "AJOL", + "year": "2023", + "sc_score": 3.2, + "sc_cats": "indigenous_knowledge,oral_tradition,african_literature", + }, + { + "title": "Ethnobotanical Survey of Medicinal Plants Used by Traditional Healers in Lagos State", + "abstract": "A systematic ethnobotanical survey of 127 medicinal plant species used by traditional Yoruba healers in Lagos State. Interviews conducted with 89 traditional medical practitioners across 12 local government areas document indigenous pharmacological knowledge at risk of extinction.", + "authors": ["Okafor, N.N.", "Adewale, P.O."], + "doi": "10.1234/uraas.2023.002", + "source": "PubMed", + "year": "2023", + "sc_score": 2.8, + "sc_cats": "indigenous_knowledge,african_literature", + }, + { + "title": "Decolonising the Nigerian University Curriculum: A Case for Indigenous Epistemologies", + "abstract": "Critical examination of colonial legacies in Nigerian higher education curricula. Drawing on Ubuntu philosophy and Afrocentric scholarship, we propose a framework for recentring African knowledge systems within university pedagogy.", + "authors": ["Nwosu, E.C.", "Bamgbose, A.L.", "Eze, F.K."], + "doi": "10.1234/uraas.2023.003", + "source": "OpenAlex", + "year": "2022", + "sc_score": 2.5, + "sc_cats": "african_literature,postcolonial_studies", + }, + { + "title": "Cultural Heritage Documentation in Post-Colonial Nigeria: The Lagos Museum Collections", + "abstract": "Systematic documentation methodology for 3,400 artefacts in the Lagos State Museum. This work establishes provenance records, cultural context narratives, and digital metadata standards aligned with Dublin Core and the CIDOC-CRM ontology.", + "authors": ["Adewale, S.O.", "Obi, T.N."], + "doi": "10.1234/uraas.2023.004", + "source": "DOAJ", + "year": "2023", + "sc_score": 2.9, + "sc_cats": "cultural_heritage,indigenous_knowledge", + }, + { + "title": "Persistent Identifiers for African Institutional Repositories: The ARK Alliance Partnership", + "abstract": "Analysis of PID adoption patterns across 47 African institutional repositories. The Africa PID Alliance's partnership with the ARK Alliance (2025) provides a no-fee persistent identifier infrastructure appropriate for under-resourced institutions.", + "authors": ["Lawal, G.A.", "Ifeanyi, C.O."], + "doi": "10.1234/uraas.2024.001", + "source": "OpenAlex", + "year": "2024", + "sc_score": 1.8, + "sc_cats": "indigenous_knowledge", + }, + { + "title": "Igbo Proverb Literature and Collective Memory: A Computational Analysis", + "abstract": "Using NLP techniques, we analyse a corpus of 12,000 Igbo proverbs collected from 1952–2020. Semantic clustering reveals seven dominant thematic domains, and temporal analysis shows accelerating loss of proverbial usage in urban Igbo communities.", + "authors": ["Okonkwo, C.F.", "Nwosu, P.E.", "Adeyemi, R.A."], + "doi": "10.1234/uraas.2022.001", + "source": "Semantic Scholar", + "year": "2022", + "sc_score": 3.1, + "sc_cats": "oral_tradition,african_literature,indigenous_knowledge", + }, + { + "title": "Traditional Governance Systems and Modern State Formation in South-West Nigeria", + "abstract": "Comparative analysis of Yoruba traditional governance structures (obas, chiefs, age-grade systems) and their integration with post-independence Nigerian state institutions. Case studies from Oyo, Osun, and Lagos states.", + "authors": ["Fashola, K.T.", "Adewale, J.O."], + "doi": "10.1234/uraas.2021.001", + "source": "DOAJ", + "year": "2021", + "sc_score": 2.3, + "sc_cats": "cultural_heritage,indigenous_knowledge", + }, + { + "title": "Lagos Market Women's Oral Histories: Gender, Trade, and Urban Memory", + "abstract": "Oral history methodology applied to 234 interviews with Lagos market women aged 60–95. Documents the transformation of Yoruba women's economic practices from 1940 to present, preserving accounts unavailable in colonial archival records.", + "authors": ["Adeola, F.N.", "Okafor, B.C."], + "doi": "10.1234/uraas.2023.005", + "source": "AJOL", + "year": "2023", + "sc_score": 3.4, + "sc_cats": "oral_tradition,cultural_heritage,african_literature", + }, + { + "title": "Hausa Manuscript Collections in Northern Nigerian Libraries: A Conservation Survey", + "abstract": "Survey of 156 manuscript collections across 23 libraries in Kano, Sokoto, and Maiduguri. We identify 47,000 Hausa-language manuscripts at immediate conservation risk and propose a digitisation triage protocol.", + "authors": ["Musa, A.B.", "Ibrahim, K.S."], + "doi": "10.1234/uraas.2022.002", + "source": "CORE", + "year": "2022", + "sc_score": 2.7, + "sc_cats": "cultural_heritage,indigenous_knowledge", + }, + { + "title": "Postcolonial African Science Fiction: Imagining Futures Beyond Extractivism", + "abstract": "Literary analysis of 78 African science fiction works published 2010–2023. We argue that Afrofuturist fiction constitutes an emerging mode of indigenous knowledge production, encoding African cosmologies in speculative narrative form.", + "authors": ["Nwosu, C.I.", "Lawal, A.O.", "Eze, K.N."], + "doi": "10.1234/uraas.2023.006", + "source": "OpenAlex", + "year": "2023", + "sc_score": 2.2, + "sc_cats": "african_literature,postcolonial_studies", + }, +] + +# Pad to 30 papers +_extra_titles = [ + ("Ubuntu Philosophy and Collective Well-being in Contemporary African Ethics", "african_literature,indigenous_knowledge"), + ("Traditional Water Management Practices of the Niger Delta Communities", "indigenous_knowledge,cultural_heritage"), + ("Afrobeat as Cultural Heritage: Fela Kuti's Archive at University of Lagos", "cultural_heritage,african_literature"), + ("Endangered Languages of the Benue-Congo Region: A Documentation Framework", "indigenous_knowledge,oral_tradition"), + ("Sacred Groves as Living Cultural Heritage in Yorubaland", "cultural_heritage,indigenous_knowledge"), + ("Knowledge Repatriation: Returning Benin Bronzes and Digital Surrogates", "cultural_heritage,postcolonial_studies"), + ("Decolonising Cartography: Mapping Indigenous Territories in Nigeria", "indigenous_knowledge,postcolonial_studies"), + ("Nollywood and the Commodification of Yoruba Oral Narratives", "african_literature,oral_tradition"), + ("Traditional Ecological Knowledge and Biodiversity in Lagos Wetlands", "indigenous_knowledge,cultural_heritage"), + ("Precolonial Trans-Saharan Trade Networks: New Archaeological Evidence", "cultural_heritage,african_literature"), + ("African Proverbs in Contemporary Diplomatic Discourse", "oral_tradition,indigenous_knowledge"), + ("The Ogboni Society: Sacred Brotherhood and Political Power in Yorubaland", "indigenous_knowledge,cultural_heritage"), + ("Digital Humanities and African Archival Futures", "cultural_heritage,african_literature"), + ("Ancestral Veneration Practices in Urban Yoruba Communities", "indigenous_knowledge,oral_tradition"), + ("Linguistic Rights and African Language Policy in Nigerian Universities", "african_literature,indigenous_knowledge"), + ("Community Archives and the Decolonisation of Memory in West Africa", "cultural_heritage,postcolonial_studies"), + ("Trado-Medical Practitioners and the Nigerian Health System", "indigenous_knowledge,cultural_heritage"), + ("Ifa Divination Corpus: Computational Approaches to Sacred Oral Literature", "oral_tradition,indigenous_knowledge"), + ("Pan-African Student Movements and the Politics of Knowledge Production", "postcolonial_studies,african_literature"), + ("Nok Terracotta Figurines: New Dating Evidence from Northern Nigeria", "cultural_heritage,indigenous_knowledge"), +] + +for i, (title, cats) in enumerate(_extra_titles): + DEMO_PAPERS.append({ + "title": title, + "abstract": f"Research paper examining {title.lower()}. " + "This study contributes to the growing body of African Special Collections scholarship " + "accessible through URAAS at the University of Lagos.", + "authors": [f"Demo Author {chr(65 + i)}", f"Demo Author {chr(66 + i)}"], + "doi": f"10.1234/uraas.demo.{i+1:03d}", + "source": ["OpenAlex", "DOAJ", "AJOL", "Crossref"][i % 4], + "year": str(2019 + (i % 6)), + "sc_score": round(1.5 + (i % 10) * 0.2, 1), + "sc_cats": cats, + }) + + +def seed(): + Base.metadata.create_all(engine) + session = SessionLocal() + try: + if session.query(Item).count() >= 10: + print(f"Database already has {session.query(Item).count()} items — skipping seed.") + return + + print(f"Seeding {len(DEMO_PAPERS)} demo papers...") + + # Create a basic community/collection + community = session.query(Community).filter_by(name="Special Collections").first() + if not community: + community = Community( + name="Special Collections", + dc_title="Special Collections", + dc_description="African literature, indigenous knowledge, and cultural heritage.", + ) + session.add(community) + session.flush() + + collection = session.query(Collection).filter_by(name="Oral Traditions & Indigenous Knowledge").first() + if not collection: + collection = Collection( + name="Oral Traditions & Indigenous Knowledge", + community_id=community.id, + ) + session.add(collection) + session.flush() + + for i, p in enumerate(DEMO_PAPERS): + doi = p["doi"] + existing = session.query(Item).filter_by(doi=doi).first() + if existing: + continue + + pub_date = datetime(int(p["year"]), 1 + (i % 12), 1 + (i % 28)) + item = Item( + title=p["title"], + dc_title=p["title"], + abstract=p["abstract"], + doi=doi, + dc_identifier_doi=doi, + dc_identifier_uri=f"https://doi.org/{doi}", + url=f"https://doi.org/{doi}", + publication_date=pub_date, + dc_date_issued=p["year"], + source_repository=p["source"], + institution="University of Lagos", + ror="05rk03822", + special_collection_score=p["sc_score"], + special_collection_categories=p["sc_cats"], + dc_rights="info:eu-repo/semantics/openAccess", + dc_description_provenance=f"Seeded for demo — URAAS {datetime.utcnow().date()}", + is_african_language=False, + cited_by_count=random.randint(0, 45), + ) + # Mint ARK + item.ark = ark_generator.mint(doi) + item.ark_assigned_at = datetime.utcnow() + item.collections.append(collection) + + for a_name in p["authors"]: + author = session.query(Author).filter_by(normalized_name=a_name.lower()).first() + if not author: + author = Author( + name=a_name, + normalized_name=a_name.lower(), + orcid="", + ror="", + ) + session.add(author) + item.authors.append(author) + + session.add(item) + + session.commit() + count = session.query(Item).count() + print(f"Done. Database has {count} items with ARKs.") + finally: + session.close() + + +if __name__ == "__main__": + seed() diff --git a/scripts/start_hf.sh b/scripts/start_hf.sh index c2add5768c3e26620dd2ce61f7783c128781f356..6ce8eb3c2c45c9152ba6e3e6776fd0cd06433236 100644 --- a/scripts/start_hf.sh +++ b/scripts/start_hf.sh @@ -1,26 +1,26 @@ -#!/bin/bash -# HF Spaces entrypoint — uses /data (persistent bucket) for database & storage. -set -e - -# Persistent bucket mounted at /data — create subdirs if first run -mkdir -p /data/pdfs /data/logs - -# Point everything at the persistent volume -export DATABASE_URL="sqlite:////data/uraas.db" -export STORAGE_PATH="/data/pdfs" - -# Init DB (creates tables if not present, skips if already exists) -echo "[INIT] Initialising database at /data/uraas.db..." -python scripts/init_db.py - -echo "[INIT] Starting URAAS dashboard on port 7860..." -exec gunicorn \ - --bind 0.0.0.0:7860 \ - --worker-class gthread \ - --workers 1 \ - --threads 4 \ - --timeout 120 \ - --keep-alive 5 \ - --access-logfile - \ - --error-logfile - \ - uraas.dashboard.app:app +#!/bin/bash +# HF Spaces entrypoint — uses /data (persistent bucket) for database & storage. +set -e + +# Persistent bucket mounted at /data — create subdirs if first run +mkdir -p /data/pdfs /data/logs + +# Point everything at the persistent volume +export DATABASE_URL="sqlite:////data/uraas.db" +export STORAGE_PATH="/data/pdfs" + +# Init DB (creates tables if not present, skips if already exists) +echo "[INIT] Initialising database at /data/uraas.db..." +python scripts/init_db.py + +echo "[INIT] Starting URAAS dashboard on port 7860..." +exec gunicorn \ + --bind 0.0.0.0:7860 \ + --worker-class gthread \ + --workers 1 \ + --threads 4 \ + --timeout 120 \ + --keep-alive 5 \ + --access-logfile - \ + --error-logfile - \ + uraas.dashboard.app:app diff --git a/scripts/test_harvest_50.py b/scripts/test_harvest_50.py index e287752df3de817c6491198988a7d6a713f46d43..0c87a35d4a461da94bad7a4ed19faf462557717c 100644 --- a/scripts/test_harvest_50.py +++ b/scripts/test_harvest_50.py @@ -1,846 +1,846 @@ -""" -Test Harvest — UNILAG Special Collections papers (dry run, multi-source). - -Discovers SC papers from the open web by querying multiple academic databases: - • OpenAlex — broad academic paper index (journals, books, preprints) - • Crossref — DOI metadata authority, strong on humanities/social sciences - • Semantic Scholar — AI-indexed full-text coverage, good for humanities - • EuropePMC — PubMed + PMC + WHO; best for ethnobotany / traditional medicine - -For each source: queries institution affiliation + SC seed keywords, then applies -the same SC classifier used by the main pipeline. Results are deduplicated by DOI -and normalised title across all sources. - -DOES NOT save anything to the local database. -DOES NOT deposit anything to the live DSpace IR. -Safe to run at any time. -""" - -import argparse -import json -import os -import sys -import time -from datetime import datetime, timezone -from typing import Any - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -import requests - -from uraas.config import config -from uraas.config.institutions import get_registry -from uraas.config.special_collections import SC_SEED_KEYWORDS -from uraas.services.sc_engine import is_special_collection - -_RATE_SLEEP = 1.0 # polite delay between requests per source - - -# ── OpenAlex ────────────────────────────────────────────────────────────────── - -def _reconstruct_abstract(inverted_index: dict) -> str: - if not inverted_index: - return "" - pairs = [(pos, word) for word, positions in inverted_index.items() for pos in positions] - pairs.sort() - return " ".join(w for _, w in pairs) - - -def _openalex_url(ror_short: str, seed: str | None, cursor: str = "*") -> str: - filters = f"institutions.ror:{ror_short}" - if seed: - filters += f",title_and_abstract.search:{seed.replace(' ','%20')}" - return ( - f"https://api.openalex.org/works" - f"?filter={filters}" - f"&select=id,doi,title,abstract_inverted_index,authorships," - f"publication_date,open_access,primary_location,concepts,type" - f"&per-page=100&cursor={cursor}&mailto={config.OPENALEX_MAILTO}" - ) - - -def harvest_openalex( - session: requests.Session, - ror_short: str, - max_results: int, - seen_dois: set, - seen_titles: set, -) -> list[dict]: - papers: list[dict] = [] - seeds = list(SC_SEED_KEYWORDS) + [None] # None = general ROR wave last - - for seed in seeds: - if len(papers) >= max_results: - break - url = _openalex_url(ror_short, seed) - page = 0 - while url and len(papers) < max_results: - page += 1 - try: - resp = session.get(url, timeout=30) - resp.raise_for_status() - except requests.RequestException as exc: - print(f" [OA] {seed or 'general'} page {page} err: {exc}", flush=True) - break - data = resp.json() - results = data.get("results", []) - for work in results: - if len(papers) >= max_results: - break - title = (work.get("title") or "").strip() - if not title: - continue - doi = (work.get("doi") or "").replace("https://doi.org/", "").strip() - norm = title.lower()[:120] - if doi and doi in seen_dois: - continue - if norm in seen_titles: - continue - abstract = _reconstruct_abstract(work.get("abstract_inverted_index") or {}) - concepts = work.get("concepts") or [] - dc_subject = ", ".join(c.get("display_name", "") for c in concepts[:6] if c) - is_sc, score, cats = is_special_collection(title, abstract, dc_subject) - if not is_sc: - continue - if doi: - seen_dois.add(doi) - seen_titles.add(norm) - authors = [ - a.get("author", {}).get("display_name", "") - for a in work.get("authorships", []) - if a.get("author", {}).get("display_name") - ] - oa = work.get("open_access") or {} - landing = (work.get("primary_location") or {}).get("landing_page_url") or "" - url_val = landing or (f"https://doi.org/{doi}" if doi else "") - papers.append(_paper( - title, abstract, authors, doi, url_val, - oa.get("oa_url") if oa.get("is_oa") else None, - work.get("publication_date") or "", - work.get("type") or "", - score, cats, "OpenAlex", - )) - _log_hit(len(papers), score, cats, title) - meta = data.get("meta") or {} - next_cursor = meta.get("next_cursor") - if next_cursor and results and len(papers) < max_results: - from urllib.parse import parse_qs, urlparse - qs = parse_qs(urlparse(url).query) - filt = (qs.get("filter") or [f"institutions.ror:{ror_short}"])[0] - url = ( - f"https://api.openalex.org/works?filter={filt}" - f"&select=id,doi,title,abstract_inverted_index,authorships," - f"publication_date,open_access,primary_location,concepts,type" - f"&per-page=100&cursor={next_cursor}&mailto={config.OPENALEX_MAILTO}" - ) - time.sleep(_RATE_SLEEP) - else: - url = "" - return papers - - -# ── Crossref ────────────────────────────────────────────────────────────────── - -def harvest_crossref( - session: requests.Session, - institution_name: str, - max_results: int, - seen_dois: set, - seen_titles: set, -) -> list[dict]: - from urllib.parse import quote - papers: list[dict] = [] - seeds = list(SC_SEED_KEYWORDS) + [None] - - for seed in seeds: - if len(papers) >= max_results: - break - q_seed = f"&query={quote(seed)}" if seed else "" - url = ( - f"https://api.crossref.org/works" - f"?query.affiliation={quote(institution_name)}" - f"{q_seed}" - f"&select=DOI,title,abstract,author,issued,URL,link" - f"&rows=50&offset=0&mailto={config.OPENALEX_MAILTO}" - ) - offset = 0 - while url and len(papers) < max_results: - try: - resp = session.get(url, timeout=30) - resp.raise_for_status() - except requests.RequestException as exc: - print(f" [CR] {seed or 'general'} err: {exc}", flush=True) - break - items = resp.json().get("message", {}).get("items", []) - for work in items: - if len(papers) >= max_results: - break - title_arr = work.get("title") or [] - title = (title_arr[0] if title_arr else "").strip() - if not title: - continue - doi = (work.get("DOI") or "").strip() - norm = title.lower()[:120] - if doi and doi in seen_dois: - continue - if norm in seen_titles: - continue - abstract = (work.get("abstract") or "").strip() - is_sc, score, cats = is_special_collection(title, abstract, "") - if not is_sc: - continue - if doi: - seen_dois.add(doi) - seen_titles.add(norm) - authors = [ - f"{a.get('given','')} {a.get('family','')}".strip() - for a in work.get("author", []) - if a.get("family") - ] - issued = work.get("issued", {}).get("date-parts", [[]])[0] - pub_date = "-".join(str(p) for p in issued) if issued else "" - pdf_url = next( - (lk["URL"] for lk in work.get("link", []) - if lk.get("content-type") == "application/pdf"), - None, - ) - url_val = work.get("URL") or (f"https://doi.org/{doi}" if doi else "") - papers.append(_paper( - title, abstract, authors, doi, url_val, pdf_url, - pub_date, "", score, cats, "Crossref", - )) - _log_hit(len(papers), score, cats, title) - offset += 50 - if items and offset < 500 and len(papers) < max_results: - q_seed2 = f"&query={quote(seed)}" if seed else "" - url = ( - f"https://api.crossref.org/works" - f"?query.affiliation={quote(institution_name)}" - f"{q_seed2}" - f"&select=DOI,title,abstract,author,issued,URL,link" - f"&rows=50&offset={offset}&mailto={config.OPENALEX_MAILTO}" - ) - time.sleep(_RATE_SLEEP) - else: - url = "" - return papers - - -# ── Semantic Scholar ────────────────────────────────────────────────────────── - -def harvest_semantic_scholar( - session: requests.Session, - institution_name: str, - max_results: int, - seen_dois: set, - seen_titles: set, -) -> list[dict]: - """ - Semantic Scholar free tier: 1 req/sec (unauthenticated). - We fire only 3 broad queries instead of one per seed to stay within rate limits. - Set S2_API_KEY in .env for a higher rate limit (free key at semanticscholar.org). - """ - from urllib.parse import quote - from uraas.config import config as cfg - - papers: list[dict] = [] - # Pull API key from env if available - api_key = getattr(cfg, "S2_API_KEY", "") or os.environ.get("S2_API_KEY", "") - delay = 1.2 if not api_key else 0.3 - - headers = {} - if api_key: - headers["x-api-key"] = api_key - - # Use 3 broad queries rather than 20+ per-seed blasts - broad_queries = [ - f"{institution_name} indigenous knowledge cultural heritage", - f"{institution_name} postcolonial african literature oral tradition", - f"{institution_name} ethnobotany traditional medicine decolonial", - ] - - for query in broad_queries: - if len(papers) >= max_results: - break - offset = 0 - while len(papers) < max_results: - url = ( - f"https://api.semanticscholar.org/graph/v1/paper/search" - f"?query={quote(query)}" - f"&fields=title,abstract,authors,year,externalIds,openAccessPdf" - f"&limit=100&offset={offset}" - ) - try: - time.sleep(delay) - resp = session.get(url, timeout=30, headers=headers) - if resp.status_code == 429: - # Rate limited — skip remaining S2 queries rather than block. - # Add S2_API_KEY to .env (free at semanticscholar.org) to lift limit. - print( - " [S2] Rate limited. Add S2_API_KEY to .env for higher quota.", - flush=True, - ) - return papers - resp.raise_for_status() - except requests.RequestException as exc: - print(f" [S2] {query[:40]} err: {exc}", flush=True) - break - data = resp.json() - results = data.get("data", []) - if not results: - break - for work in results: - if len(papers) >= max_results: - break - title = (work.get("title") or "").strip() - if not title: - continue - ext = work.get("externalIds") or {} - doi = (ext.get("DOI") or ext.get("doi") or "").strip() - norm = title.lower()[:120] - if doi and doi in seen_dois: - continue - if norm in seen_titles: - continue - abstract = (work.get("abstract") or "").strip() - is_sc, score, cats = is_special_collection(title, abstract, "") - if not is_sc: - continue - if doi: - seen_dois.add(doi) - seen_titles.add(norm) - authors = [ - a.get("name", "") for a in (work.get("authors") or []) if a.get("name") - ] - year = work.get("year") - oa = work.get("openAccessPdf") or {} - url_val = f"https://doi.org/{doi}" if doi else "" - papers.append(_paper( - title, abstract, authors, doi, url_val, - oa.get("url"), f"{year}-01-01" if year else "", - "", score, cats, "Semantic Scholar", - )) - _log_hit(len(papers), score, cats, title) - total = data.get("total", 0) - offset += 100 - if offset < min(total, 300) and len(papers) < max_results: - pass - else: - break - return papers - - -# ── EuropePMC ───────────────────────────────────────────────────────────────── - -def harvest_europepmc( - session: requests.Session, - institution_name: str, - affiliation_patterns: list[str], - max_results: int, - seen_dois: set, - seen_titles: set, -) -> list[dict]: - from urllib.parse import urlencode - papers: list[dict] = [] - - # EuropePMC AFFILIATION field matches against stored author affiliation strings. - # UNILAG papers appear under several spellings — use a short unambiguous token. - affil = '(AFFILIATION:"University of Lagos" OR AFFILIATION:"unilag" OR AFFILIATION:"UNILAG")' - - # EuropePMC is best for ethnobotany/traditional medicine SC papers - priority_seeds = [ - s for s in SC_SEED_KEYWORDS - if any(k in s.lower() for k in ( - "indigenous", "traditional", "ethnobotany", "cultural", - "oral", "decolonial", "ubuntu", "ethnomusicology", - )) - ] - - for seed in priority_seeds: - if len(papers) >= max_results: - break - query = f'{affil} AND ("{seed}")' - cursor = "*" - while len(papers) < max_results: - params = { - "query": query, - "format": "json", - "pageSize": 100, - "resultType": "core", - "cursorMark": cursor, - } - url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/search?{urlencode(params)}" - try: - resp = session.get(url, timeout=30) - resp.raise_for_status() - except requests.RequestException as exc: - print(f" [EPMC] {seed} err: {exc}", flush=True) - break - data = resp.json() - results = data.get("resultList", {}).get("result", []) - for r in results: - if len(papers) >= max_results: - break - title = (r.get("title") or "").strip().rstrip(".") - if not title: - continue - doi = (r.get("doi") or "").strip() - norm = title.lower()[:120] - if doi and doi in seen_dois: - continue - if norm in seen_titles: - continue - abstract = (r.get("abstractText") or "").strip() - is_sc, score, cats = is_special_collection(title, abstract, "") - if not is_sc: - continue - if doi: - seen_dois.add(doi) - seen_titles.add(norm) - pmid = r.get("pmid") or "" - url_val = ( - f"https://doi.org/{doi}" if doi - else (f"https://europepmc.org/article/med/{pmid}" if pmid else "") - ) - pdf_url = None - if r.get("isOpenAccess") == "Y": - for ft in ((r.get("fullTextUrlList") or {}).get("fullTextUrl") or []): - if ft.get("documentStyle") == "pdf": - pdf_url = ft.get("url") - break - authors_raw = (r.get("authorList") or {}).get("author") or [] - authors = [ - f"{a.get('firstName','')} {a.get('lastName','')}".strip() - for a in authors_raw if a.get("lastName") - ] - papers.append(_paper( - title, abstract, authors, doi, url_val, pdf_url, - r.get("firstPublicationDate") or r.get("pubYear") or "", - r.get("pubType") or "", score, cats, "EuropePMC", - )) - _log_hit(len(papers), score, cats, title) - next_cursor = data.get("nextCursorMark") - if next_cursor and next_cursor != cursor and results and len(papers) < max_results: - cursor = next_cursor - time.sleep(_RATE_SLEEP) - else: - break - time.sleep(_RATE_SLEEP) - return papers - - -# ── DOAJ ────────────────────────────────────────────────────────────────────── - -def harvest_doaj( - session: requests.Session, - institution_name: str, - max_results: int, - seen_dois: set, - seen_titles: set, -) -> list[dict]: - """ - Directory of Open Access Journals (DOAJ) — covers many African humanities - and social-science journals that are not in OpenAlex or Crossref. - Free API, no key required. - """ - from urllib.parse import quote - papers: list[dict] = [] - - # DOAJ article search: query is full-text across title/abstract/keywords - priority_seeds = [ - s for s in SC_SEED_KEYWORDS - if any(k in s.lower() for k in ( - "indigenous", "traditional", "cultural", "oral", - "postcolonial", "decolonial", "african", "ubuntu", - )) - ] - - for seed in priority_seeds: - if len(papers) >= max_results: - break - query = f'"{institution_name}" "{seed}"' - page = 1 - while len(papers) < max_results: - url = ( - f"https://doaj.org/api/search/articles/{quote(query)}" - f"?page={page}&pageSize=100" - ) - try: - resp = session.get(url, timeout=30) - resp.raise_for_status() - except requests.RequestException as exc: - print(f" [DOAJ] {seed} err: {exc}", flush=True) - break - data = resp.json() - results = data.get("results", []) - if not results: - break - for article in results: - if len(papers) >= max_results: - break - bib = article.get("bibjson") or {} - title_arr = bib.get("title") or "" - title = (title_arr if isinstance(title_arr, str) else "").strip() - if not title: - continue - # DOI from identifiers list - doi = "" - for ident in bib.get("identifier") or []: - if ident.get("type") == "doi": - doi = ident.get("id") or "" - break - norm = title.lower()[:120] - if doi and doi in seen_dois: - continue - if norm in seen_titles: - continue - abstract = (bib.get("abstract") or "").strip() - is_sc, score, cats = is_special_collection(title, abstract, "") - if not is_sc: - continue - if doi: - seen_dois.add(doi) - seen_titles.add(norm) - authors = [ - a.get("name", "") for a in (bib.get("author") or []) if a.get("name") - ] - pub_date = bib.get("year") or "" - url_val = f"https://doi.org/{doi}" if doi else "" - for lnk in bib.get("link") or []: - if lnk.get("type") in ("fulltext", "homepage"): - url_val = url_val or lnk.get("url", "") - papers.append(_paper( - title, abstract, authors, doi, url_val, None, - pub_date, bib.get("journal", {}).get("title", "") or "", - score, cats, "DOAJ", - )) - _log_hit(len(papers), score, cats, title) - total = data.get("total", 0) - if page * 100 < min(total, 500) and len(papers) < max_results: - page += 1 - time.sleep(_RATE_SLEEP) - else: - break - time.sleep(_RATE_SLEEP) - return papers - - -# ── Helpers ─────────────────────────────────────────────────────────────────── - -def _paper( - title, abstract, authors, doi, url, pdf_url, - pub_date, doc_type, score, cats, source, -) -> dict: - return { - "title": title, - "abstract": abstract[:500] + ("…" if len(abstract) > 500 else ""), - "authors": authors[:5], - "doi": doi, - "url": url, - "pdf_url": pdf_url, - "publication_date": pub_date, - "dc_type": doc_type, - "sc_score": round(score, 1), - "sc_categories": cats, - "source": source, - } - - -def _log_hit(n: int, score: float, cats: list, title: str): - safe = title.encode("ascii", errors="replace").decode("ascii") - print( - f" [SC] #{n:>3} score={score:.1f} cats={','.join(cats)[:45]} {safe[:65]}", - flush=True, - ) - - -# ── Main harvest orchestrator ───────────────────────────────────────────────── - -def harvest_all( - ror_short: str, - institution_name: str, - affiliation_patterns: list[str], - max_results: int, -) -> list[dict]: - """ - Fan out across all sources in parallel-quota mode. - - Each source gets an equal quota (max_results // 4, minimum 10). After all - four sources run, results are merged (deduplicated), sorted by SC score, and - capped at max_results. This ensures the email reflects genuine multi-source - coverage rather than being filled by whichever source responds fastest. - """ - session = requests.Session() - session.headers.update({ - "User-Agent": f"URAAS-TestHarvest/1.0 (dry-run; mailto:{config.OPENALEX_MAILTO})", - "Accept": "application/json", - }) - - # Per-source quota: each source gets at least 10, up to max_results - per_source = max(10, max_results // 4) - - # Each source uses its OWN seen sets so they don't clobber each other; - # dedup across sources happens in the merge step below. - source_results: dict[str, list[dict]] = {} - - source_fns = [ - ("OpenAlex", lambda: harvest_openalex( - session, ror_short, per_source, set(), set())), - ("Crossref", lambda: harvest_crossref( - session, institution_name, per_source, set(), set())), - ("Semantic Scholar", lambda: harvest_semantic_scholar( - session, institution_name, per_source, set(), set())), - ("DOAJ", lambda: harvest_doaj( - session, institution_name, per_source, set(), set())), - ] - - for source_name, harvest_fn in source_fns: - print(f"\n[SOURCE] {source_name} (quota: {per_source})", flush=True) - print("-" * 40, flush=True) - papers = harvest_fn() - source_results[source_name] = papers - print(f"[{source_name}] found {len(papers)} SC papers", flush=True) - - # Merge and deduplicate across sources - seen_dois: set[str] = set() - seen_titles: set[str] = set() - merged: list[dict] = [] - - # Interleave sources (round-robin) so the final list is balanced - max_src_len = max(len(v) for v in source_results.values()) if source_results else 0 - source_names = list(source_results.keys()) - for i in range(max_src_len): - for sname in source_names: - papers = source_results[sname] - if i >= len(papers): - continue - p = papers[i] - doi = p.get("doi") or "" - norm = (p.get("title") or "").lower()[:120] - if doi and doi in seen_dois: - continue - if norm and norm in seen_titles: - continue - if doi: - seen_dois.add(doi) - if norm: - seen_titles.add(norm) - merged.append(p) - - # Sort by SC score descending - merged.sort(key=lambda p: p.get("sc_score", 0), reverse=True) - return merged[:max_results] - - -# ── Email ───────────────────────────────────────────────────────────────────── - -def send_preview_email(to_email: str, institution_name: str, papers: list[dict]) -> bool: - from uraas.config import config as cfg - - if not cfg.SMTP_HOST or not cfg.SMTP_USER or not cfg.SMTP_PASSWORD: - print("[WARN] SMTP not configured — skipping email. Set SMTP_* in .env", flush=True) - preview = json.dumps(papers[:3], indent=2, ensure_ascii=True) - print(f" First 3 papers: {preview}", flush=True) - return False - - import smtplib - from email.mime.multipart import MIMEMultipart - from email.mime.text import MIMEText - - n = len(papers) - subject = f"[URAAS] Test Harvest Preview — {n} SC papers from {institution_name}" - - source_counts: dict[str, int] = {} - for p in papers: - source_counts[p.get("source", "?")] = source_counts.get(p.get("source", "?"), 0) + 1 - source_summary = " · ".join(f"{s}: {c}" for s, c in sorted(source_counts.items())) - - rows_html = "" - for i, p in enumerate(papers, 1): - cats = ", ".join(p["sc_categories"]) - url_part = ( - f'{p["url"][:55]}' - if p["url"] else "—" - ) - pdf_part = ( - f' [PDF]' - if p.get("pdf_url") else "" - ) - rows_html += f""" - - {i} - - {p['title'][:85]}
- {', '.join(p['authors'][:2])} - - {p.get('dc_type','—')[:20]} - {(p.get('publication_date') or '—')[:4]} - {cats} - {p.get('source','?')} - {url_part}{pdf_part} - """ - - plain_rows = "\n".join( - f"{i:>3}. [{p.get('source','?')}] {p['title'][:75]}\n" - f" By: {', '.join(p['authors'][:2]) or '—'} | {(p.get('publication_date') or '')[:4]}\n" - f" SC: {', '.join(p['sc_categories'])} | Score: {p['sc_score']}\n" - f" URL: {p['url'] or '—'}\n" - for i, p in enumerate(papers, 1) - ) - - html = f""" - - -
-
-

University of Lagos · URAAS

-

Test Harvest — Special Collections Preview

-

- OpenAlex · Crossref · Semantic Scholar · DOAJ · Dry run · No IR deposit -

-
-
-

- Dry-run preview — no papers were saved to the local database and nothing was - deposited to the live IR. These are Special Collections papers by {institution_name} - authors discovered from across the open web. -

- - - - - - - - - - - -
Institution{institution_name}Total SC papers{n}
Sources{source_summary}
- - - - - - - - - - - - - {rows_html} -
#Title / AuthorsTypeYearSC CategoriesSourceURL / PDF
-

- Papers above were found on the open web and are NOT yet confirmed to be in the UNILAG IR. - When ready to queue them for IR deposit, use the IR Deposit panel in the dashboard. -

-
-
- URAAS · APA Intelligence & Analytics Platform · University of Lagos · Dry-run — nothing was changed -
-
-""" - - plain = f"""URAAS Test Harvest — {institution_name} -Sources: {source_summary} -DRY RUN — nothing saved to DB, nothing deposited to IR. - -Special Collections papers found: {n} - -{plain_rows} ---- -URAAS — APA Intelligence & Analytics Platform -""" - - msg = MIMEMultipart("alternative") - msg["Subject"] = subject - msg["From"] = cfg.SMTP_FROM - msg["To"] = to_email - msg.attach(MIMEText(plain, "plain", "utf-8")) - msg.attach(MIMEText(html, "html", "utf-8")) - - try: - if cfg.SMTP_USE_TLS: - srv = smtplib.SMTP(cfg.SMTP_HOST, cfg.SMTP_PORT, timeout=30) - srv.ehlo() - srv.starttls() - srv.ehlo() - else: - srv = smtplib.SMTP_SSL(cfg.SMTP_HOST, cfg.SMTP_PORT, timeout=30) - srv.login(cfg.SMTP_USER, cfg.SMTP_PASSWORD) - srv.sendmail(cfg.SMTP_FROM, [to_email], msg.as_bytes()) - srv.quit() - print(f"[OK] Email sent to {to_email}", flush=True) - return True - except Exception as exc: - print(f"[ERR] Email failed: {exc}", flush=True) - return False - - -def save_json_preview(papers: list[dict], institution: str) -> str: - out_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "storage", - f"test_harvest_{institution}_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}.json", - ) - os.makedirs(os.path.dirname(out_path), exist_ok=True) - with open(out_path, "w", encoding="utf-8") as f: - json.dump( - {"institution": institution, "count": len(papers), "papers": papers}, - f, indent=2, ensure_ascii=False, - ) - print(f"[OK] Preview saved to: {out_path}", flush=True) - return out_path - - -def main(): - parser = argparse.ArgumentParser( - description="Multi-source test harvest (dry run — no DB, no IR deposit)" - ) - parser.add_argument("--institution", default="unilag") - parser.add_argument("--count", type=int, default=50, help="Max SC papers to collect") - parser.add_argument("--email", default="lawalgiyath200716@gmail.com") - args = parser.parse_args() - - registry = get_registry() - inst_cfg = registry.get(args.institution) - if not inst_cfg: - print(f"[ERR] Institution '{args.institution}' not found", flush=True) - sys.exit(1) - - ror_short = inst_cfg.ror.split("/")[-1] - - print(f"\n{'='*60}", flush=True) - print(f"URAAS DRY-RUN HARVEST — {inst_cfg.name}", flush=True) - print(f"Sources: OpenAlex · Crossref · Semantic Scholar · DOAJ", flush=True) - print(f"{'='*60}", flush=True) - - papers = harvest_all( - ror_short, - inst_cfg.name, - inst_cfg.affiliation_patterns, - args.count, - ) - - save_json_preview(papers, args.institution) - send_preview_email(args.email, inst_cfg.name, papers) - - # Source breakdown - source_counts: dict[str, int] = {} - for p in papers: - source_counts[p.get("source", "?")] = source_counts.get(p.get("source", "?"), 0) + 1 - - print(f"\n{'='*60}", flush=True) - print("HARVEST SUMMARY", flush=True) - print(f" Institution : {inst_cfg.name}", flush=True) - print(f" Total SC : {len(papers)}", flush=True) - for src, cnt in sorted(source_counts.items()): - print(f" {src:<22}: {cnt}", flush=True) - print(f" Email : {args.email}", flush=True) - print(" IR deposit : NOT performed (dry run)", flush=True) - print(f"{'='*60}\n", flush=True) - - -if __name__ == "__main__": - main() +""" +Test Harvest — UNILAG Special Collections papers (dry run, multi-source). + +Discovers SC papers from the open web by querying multiple academic databases: + • OpenAlex — broad academic paper index (journals, books, preprints) + • Crossref — DOI metadata authority, strong on humanities/social sciences + • Semantic Scholar — AI-indexed full-text coverage, good for humanities + • EuropePMC — PubMed + PMC + WHO; best for ethnobotany / traditional medicine + +For each source: queries institution affiliation + SC seed keywords, then applies +the same SC classifier used by the main pipeline. Results are deduplicated by DOI +and normalised title across all sources. + +DOES NOT save anything to the local database. +DOES NOT deposit anything to the live DSpace IR. +Safe to run at any time. +""" + +import argparse +import json +import os +import sys +import time +from datetime import datetime, timezone +from typing import Any + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import requests + +from uraas.config import config +from uraas.config.institutions import get_registry +from uraas.config.special_collections import SC_SEED_KEYWORDS +from uraas.services.sc_engine import is_special_collection + +_RATE_SLEEP = 1.0 # polite delay between requests per source + + +# ── OpenAlex ────────────────────────────────────────────────────────────────── + +def _reconstruct_abstract(inverted_index: dict) -> str: + if not inverted_index: + return "" + pairs = [(pos, word) for word, positions in inverted_index.items() for pos in positions] + pairs.sort() + return " ".join(w for _, w in pairs) + + +def _openalex_url(ror_short: str, seed: str | None, cursor: str = "*") -> str: + filters = f"institutions.ror:{ror_short}" + if seed: + filters += f",title_and_abstract.search:{seed.replace(' ','%20')}" + return ( + f"https://api.openalex.org/works" + f"?filter={filters}" + f"&select=id,doi,title,abstract_inverted_index,authorships," + f"publication_date,open_access,primary_location,concepts,type" + f"&per-page=100&cursor={cursor}&mailto={config.OPENALEX_MAILTO}" + ) + + +def harvest_openalex( + session: requests.Session, + ror_short: str, + max_results: int, + seen_dois: set, + seen_titles: set, +) -> list[dict]: + papers: list[dict] = [] + seeds = list(SC_SEED_KEYWORDS) + [None] # None = general ROR wave last + + for seed in seeds: + if len(papers) >= max_results: + break + url = _openalex_url(ror_short, seed) + page = 0 + while url and len(papers) < max_results: + page += 1 + try: + resp = session.get(url, timeout=30) + resp.raise_for_status() + except requests.RequestException as exc: + print(f" [OA] {seed or 'general'} page {page} err: {exc}", flush=True) + break + data = resp.json() + results = data.get("results", []) + for work in results: + if len(papers) >= max_results: + break + title = (work.get("title") or "").strip() + if not title: + continue + doi = (work.get("doi") or "").replace("https://doi.org/", "").strip() + norm = title.lower()[:120] + if doi and doi in seen_dois: + continue + if norm in seen_titles: + continue + abstract = _reconstruct_abstract(work.get("abstract_inverted_index") or {}) + concepts = work.get("concepts") or [] + dc_subject = ", ".join(c.get("display_name", "") for c in concepts[:6] if c) + is_sc, score, cats = is_special_collection(title, abstract, dc_subject) + if not is_sc: + continue + if doi: + seen_dois.add(doi) + seen_titles.add(norm) + authors = [ + a.get("author", {}).get("display_name", "") + for a in work.get("authorships", []) + if a.get("author", {}).get("display_name") + ] + oa = work.get("open_access") or {} + landing = (work.get("primary_location") or {}).get("landing_page_url") or "" + url_val = landing or (f"https://doi.org/{doi}" if doi else "") + papers.append(_paper( + title, abstract, authors, doi, url_val, + oa.get("oa_url") if oa.get("is_oa") else None, + work.get("publication_date") or "", + work.get("type") or "", + score, cats, "OpenAlex", + )) + _log_hit(len(papers), score, cats, title) + meta = data.get("meta") or {} + next_cursor = meta.get("next_cursor") + if next_cursor and results and len(papers) < max_results: + from urllib.parse import parse_qs, urlparse + qs = parse_qs(urlparse(url).query) + filt = (qs.get("filter") or [f"institutions.ror:{ror_short}"])[0] + url = ( + f"https://api.openalex.org/works?filter={filt}" + f"&select=id,doi,title,abstract_inverted_index,authorships," + f"publication_date,open_access,primary_location,concepts,type" + f"&per-page=100&cursor={next_cursor}&mailto={config.OPENALEX_MAILTO}" + ) + time.sleep(_RATE_SLEEP) + else: + url = "" + return papers + + +# ── Crossref ────────────────────────────────────────────────────────────────── + +def harvest_crossref( + session: requests.Session, + institution_name: str, + max_results: int, + seen_dois: set, + seen_titles: set, +) -> list[dict]: + from urllib.parse import quote + papers: list[dict] = [] + seeds = list(SC_SEED_KEYWORDS) + [None] + + for seed in seeds: + if len(papers) >= max_results: + break + q_seed = f"&query={quote(seed)}" if seed else "" + url = ( + f"https://api.crossref.org/works" + f"?query.affiliation={quote(institution_name)}" + f"{q_seed}" + f"&select=DOI,title,abstract,author,issued,URL,link" + f"&rows=50&offset=0&mailto={config.OPENALEX_MAILTO}" + ) + offset = 0 + while url and len(papers) < max_results: + try: + resp = session.get(url, timeout=30) + resp.raise_for_status() + except requests.RequestException as exc: + print(f" [CR] {seed or 'general'} err: {exc}", flush=True) + break + items = resp.json().get("message", {}).get("items", []) + for work in items: + if len(papers) >= max_results: + break + title_arr = work.get("title") or [] + title = (title_arr[0] if title_arr else "").strip() + if not title: + continue + doi = (work.get("DOI") or "").strip() + norm = title.lower()[:120] + if doi and doi in seen_dois: + continue + if norm in seen_titles: + continue + abstract = (work.get("abstract") or "").strip() + is_sc, score, cats = is_special_collection(title, abstract, "") + if not is_sc: + continue + if doi: + seen_dois.add(doi) + seen_titles.add(norm) + authors = [ + f"{a.get('given','')} {a.get('family','')}".strip() + for a in work.get("author", []) + if a.get("family") + ] + issued = work.get("issued", {}).get("date-parts", [[]])[0] + pub_date = "-".join(str(p) for p in issued) if issued else "" + pdf_url = next( + (lk["URL"] for lk in work.get("link", []) + if lk.get("content-type") == "application/pdf"), + None, + ) + url_val = work.get("URL") or (f"https://doi.org/{doi}" if doi else "") + papers.append(_paper( + title, abstract, authors, doi, url_val, pdf_url, + pub_date, "", score, cats, "Crossref", + )) + _log_hit(len(papers), score, cats, title) + offset += 50 + if items and offset < 500 and len(papers) < max_results: + q_seed2 = f"&query={quote(seed)}" if seed else "" + url = ( + f"https://api.crossref.org/works" + f"?query.affiliation={quote(institution_name)}" + f"{q_seed2}" + f"&select=DOI,title,abstract,author,issued,URL,link" + f"&rows=50&offset={offset}&mailto={config.OPENALEX_MAILTO}" + ) + time.sleep(_RATE_SLEEP) + else: + url = "" + return papers + + +# ── Semantic Scholar ────────────────────────────────────────────────────────── + +def harvest_semantic_scholar( + session: requests.Session, + institution_name: str, + max_results: int, + seen_dois: set, + seen_titles: set, +) -> list[dict]: + """ + Semantic Scholar free tier: 1 req/sec (unauthenticated). + We fire only 3 broad queries instead of one per seed to stay within rate limits. + Set S2_API_KEY in .env for a higher rate limit (free key at semanticscholar.org). + """ + from urllib.parse import quote + from uraas.config import config as cfg + + papers: list[dict] = [] + # Pull API key from env if available + api_key = getattr(cfg, "S2_API_KEY", "") or os.environ.get("S2_API_KEY", "") + delay = 1.2 if not api_key else 0.3 + + headers = {} + if api_key: + headers["x-api-key"] = api_key + + # Use 3 broad queries rather than 20+ per-seed blasts + broad_queries = [ + f"{institution_name} indigenous knowledge cultural heritage", + f"{institution_name} postcolonial african literature oral tradition", + f"{institution_name} ethnobotany traditional medicine decolonial", + ] + + for query in broad_queries: + if len(papers) >= max_results: + break + offset = 0 + while len(papers) < max_results: + url = ( + f"https://api.semanticscholar.org/graph/v1/paper/search" + f"?query={quote(query)}" + f"&fields=title,abstract,authors,year,externalIds,openAccessPdf" + f"&limit=100&offset={offset}" + ) + try: + time.sleep(delay) + resp = session.get(url, timeout=30, headers=headers) + if resp.status_code == 429: + # Rate limited — skip remaining S2 queries rather than block. + # Add S2_API_KEY to .env (free at semanticscholar.org) to lift limit. + print( + " [S2] Rate limited. Add S2_API_KEY to .env for higher quota.", + flush=True, + ) + return papers + resp.raise_for_status() + except requests.RequestException as exc: + print(f" [S2] {query[:40]} err: {exc}", flush=True) + break + data = resp.json() + results = data.get("data", []) + if not results: + break + for work in results: + if len(papers) >= max_results: + break + title = (work.get("title") or "").strip() + if not title: + continue + ext = work.get("externalIds") or {} + doi = (ext.get("DOI") or ext.get("doi") or "").strip() + norm = title.lower()[:120] + if doi and doi in seen_dois: + continue + if norm in seen_titles: + continue + abstract = (work.get("abstract") or "").strip() + is_sc, score, cats = is_special_collection(title, abstract, "") + if not is_sc: + continue + if doi: + seen_dois.add(doi) + seen_titles.add(norm) + authors = [ + a.get("name", "") for a in (work.get("authors") or []) if a.get("name") + ] + year = work.get("year") + oa = work.get("openAccessPdf") or {} + url_val = f"https://doi.org/{doi}" if doi else "" + papers.append(_paper( + title, abstract, authors, doi, url_val, + oa.get("url"), f"{year}-01-01" if year else "", + "", score, cats, "Semantic Scholar", + )) + _log_hit(len(papers), score, cats, title) + total = data.get("total", 0) + offset += 100 + if offset < min(total, 300) and len(papers) < max_results: + pass + else: + break + return papers + + +# ── EuropePMC ───────────────────────────────────────────────────────────────── + +def harvest_europepmc( + session: requests.Session, + institution_name: str, + affiliation_patterns: list[str], + max_results: int, + seen_dois: set, + seen_titles: set, +) -> list[dict]: + from urllib.parse import urlencode + papers: list[dict] = [] + + # EuropePMC AFFILIATION field matches against stored author affiliation strings. + # UNILAG papers appear under several spellings — use a short unambiguous token. + affil = '(AFFILIATION:"University of Lagos" OR AFFILIATION:"unilag" OR AFFILIATION:"UNILAG")' + + # EuropePMC is best for ethnobotany/traditional medicine SC papers + priority_seeds = [ + s for s in SC_SEED_KEYWORDS + if any(k in s.lower() for k in ( + "indigenous", "traditional", "ethnobotany", "cultural", + "oral", "decolonial", "ubuntu", "ethnomusicology", + )) + ] + + for seed in priority_seeds: + if len(papers) >= max_results: + break + query = f'{affil} AND ("{seed}")' + cursor = "*" + while len(papers) < max_results: + params = { + "query": query, + "format": "json", + "pageSize": 100, + "resultType": "core", + "cursorMark": cursor, + } + url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/search?{urlencode(params)}" + try: + resp = session.get(url, timeout=30) + resp.raise_for_status() + except requests.RequestException as exc: + print(f" [EPMC] {seed} err: {exc}", flush=True) + break + data = resp.json() + results = data.get("resultList", {}).get("result", []) + for r in results: + if len(papers) >= max_results: + break + title = (r.get("title") or "").strip().rstrip(".") + if not title: + continue + doi = (r.get("doi") or "").strip() + norm = title.lower()[:120] + if doi and doi in seen_dois: + continue + if norm in seen_titles: + continue + abstract = (r.get("abstractText") or "").strip() + is_sc, score, cats = is_special_collection(title, abstract, "") + if not is_sc: + continue + if doi: + seen_dois.add(doi) + seen_titles.add(norm) + pmid = r.get("pmid") or "" + url_val = ( + f"https://doi.org/{doi}" if doi + else (f"https://europepmc.org/article/med/{pmid}" if pmid else "") + ) + pdf_url = None + if r.get("isOpenAccess") == "Y": + for ft in ((r.get("fullTextUrlList") or {}).get("fullTextUrl") or []): + if ft.get("documentStyle") == "pdf": + pdf_url = ft.get("url") + break + authors_raw = (r.get("authorList") or {}).get("author") or [] + authors = [ + f"{a.get('firstName','')} {a.get('lastName','')}".strip() + for a in authors_raw if a.get("lastName") + ] + papers.append(_paper( + title, abstract, authors, doi, url_val, pdf_url, + r.get("firstPublicationDate") or r.get("pubYear") or "", + r.get("pubType") or "", score, cats, "EuropePMC", + )) + _log_hit(len(papers), score, cats, title) + next_cursor = data.get("nextCursorMark") + if next_cursor and next_cursor != cursor and results and len(papers) < max_results: + cursor = next_cursor + time.sleep(_RATE_SLEEP) + else: + break + time.sleep(_RATE_SLEEP) + return papers + + +# ── DOAJ ────────────────────────────────────────────────────────────────────── + +def harvest_doaj( + session: requests.Session, + institution_name: str, + max_results: int, + seen_dois: set, + seen_titles: set, +) -> list[dict]: + """ + Directory of Open Access Journals (DOAJ) — covers many African humanities + and social-science journals that are not in OpenAlex or Crossref. + Free API, no key required. + """ + from urllib.parse import quote + papers: list[dict] = [] + + # DOAJ article search: query is full-text across title/abstract/keywords + priority_seeds = [ + s for s in SC_SEED_KEYWORDS + if any(k in s.lower() for k in ( + "indigenous", "traditional", "cultural", "oral", + "postcolonial", "decolonial", "african", "ubuntu", + )) + ] + + for seed in priority_seeds: + if len(papers) >= max_results: + break + query = f'"{institution_name}" "{seed}"' + page = 1 + while len(papers) < max_results: + url = ( + f"https://doaj.org/api/search/articles/{quote(query)}" + f"?page={page}&pageSize=100" + ) + try: + resp = session.get(url, timeout=30) + resp.raise_for_status() + except requests.RequestException as exc: + print(f" [DOAJ] {seed} err: {exc}", flush=True) + break + data = resp.json() + results = data.get("results", []) + if not results: + break + for article in results: + if len(papers) >= max_results: + break + bib = article.get("bibjson") or {} + title_arr = bib.get("title") or "" + title = (title_arr if isinstance(title_arr, str) else "").strip() + if not title: + continue + # DOI from identifiers list + doi = "" + for ident in bib.get("identifier") or []: + if ident.get("type") == "doi": + doi = ident.get("id") or "" + break + norm = title.lower()[:120] + if doi and doi in seen_dois: + continue + if norm in seen_titles: + continue + abstract = (bib.get("abstract") or "").strip() + is_sc, score, cats = is_special_collection(title, abstract, "") + if not is_sc: + continue + if doi: + seen_dois.add(doi) + seen_titles.add(norm) + authors = [ + a.get("name", "") for a in (bib.get("author") or []) if a.get("name") + ] + pub_date = bib.get("year") or "" + url_val = f"https://doi.org/{doi}" if doi else "" + for lnk in bib.get("link") or []: + if lnk.get("type") in ("fulltext", "homepage"): + url_val = url_val or lnk.get("url", "") + papers.append(_paper( + title, abstract, authors, doi, url_val, None, + pub_date, bib.get("journal", {}).get("title", "") or "", + score, cats, "DOAJ", + )) + _log_hit(len(papers), score, cats, title) + total = data.get("total", 0) + if page * 100 < min(total, 500) and len(papers) < max_results: + page += 1 + time.sleep(_RATE_SLEEP) + else: + break + time.sleep(_RATE_SLEEP) + return papers + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +def _paper( + title, abstract, authors, doi, url, pdf_url, + pub_date, doc_type, score, cats, source, +) -> dict: + return { + "title": title, + "abstract": abstract[:500] + ("…" if len(abstract) > 500 else ""), + "authors": authors[:5], + "doi": doi, + "url": url, + "pdf_url": pdf_url, + "publication_date": pub_date, + "dc_type": doc_type, + "sc_score": round(score, 1), + "sc_categories": cats, + "source": source, + } + + +def _log_hit(n: int, score: float, cats: list, title: str): + safe = title.encode("ascii", errors="replace").decode("ascii") + print( + f" [SC] #{n:>3} score={score:.1f} cats={','.join(cats)[:45]} {safe[:65]}", + flush=True, + ) + + +# ── Main harvest orchestrator ───────────────────────────────────────────────── + +def harvest_all( + ror_short: str, + institution_name: str, + affiliation_patterns: list[str], + max_results: int, +) -> list[dict]: + """ + Fan out across all sources in parallel-quota mode. + + Each source gets an equal quota (max_results // 4, minimum 10). After all + four sources run, results are merged (deduplicated), sorted by SC score, and + capped at max_results. This ensures the email reflects genuine multi-source + coverage rather than being filled by whichever source responds fastest. + """ + session = requests.Session() + session.headers.update({ + "User-Agent": f"URAAS-TestHarvest/1.0 (dry-run; mailto:{config.OPENALEX_MAILTO})", + "Accept": "application/json", + }) + + # Per-source quota: each source gets at least 10, up to max_results + per_source = max(10, max_results // 4) + + # Each source uses its OWN seen sets so they don't clobber each other; + # dedup across sources happens in the merge step below. + source_results: dict[str, list[dict]] = {} + + source_fns = [ + ("OpenAlex", lambda: harvest_openalex( + session, ror_short, per_source, set(), set())), + ("Crossref", lambda: harvest_crossref( + session, institution_name, per_source, set(), set())), + ("Semantic Scholar", lambda: harvest_semantic_scholar( + session, institution_name, per_source, set(), set())), + ("DOAJ", lambda: harvest_doaj( + session, institution_name, per_source, set(), set())), + ] + + for source_name, harvest_fn in source_fns: + print(f"\n[SOURCE] {source_name} (quota: {per_source})", flush=True) + print("-" * 40, flush=True) + papers = harvest_fn() + source_results[source_name] = papers + print(f"[{source_name}] found {len(papers)} SC papers", flush=True) + + # Merge and deduplicate across sources + seen_dois: set[str] = set() + seen_titles: set[str] = set() + merged: list[dict] = [] + + # Interleave sources (round-robin) so the final list is balanced + max_src_len = max(len(v) for v in source_results.values()) if source_results else 0 + source_names = list(source_results.keys()) + for i in range(max_src_len): + for sname in source_names: + papers = source_results[sname] + if i >= len(papers): + continue + p = papers[i] + doi = p.get("doi") or "" + norm = (p.get("title") or "").lower()[:120] + if doi and doi in seen_dois: + continue + if norm and norm in seen_titles: + continue + if doi: + seen_dois.add(doi) + if norm: + seen_titles.add(norm) + merged.append(p) + + # Sort by SC score descending + merged.sort(key=lambda p: p.get("sc_score", 0), reverse=True) + return merged[:max_results] + + +# ── Email ───────────────────────────────────────────────────────────────────── + +def send_preview_email(to_email: str, institution_name: str, papers: list[dict]) -> bool: + from uraas.config import config as cfg + + if not cfg.SMTP_HOST or not cfg.SMTP_USER or not cfg.SMTP_PASSWORD: + print("[WARN] SMTP not configured — skipping email. Set SMTP_* in .env", flush=True) + preview = json.dumps(papers[:3], indent=2, ensure_ascii=True) + print(f" First 3 papers: {preview}", flush=True) + return False + + import smtplib + from email.mime.multipart import MIMEMultipart + from email.mime.text import MIMEText + + n = len(papers) + subject = f"[URAAS] Test Harvest Preview — {n} SC papers from {institution_name}" + + source_counts: dict[str, int] = {} + for p in papers: + source_counts[p.get("source", "?")] = source_counts.get(p.get("source", "?"), 0) + 1 + source_summary = " · ".join(f"{s}: {c}" for s, c in sorted(source_counts.items())) + + rows_html = "" + for i, p in enumerate(papers, 1): + cats = ", ".join(p["sc_categories"]) + url_part = ( + f'{p["url"][:55]}' + if p["url"] else "—" + ) + pdf_part = ( + f' [PDF]' + if p.get("pdf_url") else "" + ) + rows_html += f""" + + {i} + + {p['title'][:85]}
+ {', '.join(p['authors'][:2])} + + {p.get('dc_type','—')[:20]} + {(p.get('publication_date') or '—')[:4]} + {cats} + {p.get('source','?')} + {url_part}{pdf_part} + """ + + plain_rows = "\n".join( + f"{i:>3}. [{p.get('source','?')}] {p['title'][:75]}\n" + f" By: {', '.join(p['authors'][:2]) or '—'} | {(p.get('publication_date') or '')[:4]}\n" + f" SC: {', '.join(p['sc_categories'])} | Score: {p['sc_score']}\n" + f" URL: {p['url'] or '—'}\n" + for i, p in enumerate(papers, 1) + ) + + html = f""" + + +
+
+

University of Lagos · URAAS

+

Test Harvest — Special Collections Preview

+

+ OpenAlex · Crossref · Semantic Scholar · DOAJ · Dry run · No IR deposit +

+
+
+

+ Dry-run preview — no papers were saved to the local database and nothing was + deposited to the live IR. These are Special Collections papers by {institution_name} + authors discovered from across the open web. +

+ + + + + + + + + + + +
Institution{institution_name}Total SC papers{n}
Sources{source_summary}
+ + + + + + + + + + + + + {rows_html} +
#Title / AuthorsTypeYearSC CategoriesSourceURL / PDF
+

+ Papers above were found on the open web and are NOT yet confirmed to be in the UNILAG IR. + When ready to queue them for IR deposit, use the IR Deposit panel in the dashboard. +

+
+
+ URAAS · APA Intelligence & Analytics Platform · University of Lagos · Dry-run — nothing was changed +
+
+""" + + plain = f"""URAAS Test Harvest — {institution_name} +Sources: {source_summary} +DRY RUN — nothing saved to DB, nothing deposited to IR. + +Special Collections papers found: {n} + +{plain_rows} +--- +URAAS — APA Intelligence & Analytics Platform +""" + + msg = MIMEMultipart("alternative") + msg["Subject"] = subject + msg["From"] = cfg.SMTP_FROM + msg["To"] = to_email + msg.attach(MIMEText(plain, "plain", "utf-8")) + msg.attach(MIMEText(html, "html", "utf-8")) + + try: + if cfg.SMTP_USE_TLS: + srv = smtplib.SMTP(cfg.SMTP_HOST, cfg.SMTP_PORT, timeout=30) + srv.ehlo() + srv.starttls() + srv.ehlo() + else: + srv = smtplib.SMTP_SSL(cfg.SMTP_HOST, cfg.SMTP_PORT, timeout=30) + srv.login(cfg.SMTP_USER, cfg.SMTP_PASSWORD) + srv.sendmail(cfg.SMTP_FROM, [to_email], msg.as_bytes()) + srv.quit() + print(f"[OK] Email sent to {to_email}", flush=True) + return True + except Exception as exc: + print(f"[ERR] Email failed: {exc}", flush=True) + return False + + +def save_json_preview(papers: list[dict], institution: str) -> str: + out_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "storage", + f"test_harvest_{institution}_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}.json", + ) + os.makedirs(os.path.dirname(out_path), exist_ok=True) + with open(out_path, "w", encoding="utf-8") as f: + json.dump( + {"institution": institution, "count": len(papers), "papers": papers}, + f, indent=2, ensure_ascii=False, + ) + print(f"[OK] Preview saved to: {out_path}", flush=True) + return out_path + + +def main(): + parser = argparse.ArgumentParser( + description="Multi-source test harvest (dry run — no DB, no IR deposit)" + ) + parser.add_argument("--institution", default="unilag") + parser.add_argument("--count", type=int, default=50, help="Max SC papers to collect") + parser.add_argument("--email", default="lawalgiyath200716@gmail.com") + args = parser.parse_args() + + registry = get_registry() + inst_cfg = registry.get(args.institution) + if not inst_cfg: + print(f"[ERR] Institution '{args.institution}' not found", flush=True) + sys.exit(1) + + ror_short = inst_cfg.ror.split("/")[-1] + + print(f"\n{'='*60}", flush=True) + print(f"URAAS DRY-RUN HARVEST — {inst_cfg.name}", flush=True) + print(f"Sources: OpenAlex · Crossref · Semantic Scholar · DOAJ", flush=True) + print(f"{'='*60}", flush=True) + + papers = harvest_all( + ror_short, + inst_cfg.name, + inst_cfg.affiliation_patterns, + args.count, + ) + + save_json_preview(papers, args.institution) + send_preview_email(args.email, inst_cfg.name, papers) + + # Source breakdown + source_counts: dict[str, int] = {} + for p in papers: + source_counts[p.get("source", "?")] = source_counts.get(p.get("source", "?"), 0) + 1 + + print(f"\n{'='*60}", flush=True) + print("HARVEST SUMMARY", flush=True) + print(f" Institution : {inst_cfg.name}", flush=True) + print(f" Total SC : {len(papers)}", flush=True) + for src, cnt in sorted(source_counts.items()): + print(f" {src:<22}: {cnt}", flush=True) + print(f" Email : {args.email}", flush=True) + print(" IR deposit : NOT performed (dry run)", flush=True) + print(f"{'='*60}\n", flush=True) + + +if __name__ == "__main__": + main() diff --git a/start_dashboard.py b/start_dashboard.py index 28eb14261720ab696d172820592041abb9b1c8e8..9fd3cac8feb06f9c5c7c427c635119e2ef068290 100644 --- a/start_dashboard.py +++ b/start_dashboard.py @@ -1,32 +1,32 @@ -""" -Simple script to start the URAAS dashboard. -Handles Python path setup automatically. -""" - -import os -import sys - -# Add project root to Python path -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -# Now import and run the dashboard -import uraas - -print(f"DEBUG: uraas path: {uraas.__path__}", flush=True) -from uraas.config import config -from uraas.dashboard.app import app, socketio - -if __name__ == "__main__": - print("=" * 70, flush=True) - print("URAAS Dashboard Starting...", flush=True) - print("=" * 70, flush=True) - print(f"Dashboard URL: http://localhost:{config.DASHBOARD_PORT}", flush=True) - print("Press Ctrl+C to stop", flush=True) - print("=" * 70, flush=True) - socketio.run( - app, - host="0.0.0.0", - port=config.DASHBOARD_PORT, - debug=False, - allow_unsafe_werkzeug=True, - ) +""" +Simple script to start the URAAS dashboard. +Handles Python path setup automatically. +""" + +import os +import sys + +# Add project root to Python path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +# Now import and run the dashboard +import uraas + +print(f"DEBUG: uraas path: {uraas.__path__}", flush=True) +from uraas.config import config +from uraas.dashboard.app import app, socketio + +if __name__ == "__main__": + print("=" * 70, flush=True) + print("URAAS Dashboard Starting...", flush=True) + print("=" * 70, flush=True) + print(f"Dashboard URL: http://localhost:{config.DASHBOARD_PORT}", flush=True) + print("Press Ctrl+C to stop", flush=True) + print("=" * 70, flush=True) + socketio.run( + app, + host="0.0.0.0", + port=config.DASHBOARD_PORT, + debug=False, + allow_unsafe_werkzeug=True, + ) diff --git a/tests/test_all_spiders.py b/tests/test_all_spiders.py index 8835ccc7cd8741cb129fbb1da4e0f6de13f00dba..417f897110e83f2e38b9edfed9eaef219df93523 100644 --- a/tests/test_all_spiders.py +++ b/tests/test_all_spiders.py @@ -1,202 +1,202 @@ -""" -Comprehensive test for all multi-institution spiders -Tests initialization and configuration for all spider types -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from uraas.config.institutions import get_registry -from uraas.spiders.sources.arxiv_spider import ArxivSpider -from uraas.spiders.sources.crossref_spider import CrossrefSpider -from uraas.spiders.sources.openalex_spider import OpenAlexSpider -from uraas.spiders.sources.orcid_spider import ORCIDSpider -from uraas.spiders.sources.scholar_spider import ScholarSpider - - -def test_all_spiders(): - """Test that all spiders can be initialized with different institutions""" - print("\n" + "=" * 60) - print("COMPREHENSIVE SPIDER TEST") - print("=" * 60) - - registry = get_registry() - institutions = ["unilag", "ui", "oau"] - - spider_classes = { - "OpenAlex": OpenAlexSpider, - "Crossref": CrossrefSpider, - "ArXiv": ArxivSpider, - "Scholar": ScholarSpider, - "ORCID": ORCIDSpider, - } - - results = {} - - for spider_name, spider_class in spider_classes.items(): - print(f"\n{'='*60}") - print(f"Testing {spider_name} Spider") - print("=" * 60) - - spider_results = {} - - for inst in institutions: - try: - config = registry.get(inst) - if not config: - print(f" ✗ {inst}: Configuration not found") - spider_results[inst] = False - continue - - # Try to initialize spider - spider = spider_class(institution=inst) - - print(f" ✓ {inst}: {spider.institution_name}") - print(f" ROR: {spider.ror_id}") - print(f" Staff: {len(config.staff_names)}") - - spider_results[inst] = True - - except Exception as e: - print(f" ✗ {inst}: Failed - {e}") - spider_results[inst] = False - - results[spider_name] = spider_results - - # Summary - print("\n" + "=" * 60) - print("COMPREHENSIVE SUMMARY") - print("=" * 60) - - total_tests = len(spider_classes) * len(institutions) - passed_tests = sum( - 1 - for spider_results in results.values() - for success in spider_results.values() - if success - ) - - print(f"\nTotal Tests: {passed_tests}/{total_tests}") - print(f"\nResults by Spider:") - - for spider_name, spider_results in results.items(): - passed = sum(1 for v in spider_results.values() if v) - total = len(spider_results) - status = "✓" if passed == total else "✗" - print(f" {status} {spider_name}: {passed}/{total}") - - for inst, success in spider_results.items(): - inst_status = "✓" if success else "✗" - print(f" {inst_status} {inst}") - - if passed_tests == total_tests: - print("\n" + "=" * 60) - print("✓ ALL SPIDERS READY FOR MULTI-INSTITUTION CRAWLING") - print("=" * 60) - print("\nNext Steps:") - print( - " 1. Test crawl: python crawl_multi_institution.py --institutions unilag,ui --target 10 --spider openalex" - ) - print(" 2. Verify database: Check for papers with institution_ror tags") - print(" 3. Test dashboard: Verify multi-institution comparison works") - print(" 4. Production crawl: Run with higher targets for all institutions") - return True - else: - print("\n✗ SOME TESTS FAILED") - return False - - -def test_spider_metadata(): - """Test that spiders have correct metadata""" - print("\n" + "=" * 60) - print("SPIDER METADATA TEST") - print("=" * 60) - - spider_classes = { - "OpenAlex": OpenAlexSpider, - "Crossref": CrossrefSpider, - "ArXiv": ArxivSpider, - "Scholar": ScholarSpider, - "ORCID": ORCIDSpider, - } - - for spider_name, spider_class in spider_classes.items(): - spider = spider_class(institution="unilag") - print(f"\n{spider_name}:") - print(f" Name: {spider.name}") - print(f" Institution: {spider.institution_name}") - print(f" ROR: {spider.ror_id}") - - # Check for required attributes - required_attrs = ["institution_name", "ror_id", "institution_config"] - missing = [attr for attr in required_attrs if not hasattr(spider, attr)] - - if missing: - print(f" ✗ Missing attributes: {missing}") - return False - else: - print(f" ✓ All required attributes present") - - return True - - -def main(): - """Run all tests""" - print("\n" + "=" * 60) - print("MULTI-INSTITUTION SPIDER TEST SUITE") - print("=" * 60) - - tests = [ - ("Spider Metadata", test_spider_metadata), - ("All Spiders Initialization", test_all_spiders), - ] - - results = {} - - for test_name, test_func in tests: - try: - result = test_func() - results[test_name] = result - except Exception as e: - print(f"\n✗ {test_name} FAILED: {e}") - import traceback - - traceback.print_exc() - results[test_name] = False - - # Final summary - print("\n" + "=" * 60) - print("FINAL TEST SUMMARY") - print("=" * 60) - - passed = sum(1 for v in results.values() if v) - total = len(results) - - print(f"\nTests passed: {passed}/{total}\n") - - for test_name, success in results.items(): - status = "✓ PASS" if success else "✗ FAIL" - print(f" {status}: {test_name}") - - if passed == total: - print("\n" + "=" * 60) - print("✓ WEEK 1 DAY 5-7 COMPLETE") - print("=" * 60) - print("\nAll spiders updated for multi-institution support!") - print("\nImplementation Summary:") - print(" • 5 spiders updated: OpenAlex, Crossref, ArXiv, Scholar, ORCID") - print(" • 5 institutions configured: UNILAG, UI, OAU, UNN, ABU") - print(" • 2,146 total staff members loaded") - print(" • ROR-based identification implemented") - print(" • Backward compatibility maintained") - print("\nReady for production crawling!") - return 0 - else: - print("\n✗ SOME TESTS FAILED") - return 1 - - -if __name__ == "__main__": - sys.exit(main()) +""" +Comprehensive test for all multi-institution spiders +Tests initialization and configuration for all spider types +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from uraas.config.institutions import get_registry +from uraas.spiders.sources.arxiv_spider import ArxivSpider +from uraas.spiders.sources.crossref_spider import CrossrefSpider +from uraas.spiders.sources.openalex_spider import OpenAlexSpider +from uraas.spiders.sources.orcid_spider import ORCIDSpider +from uraas.spiders.sources.scholar_spider import ScholarSpider + + +def test_all_spiders(): + """Test that all spiders can be initialized with different institutions""" + print("\n" + "=" * 60) + print("COMPREHENSIVE SPIDER TEST") + print("=" * 60) + + registry = get_registry() + institutions = ["unilag", "ui", "oau"] + + spider_classes = { + "OpenAlex": OpenAlexSpider, + "Crossref": CrossrefSpider, + "ArXiv": ArxivSpider, + "Scholar": ScholarSpider, + "ORCID": ORCIDSpider, + } + + results = {} + + for spider_name, spider_class in spider_classes.items(): + print(f"\n{'='*60}") + print(f"Testing {spider_name} Spider") + print("=" * 60) + + spider_results = {} + + for inst in institutions: + try: + config = registry.get(inst) + if not config: + print(f" ✗ {inst}: Configuration not found") + spider_results[inst] = False + continue + + # Try to initialize spider + spider = spider_class(institution=inst) + + print(f" ✓ {inst}: {spider.institution_name}") + print(f" ROR: {spider.ror_id}") + print(f" Staff: {len(config.staff_names)}") + + spider_results[inst] = True + + except Exception as e: + print(f" ✗ {inst}: Failed - {e}") + spider_results[inst] = False + + results[spider_name] = spider_results + + # Summary + print("\n" + "=" * 60) + print("COMPREHENSIVE SUMMARY") + print("=" * 60) + + total_tests = len(spider_classes) * len(institutions) + passed_tests = sum( + 1 + for spider_results in results.values() + for success in spider_results.values() + if success + ) + + print(f"\nTotal Tests: {passed_tests}/{total_tests}") + print(f"\nResults by Spider:") + + for spider_name, spider_results in results.items(): + passed = sum(1 for v in spider_results.values() if v) + total = len(spider_results) + status = "✓" if passed == total else "✗" + print(f" {status} {spider_name}: {passed}/{total}") + + for inst, success in spider_results.items(): + inst_status = "✓" if success else "✗" + print(f" {inst_status} {inst}") + + if passed_tests == total_tests: + print("\n" + "=" * 60) + print("✓ ALL SPIDERS READY FOR MULTI-INSTITUTION CRAWLING") + print("=" * 60) + print("\nNext Steps:") + print( + " 1. Test crawl: python crawl_multi_institution.py --institutions unilag,ui --target 10 --spider openalex" + ) + print(" 2. Verify database: Check for papers with institution_ror tags") + print(" 3. Test dashboard: Verify multi-institution comparison works") + print(" 4. Production crawl: Run with higher targets for all institutions") + return True + else: + print("\n✗ SOME TESTS FAILED") + return False + + +def test_spider_metadata(): + """Test that spiders have correct metadata""" + print("\n" + "=" * 60) + print("SPIDER METADATA TEST") + print("=" * 60) + + spider_classes = { + "OpenAlex": OpenAlexSpider, + "Crossref": CrossrefSpider, + "ArXiv": ArxivSpider, + "Scholar": ScholarSpider, + "ORCID": ORCIDSpider, + } + + for spider_name, spider_class in spider_classes.items(): + spider = spider_class(institution="unilag") + print(f"\n{spider_name}:") + print(f" Name: {spider.name}") + print(f" Institution: {spider.institution_name}") + print(f" ROR: {spider.ror_id}") + + # Check for required attributes + required_attrs = ["institution_name", "ror_id", "institution_config"] + missing = [attr for attr in required_attrs if not hasattr(spider, attr)] + + if missing: + print(f" ✗ Missing attributes: {missing}") + return False + else: + print(f" ✓ All required attributes present") + + return True + + +def main(): + """Run all tests""" + print("\n" + "=" * 60) + print("MULTI-INSTITUTION SPIDER TEST SUITE") + print("=" * 60) + + tests = [ + ("Spider Metadata", test_spider_metadata), + ("All Spiders Initialization", test_all_spiders), + ] + + results = {} + + for test_name, test_func in tests: + try: + result = test_func() + results[test_name] = result + except Exception as e: + print(f"\n✗ {test_name} FAILED: {e}") + import traceback + + traceback.print_exc() + results[test_name] = False + + # Final summary + print("\n" + "=" * 60) + print("FINAL TEST SUMMARY") + print("=" * 60) + + passed = sum(1 for v in results.values() if v) + total = len(results) + + print(f"\nTests passed: {passed}/{total}\n") + + for test_name, success in results.items(): + status = "✓ PASS" if success else "✗ FAIL" + print(f" {status}: {test_name}") + + if passed == total: + print("\n" + "=" * 60) + print("✓ WEEK 1 DAY 5-7 COMPLETE") + print("=" * 60) + print("\nAll spiders updated for multi-institution support!") + print("\nImplementation Summary:") + print(" • 5 spiders updated: OpenAlex, Crossref, ArXiv, Scholar, ORCID") + print(" • 5 institutions configured: UNILAG, UI, OAU, UNN, ABU") + print(" • 2,146 total staff members loaded") + print(" • ROR-based identification implemented") + print(" • Backward compatibility maintained") + print("\nReady for production crawling!") + return 0 + else: + print("\n✗ SOME TESTS FAILED") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_api.py b/tests/test_api.py index d64a118cfb0a1f77989e2ee94315b91fe482920a..46bb763c3c84a4fd672027510442825a18079f6d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,473 +1,473 @@ -""" -URAAS Test Suite covers every API endpoint and APA analytics metrics. -Run: pytest tests/test_api.py -v -""" - -import os -import sys - -import pytest - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from uraas.analytics.engine import URAASAnalyticsEngine, analytics -from uraas.dashboard.app import app as flask_app -from uraas.database import Author, Collection, Community, Item, SessionLocal -from uraas.utils.ai_keyword_extractor import ai_extractor -from uraas.utils.docid_generator import docid_generator - - -@pytest.fixture(scope="module") -def client(): - flask_app.config["TESTING"] = True - with flask_app.test_client() as c: - yield c - - -# Core page - - -def test_index_loads(client): - r = client.get("/") - assert r.status_code == 200 - - -# Analytics overview - - -def test_analytics_overview(client): - r = client.get("/api/analytics/overview") - assert r.status_code == 200 - d = r.get_json() - assert "total_papers" in d - assert "total_authors" in d - assert "oa_percentage" in d - assert isinstance(d["total_papers"], int) - assert 0 <= d["oa_percentage"] <= 100 - - -def test_publications_by_year(client): - r = client.get("/api/analytics/publications-by-year") - assert r.status_code == 200 - d = r.get_json() - assert isinstance(d, list) - for item in d: - assert "year" in item and "count" in item - assert isinstance(item["year"], int) - assert item["count"] >= 0 - - -def test_papers_by_faculty(client): - r = client.get("/api/analytics/papers-by-faculty") - assert r.status_code == 200 - d = r.get_json() - assert isinstance(d, list) - - -def test_top_authors(client): - r = client.get("/api/analytics/top-authors?limit=10") - assert r.status_code == 200 - d = r.get_json() - assert isinstance(d, list) - assert len(d) <= 10 - - -def test_oa_breakdown(client): - r = client.get("/api/analytics/open-access-breakdown") - assert r.status_code == 200 - d = r.get_json() - assert isinstance(d, list) - labels = [x["label"] for x in d] - assert "Open Access" in labels - - -def test_recent_papers(client): - r = client.get("/api/analytics/recent-papers?limit=5") - assert r.status_code == 200 - d = r.get_json() - assert isinstance(d, list) - assert len(d) <= 5 - - -def test_impact_metrics(client): - r = client.get("/api/analytics/impact-metrics") - assert r.status_code == 200 - d = r.get_json() - assert "total_papers" in d - assert "oa_rate" in d - assert "doi_rate" in d - - -def test_faculties_list(client): - r = client.get("/api/analytics/faculties") - assert r.status_code == 200 - d = r.get_json() - assert isinstance(d, list) - - -# ── Search ──────────────────────────────────────────────────────────────────── - - -def test_search_empty(client): - r = client.get("/api/analytics/search?q=&limit=10") - assert r.status_code == 200 - assert isinstance(r.get_json(), list) - - -def test_search_with_query(client): - r = client.get("/api/analytics/search?q=health&limit=10") - assert r.status_code == 200 - d = r.get_json() - assert isinstance(d, list) - assert len(d) <= 10 - - -def test_search_oa_filter(client): - r = client.get("/api/analytics/search?oa_only=true&limit=20") - assert r.status_code == 200 - d = r.get_json() - for item in d: - assert item["is_oa"] == True - - -def test_search_sql_injection(client): - r = client.get("/api/analytics/search?q='; DROP TABLE items; --") - assert r.status_code == 200 # should not crash - - -# Papers tree - - -def test_papers_tree(client): - r = client.get("/api/papers/tree") - assert r.status_code == 200 - d = r.get_json() - assert "status" in d - assert "data" in d - - -# Paper detail - - -def test_paper_not_found(client): - r = client.get("/api/papers/999999") - assert r.status_code == 404 - - -def test_paper_detail_if_exists(client): - session = SessionLocal() - try: - item = session.query(Item).first() - if item: - r = client.get(f"/api/papers/{item.id}") - assert r.status_code == 200 - d = r.get_json() - assert "title" in d - assert "authors" in d - assert "dc" in d - finally: - session.close() - - -# Keyword cloud - - -def test_keyword_cloud(client): - r = client.get("/api/analytics/keyword-cloud") - assert r.status_code == 200 - d = r.get_json() - assert isinstance(d, list) - for item in d: - assert "word" in item - assert "count" in item - assert "score" in item - - -# Research trends - - -def test_research_trends(client): - r = client.get("/api/analytics/research-trends") - assert r.status_code == 200 - d = r.get_json() - assert isinstance(d, list) - for item in d: - assert "topic" in item - assert "total" in item - assert "by_year" in item - - -# Language research - - -def test_language_research(client): - r = client.get("/api/analytics/language-research") - assert r.status_code == 200 - d = r.get_json() - assert "total_language_papers" in d - assert "papers" in d - assert "top_keywords" in d - # Verify no false positives - bad_terms = [ - "machine learning", - "concrete", - "cancer", - "covid", - "petroleum", - "galaxy", - ] - for paper in d["papers"]: - title_lower = (paper.get("title") or "").lower() - for bad in bad_terms: - assert bad not in title_lower, f"False positive: '{bad}' in '{title_lower}'" - - -# APA Novel Metrics - - -def test_tk_vitality_score(client): - r = client.get("/api/analytics/tk-vitality-score") - assert r.status_code == 200 - d = r.get_json() - assert "score" in d - assert 0 <= d["score"] <= 100 - assert "breakdown" in d - assert "total_items" in d - - -def test_linguistic_diversity_index(client): - r = client.get("/api/analytics/linguistic-diversity-index") - assert r.status_code == 200 - d = r.get_json() - assert "index" in d - assert 0 <= d["index"] <= 100 - assert "breakdown" in d - - -def test_patent_velocity(client): - r = client.get("/api/analytics/patent-velocity") - assert r.status_code == 200 - d = r.get_json() - assert "total_patents" in d - assert "velocity_distribution" in d - - -def test_docid_coverage(client): - r = client.get("/api/analytics/docid-coverage") - assert r.status_code == 200 - d = r.get_json() - assert "total_papers" in d - assert "docid_assigned" in d - assert "coverage_percent" in d - assert 0 <= d["coverage_percent"] <= 100 - - -def test_docid_stats(client): - r = client.get("/api/docid/stats") - assert r.status_code == 200 - d = r.get_json() - assert "total_docid_papers" in d - assert "docid_coverage" in d - - -# Author network - - -def test_author_network_global(client): - r = client.get("/api/analytics/author-network") - assert r.status_code == 200 - d = r.get_json() - assert "nodes" in d - assert "edges" in d - - -def test_authors_search(client): - r = client.get("/api/analytics/authors-search?q=a&limit=5") - assert r.status_code == 200 - d = r.get_json() - assert isinstance(d, list) - assert len(d) <= 5 - - -# Faculty comparison - - -def test_faculty_comparison_empty(client): - r = client.get("/api/analytics/faculty-comparison") - assert r.status_code == 200 - assert isinstance(r.get_json(), dict) - - -# Exports - - -def test_export_csv(client): - r = client.get("/api/export/papers.csv") - assert r.status_code == 200 - assert "text/csv" in r.content_type - data = r.data.decode("utf-8") - assert "Title" in data or "ID" in data - - -def test_export_bibtex(client): - r = client.get("/api/export/papers.bibtex") - assert r.status_code == 200 - - -# Crawler status - - -def test_crawler_status(client): - r = client.get("/api/crawler/status") - assert r.status_code == 200 - d = r.get_json() - assert d["status"] in ("running", "idle") - - -def test_docid_crawler_status(client): - r = client.get("/api/docid-crawler/status") - assert r.status_code == 200 - d = r.get_json() - assert d["status"] in ("running", "idle") - - -# Analytics engine unit tests - - -def test_engine_top_authors(): - result = analytics.get_top_authors(limit=5) - assert isinstance(result, list) - assert len(result) <= 5 - for r in result: - assert "author" in r - assert "count" in r - assert r["count"] > 0 - - -def test_engine_sdg_alignment(): - result = analytics.get_sdg_alignment() - assert isinstance(result, list) - sdg_names = [r["sdg"] for r in result] - # Should have at least some SDGs with papers - assert len(result) >= 0 - - -def test_engine_keyword_cloud(): - result = analytics.get_keyword_cloud(top_n=20) - assert isinstance(result, list) - assert len(result) <= 20 - for item in result: - assert "word" in item - assert "score" in item - assert item["score"] > 0 - - -def test_engine_tk_vitality(): - result = analytics.get_tk_vitality_score() - assert "score" in result - assert 0 <= result["score"] <= 100 - - -def test_engine_linguistic_diversity(): - result = analytics.get_linguistic_diversity_index() - assert "index" in result - assert 0 <= result["index"] <= 100 - - -def test_engine_patent_velocity(): - result = analytics.get_patent_velocity() - assert "total_patents" in result - assert isinstance(result["total_patents"], int) - - -def test_engine_docid_coverage(): - result = analytics.get_docid_coverage() - assert "coverage_percent" in result - assert 0 <= result["coverage_percent"] <= 100 - - -# DocID generator - - -def test_docid_generation(): - docid = docid_generator.generate_docid("Test Paper Title", doi="10.1234/test") - assert docid.startswith("20.500.14351/") - parts = docid.split("/") - assert len(parts) == 2 - assert len(parts[1]) >= 10 - - -def test_docid_validation(): - valid = docid_generator.generate_docid("Test") - assert docid_generator.validate_docid(valid) == True - assert docid_generator.validate_docid("") == False - assert docid_generator.validate_docid("invalid") == False - assert docid_generator.validate_docid("99.999.99999/abc") == False - - -def test_docid_uniqueness(): - ids = {docid_generator.generate_docid("Same Title") for _ in range(10)} - assert len(ids) == 10 # all unique due to uuid4 - - -# AI keyword extractor - - -def test_keyword_extraction(): - text = ( - "machine learning deep neural networks artificial intelligence computer vision" - ) - kws = ai_extractor.extract_keywords(text, top_n=5) - assert len(kws) > 0 - assert all(isinstance(k, tuple) and len(k) == 2 for k in kws) - - -def test_keyword_extraction_empty(): - assert ai_extractor.extract_keywords("", top_n=5) == [] - - -def test_domain_classification(): - text = "algorithm data structure programming software engineering database" - domains = ai_extractor.classify_domain(text) - assert len(domains) > 0 - assert domains[0][0] == "computer_science" - - -def test_paper_scoring(): - score = ai_extractor.score_paper( - "Machine Learning for Medical Diagnosis", - "This study investigates machine learning algorithms for medical diagnosis using deep neural networks to classify medical images with significant improvement over existing methods.", - ) - assert "quality_score" in score - assert 0 <= score["quality_score"] <= 1 - assert "keywords" in score - - -# Performance - - -def test_overview_response_time(client): - import time - - start = time.time() - client.get("/api/analytics/overview") - elapsed = time.time() - start - assert elapsed < 3.0, f"Overview took {elapsed:.2f}s, should be < 3s" - - -def test_search_response_time(client): - import time - - start = time.time() - client.get("/api/analytics/search?q=health&limit=20") - elapsed = time.time() - start - assert elapsed < 5.0, f"Search took {elapsed:.2f}s, should be < 5s" - - -def test_keyword_cloud_response_time(client): - import time - - start = time.time() - client.get("/api/analytics/keyword-cloud") - elapsed = time.time() - start - assert elapsed < 10.0, f"Keyword cloud took {elapsed:.2f}s, should be < 10s" +""" +URAAS Test Suite covers every API endpoint and APA analytics metrics. +Run: pytest tests/test_api.py -v +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from uraas.analytics.engine import URAASAnalyticsEngine, analytics +from uraas.dashboard.app import app as flask_app +from uraas.database import Author, Collection, Community, Item, SessionLocal +from uraas.utils.ai_keyword_extractor import ai_extractor +from uraas.utils.docid_generator import docid_generator + + +@pytest.fixture(scope="module") +def client(): + flask_app.config["TESTING"] = True + with flask_app.test_client() as c: + yield c + + +# Core page + + +def test_index_loads(client): + r = client.get("/") + assert r.status_code == 200 + + +# Analytics overview + + +def test_analytics_overview(client): + r = client.get("/api/analytics/overview") + assert r.status_code == 200 + d = r.get_json() + assert "total_papers" in d + assert "total_authors" in d + assert "oa_percentage" in d + assert isinstance(d["total_papers"], int) + assert 0 <= d["oa_percentage"] <= 100 + + +def test_publications_by_year(client): + r = client.get("/api/analytics/publications-by-year") + assert r.status_code == 200 + d = r.get_json() + assert isinstance(d, list) + for item in d: + assert "year" in item and "count" in item + assert isinstance(item["year"], int) + assert item["count"] >= 0 + + +def test_papers_by_faculty(client): + r = client.get("/api/analytics/papers-by-faculty") + assert r.status_code == 200 + d = r.get_json() + assert isinstance(d, list) + + +def test_top_authors(client): + r = client.get("/api/analytics/top-authors?limit=10") + assert r.status_code == 200 + d = r.get_json() + assert isinstance(d, list) + assert len(d) <= 10 + + +def test_oa_breakdown(client): + r = client.get("/api/analytics/open-access-breakdown") + assert r.status_code == 200 + d = r.get_json() + assert isinstance(d, list) + labels = [x["label"] for x in d] + assert "Open Access" in labels + + +def test_recent_papers(client): + r = client.get("/api/analytics/recent-papers?limit=5") + assert r.status_code == 200 + d = r.get_json() + assert isinstance(d, list) + assert len(d) <= 5 + + +def test_impact_metrics(client): + r = client.get("/api/analytics/impact-metrics") + assert r.status_code == 200 + d = r.get_json() + assert "total_papers" in d + assert "oa_rate" in d + assert "doi_rate" in d + + +def test_faculties_list(client): + r = client.get("/api/analytics/faculties") + assert r.status_code == 200 + d = r.get_json() + assert isinstance(d, list) + + +# ── Search ──────────────────────────────────────────────────────────────────── + + +def test_search_empty(client): + r = client.get("/api/analytics/search?q=&limit=10") + assert r.status_code == 200 + assert isinstance(r.get_json(), list) + + +def test_search_with_query(client): + r = client.get("/api/analytics/search?q=health&limit=10") + assert r.status_code == 200 + d = r.get_json() + assert isinstance(d, list) + assert len(d) <= 10 + + +def test_search_oa_filter(client): + r = client.get("/api/analytics/search?oa_only=true&limit=20") + assert r.status_code == 200 + d = r.get_json() + for item in d: + assert item["is_oa"] == True + + +def test_search_sql_injection(client): + r = client.get("/api/analytics/search?q='; DROP TABLE items; --") + assert r.status_code == 200 # should not crash + + +# Papers tree + + +def test_papers_tree(client): + r = client.get("/api/papers/tree") + assert r.status_code == 200 + d = r.get_json() + assert "status" in d + assert "data" in d + + +# Paper detail + + +def test_paper_not_found(client): + r = client.get("/api/papers/999999") + assert r.status_code == 404 + + +def test_paper_detail_if_exists(client): + session = SessionLocal() + try: + item = session.query(Item).first() + if item: + r = client.get(f"/api/papers/{item.id}") + assert r.status_code == 200 + d = r.get_json() + assert "title" in d + assert "authors" in d + assert "dc" in d + finally: + session.close() + + +# Keyword cloud + + +def test_keyword_cloud(client): + r = client.get("/api/analytics/keyword-cloud") + assert r.status_code == 200 + d = r.get_json() + assert isinstance(d, list) + for item in d: + assert "word" in item + assert "count" in item + assert "score" in item + + +# Research trends + + +def test_research_trends(client): + r = client.get("/api/analytics/research-trends") + assert r.status_code == 200 + d = r.get_json() + assert isinstance(d, list) + for item in d: + assert "topic" in item + assert "total" in item + assert "by_year" in item + + +# Language research + + +def test_language_research(client): + r = client.get("/api/analytics/language-research") + assert r.status_code == 200 + d = r.get_json() + assert "total_language_papers" in d + assert "papers" in d + assert "top_keywords" in d + # Verify no false positives + bad_terms = [ + "machine learning", + "concrete", + "cancer", + "covid", + "petroleum", + "galaxy", + ] + for paper in d["papers"]: + title_lower = (paper.get("title") or "").lower() + for bad in bad_terms: + assert bad not in title_lower, f"False positive: '{bad}' in '{title_lower}'" + + +# APA Novel Metrics + + +def test_tk_vitality_score(client): + r = client.get("/api/analytics/tk-vitality-score") + assert r.status_code == 200 + d = r.get_json() + assert "score" in d + assert 0 <= d["score"] <= 100 + assert "breakdown" in d + assert "total_items" in d + + +def test_linguistic_diversity_index(client): + r = client.get("/api/analytics/linguistic-diversity-index") + assert r.status_code == 200 + d = r.get_json() + assert "index" in d + assert 0 <= d["index"] <= 100 + assert "breakdown" in d + + +def test_patent_velocity(client): + r = client.get("/api/analytics/patent-velocity") + assert r.status_code == 200 + d = r.get_json() + assert "total_patents" in d + assert "velocity_distribution" in d + + +def test_docid_coverage(client): + r = client.get("/api/analytics/docid-coverage") + assert r.status_code == 200 + d = r.get_json() + assert "total_papers" in d + assert "docid_assigned" in d + assert "coverage_percent" in d + assert 0 <= d["coverage_percent"] <= 100 + + +def test_docid_stats(client): + r = client.get("/api/docid/stats") + assert r.status_code == 200 + d = r.get_json() + assert "total_docid_papers" in d + assert "docid_coverage" in d + + +# Author network + + +def test_author_network_global(client): + r = client.get("/api/analytics/author-network") + assert r.status_code == 200 + d = r.get_json() + assert "nodes" in d + assert "edges" in d + + +def test_authors_search(client): + r = client.get("/api/analytics/authors-search?q=a&limit=5") + assert r.status_code == 200 + d = r.get_json() + assert isinstance(d, list) + assert len(d) <= 5 + + +# Faculty comparison + + +def test_faculty_comparison_empty(client): + r = client.get("/api/analytics/faculty-comparison") + assert r.status_code == 200 + assert isinstance(r.get_json(), dict) + + +# Exports + + +def test_export_csv(client): + r = client.get("/api/export/papers.csv") + assert r.status_code == 200 + assert "text/csv" in r.content_type + data = r.data.decode("utf-8") + assert "Title" in data or "ID" in data + + +def test_export_bibtex(client): + r = client.get("/api/export/papers.bibtex") + assert r.status_code == 200 + + +# Crawler status + + +def test_crawler_status(client): + r = client.get("/api/crawler/status") + assert r.status_code == 200 + d = r.get_json() + assert d["status"] in ("running", "idle") + + +def test_docid_crawler_status(client): + r = client.get("/api/docid-crawler/status") + assert r.status_code == 200 + d = r.get_json() + assert d["status"] in ("running", "idle") + + +# Analytics engine unit tests + + +def test_engine_top_authors(): + result = analytics.get_top_authors(limit=5) + assert isinstance(result, list) + assert len(result) <= 5 + for r in result: + assert "author" in r + assert "count" in r + assert r["count"] > 0 + + +def test_engine_sdg_alignment(): + result = analytics.get_sdg_alignment() + assert isinstance(result, list) + sdg_names = [r["sdg"] for r in result] + # Should have at least some SDGs with papers + assert len(result) >= 0 + + +def test_engine_keyword_cloud(): + result = analytics.get_keyword_cloud(top_n=20) + assert isinstance(result, list) + assert len(result) <= 20 + for item in result: + assert "word" in item + assert "score" in item + assert item["score"] > 0 + + +def test_engine_tk_vitality(): + result = analytics.get_tk_vitality_score() + assert "score" in result + assert 0 <= result["score"] <= 100 + + +def test_engine_linguistic_diversity(): + result = analytics.get_linguistic_diversity_index() + assert "index" in result + assert 0 <= result["index"] <= 100 + + +def test_engine_patent_velocity(): + result = analytics.get_patent_velocity() + assert "total_patents" in result + assert isinstance(result["total_patents"], int) + + +def test_engine_docid_coverage(): + result = analytics.get_docid_coverage() + assert "coverage_percent" in result + assert 0 <= result["coverage_percent"] <= 100 + + +# DocID generator + + +def test_docid_generation(): + docid = docid_generator.generate_docid("Test Paper Title", doi="10.1234/test") + assert docid.startswith("20.500.14351/") + parts = docid.split("/") + assert len(parts) == 2 + assert len(parts[1]) >= 10 + + +def test_docid_validation(): + valid = docid_generator.generate_docid("Test") + assert docid_generator.validate_docid(valid) == True + assert docid_generator.validate_docid("") == False + assert docid_generator.validate_docid("invalid") == False + assert docid_generator.validate_docid("99.999.99999/abc") == False + + +def test_docid_uniqueness(): + ids = {docid_generator.generate_docid("Same Title") for _ in range(10)} + assert len(ids) == 10 # all unique due to uuid4 + + +# AI keyword extractor + + +def test_keyword_extraction(): + text = ( + "machine learning deep neural networks artificial intelligence computer vision" + ) + kws = ai_extractor.extract_keywords(text, top_n=5) + assert len(kws) > 0 + assert all(isinstance(k, tuple) and len(k) == 2 for k in kws) + + +def test_keyword_extraction_empty(): + assert ai_extractor.extract_keywords("", top_n=5) == [] + + +def test_domain_classification(): + text = "algorithm data structure programming software engineering database" + domains = ai_extractor.classify_domain(text) + assert len(domains) > 0 + assert domains[0][0] == "computer_science" + + +def test_paper_scoring(): + score = ai_extractor.score_paper( + "Machine Learning for Medical Diagnosis", + "This study investigates machine learning algorithms for medical diagnosis using deep neural networks to classify medical images with significant improvement over existing methods.", + ) + assert "quality_score" in score + assert 0 <= score["quality_score"] <= 1 + assert "keywords" in score + + +# Performance + + +def test_overview_response_time(client): + import time + + start = time.time() + client.get("/api/analytics/overview") + elapsed = time.time() - start + assert elapsed < 3.0, f"Overview took {elapsed:.2f}s, should be < 3s" + + +def test_search_response_time(client): + import time + + start = time.time() + client.get("/api/analytics/search?q=health&limit=20") + elapsed = time.time() - start + assert elapsed < 5.0, f"Search took {elapsed:.2f}s, should be < 5s" + + +def test_keyword_cloud_response_time(client): + import time + + start = time.time() + client.get("/api/analytics/keyword-cloud") + elapsed = time.time() - start + assert elapsed < 10.0, f"Keyword cloud took {elapsed:.2f}s, should be < 10s" diff --git a/tests/test_multi_institution.py b/tests/test_multi_institution.py index 48fdb2496ceb4ed5017cf01220947069164d5805..69db4d31a142a932c82ba0e68fb2065cbcf0670a 100644 --- a/tests/test_multi_institution.py +++ b/tests/test_multi_institution.py @@ -1,175 +1,175 @@ -""" -Test script for multi-institution support -Tests institution configuration and staff validation -""" - -import os -import sys - -import pytest - -# Add project root to path -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from uraas.config.institutions import InstitutionRegistry, get_registry -from uraas.utils.staff_validator import StaffValidator - - -@pytest.fixture -def registry(): - """Provide the institution registry as a pytest fixture.""" - return get_registry() - - -def test_institution_registry(): - """Test institution registry loading""" - print("=" * 60) - print("TEST 1: Institution Registry") - print("=" * 60) - - registry = get_registry() - - print(f"\nLoaded {len(registry.institutions)} institutions:") - for config in registry.list_all(): - print(f" - {config.name} ({config.short_name})") - print(f" ROR: {config.ror}") - print(f" Country: {config.country}") - print(f" Staff count: {len(config.staff_names)}") - print(f" Affiliation patterns: {len(config.affiliation_patterns)}") - print() - - # Test retrieval by short name - print("\nTest retrieval by short name:") - unilag = registry.get("unilag") - if unilag: - print(f" ✓ Found UNILAG: {unilag.name}") - else: - print(f" ✗ UNILAG not found") - - # Test retrieval by ROR - print("\nTest retrieval by ROR:") - ui = registry.get_by_ror("https://ror.org/01js2sh04") - if ui: - print(f" ✓ Found UI: {ui.name}") - else: - print(f" ✗ UI not found") - - # Test affiliation matching - print("\nTest affiliation matching:") - test_affiliations = [ - ("University of Lagos, Nigeria", "unilag"), - ("Department of Physics, University of Ibadan", "ui"), - ("OAU Ile-Ife, Nigeria", "oau"), - ("Ahmadu Bello University, Zaria", "abu"), - ] - - for affiliation, expected_short_name in test_affiliations: - matched = False - for config in registry.list_all(): - if config.matches_affiliation(affiliation): - print(f" ✓ '{affiliation}' → {config.short_name}") - if config.short_name.lower() == expected_short_name.lower(): - matched = True - break - if not matched: - print(f" ✗ '{affiliation}' not matched correctly") - - return registry - - -def test_staff_validator(registry): - """Test staff validator with multi-institution support""" - print("\n" + "=" * 60) - print("TEST 2: Staff Validator") - print("=" * 60) - - # Test UNILAG validator - print("\nTesting UNILAG validator:") - unilag_config = registry.get("unilag") - if unilag_config: - validator = StaffValidator(institution_config=unilag_config) - print(f" Institution: {validator.institution_name}") - print(f" ROR: {validator.ror}") - print(f" Staff count: {len(validator.staff_names)}") - - # Test some known UNILAG staff (if any) - test_authors = [ - "Prof. A. O. Adeyemi", - "Dr. John Smith", # Should not match - "O. A. Ogunlana", - ] - - print("\n Testing author validation:") - for author in test_authors: - is_staff = validator.is_staff_member(author) - print( - f" {'✓' if is_staff else '✗'} {author}: {'Staff' if is_staff else 'Not staff'}" - ) - - # Test UI validator (will have empty staff list for now) - print("\nTesting UI validator:") - ui_config = registry.get("ui") - if ui_config: - validator = StaffValidator(institution_config=ui_config) - print(f" Institution: {validator.institution_name}") - print(f" ROR: {validator.ror}") - print(f" Staff count: {len(validator.staff_names)}") - print(f" Note: Staff file not yet populated") - - -def test_backward_compatibility(): - """Test that old code still works""" - print("\n" + "=" * 60) - print("TEST 3: Backward Compatibility") - print("=" * 60) - - # Test default validator (should still work for UNILAG) - from uraas.utils.staff_validator import staff_validator - - print(f"\nDefault validator:") - print(f" Institution: {staff_validator.institution_name}") - print(f" Staff count: {len(staff_validator.staff_names)}") - print(f" ✓ Backward compatibility maintained") - - -def main(): - """Run all tests""" - print("\n" + "=" * 60) - print("MULTI-INSTITUTION SUPPORT TEST SUITE") - print("=" * 60) - - try: - # Test 1: Institution Registry - registry = test_institution_registry() - - # Test 2: Staff Validator - test_staff_validator(registry) - - # Test 3: Backward Compatibility - test_backward_compatibility() - - print("\n" + "=" * 60) - print("ALL TESTS COMPLETED") - print("=" * 60) - print("\nSummary:") - print(f" - {len(registry.institutions)} institutions configured") - print(f" - Institution registry operational") - print(f" - Staff validator supports multi-institution") - print(f" - Backward compatibility maintained") - print("\nNext steps:") - print(" 1. Populate staff files for UI, OAU, UNN, ABU") - print(" 2. Update spiders to accept institution parameter") - print(" 3. Test multi-institution crawling") - - except Exception as e: - print(f"\n✗ TEST FAILED: {e}") - import traceback - - traceback.print_exc() - return 1 - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +""" +Test script for multi-institution support +Tests institution configuration and staff validation +""" + +import os +import sys + +import pytest + +# Add project root to path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from uraas.config.institutions import InstitutionRegistry, get_registry +from uraas.utils.staff_validator import StaffValidator + + +@pytest.fixture +def registry(): + """Provide the institution registry as a pytest fixture.""" + return get_registry() + + +def test_institution_registry(): + """Test institution registry loading""" + print("=" * 60) + print("TEST 1: Institution Registry") + print("=" * 60) + + registry = get_registry() + + print(f"\nLoaded {len(registry.institutions)} institutions:") + for config in registry.list_all(): + print(f" - {config.name} ({config.short_name})") + print(f" ROR: {config.ror}") + print(f" Country: {config.country}") + print(f" Staff count: {len(config.staff_names)}") + print(f" Affiliation patterns: {len(config.affiliation_patterns)}") + print() + + # Test retrieval by short name + print("\nTest retrieval by short name:") + unilag = registry.get("unilag") + if unilag: + print(f" ✓ Found UNILAG: {unilag.name}") + else: + print(f" ✗ UNILAG not found") + + # Test retrieval by ROR + print("\nTest retrieval by ROR:") + ui = registry.get_by_ror("https://ror.org/01js2sh04") + if ui: + print(f" ✓ Found UI: {ui.name}") + else: + print(f" ✗ UI not found") + + # Test affiliation matching + print("\nTest affiliation matching:") + test_affiliations = [ + ("University of Lagos, Nigeria", "unilag"), + ("Department of Physics, University of Ibadan", "ui"), + ("OAU Ile-Ife, Nigeria", "oau"), + ("Ahmadu Bello University, Zaria", "abu"), + ] + + for affiliation, expected_short_name in test_affiliations: + matched = False + for config in registry.list_all(): + if config.matches_affiliation(affiliation): + print(f" ✓ '{affiliation}' → {config.short_name}") + if config.short_name.lower() == expected_short_name.lower(): + matched = True + break + if not matched: + print(f" ✗ '{affiliation}' not matched correctly") + + return registry + + +def test_staff_validator(registry): + """Test staff validator with multi-institution support""" + print("\n" + "=" * 60) + print("TEST 2: Staff Validator") + print("=" * 60) + + # Test UNILAG validator + print("\nTesting UNILAG validator:") + unilag_config = registry.get("unilag") + if unilag_config: + validator = StaffValidator(institution_config=unilag_config) + print(f" Institution: {validator.institution_name}") + print(f" ROR: {validator.ror}") + print(f" Staff count: {len(validator.staff_names)}") + + # Test some known UNILAG staff (if any) + test_authors = [ + "Prof. A. O. Adeyemi", + "Dr. John Smith", # Should not match + "O. A. Ogunlana", + ] + + print("\n Testing author validation:") + for author in test_authors: + is_staff = validator.is_staff_member(author) + print( + f" {'✓' if is_staff else '✗'} {author}: {'Staff' if is_staff else 'Not staff'}" + ) + + # Test UI validator (will have empty staff list for now) + print("\nTesting UI validator:") + ui_config = registry.get("ui") + if ui_config: + validator = StaffValidator(institution_config=ui_config) + print(f" Institution: {validator.institution_name}") + print(f" ROR: {validator.ror}") + print(f" Staff count: {len(validator.staff_names)}") + print(f" Note: Staff file not yet populated") + + +def test_backward_compatibility(): + """Test that old code still works""" + print("\n" + "=" * 60) + print("TEST 3: Backward Compatibility") + print("=" * 60) + + # Test default validator (should still work for UNILAG) + from uraas.utils.staff_validator import staff_validator + + print(f"\nDefault validator:") + print(f" Institution: {staff_validator.institution_name}") + print(f" Staff count: {len(staff_validator.staff_names)}") + print(f" ✓ Backward compatibility maintained") + + +def main(): + """Run all tests""" + print("\n" + "=" * 60) + print("MULTI-INSTITUTION SUPPORT TEST SUITE") + print("=" * 60) + + try: + # Test 1: Institution Registry + registry = test_institution_registry() + + # Test 2: Staff Validator + test_staff_validator(registry) + + # Test 3: Backward Compatibility + test_backward_compatibility() + + print("\n" + "=" * 60) + print("ALL TESTS COMPLETED") + print("=" * 60) + print("\nSummary:") + print(f" - {len(registry.institutions)} institutions configured") + print(f" - Institution registry operational") + print(f" - Staff validator supports multi-institution") + print(f" - Backward compatibility maintained") + print("\nNext steps:") + print(" 1. Populate staff files for UI, OAU, UNN, ABU") + print(" 2. Update spiders to accept institution parameter") + print(" 3. Test multi-institution crawling") + + except Exception as e: + print(f"\n✗ TEST FAILED: {e}") + import traceback + + traceback.print_exc() + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_multi_institution_crawl.py b/tests/test_multi_institution_crawl.py index be710fd5041e5d2dadf7881af3b07081f467163b..091098dac2528b9aed733a707b5a06d410d1cfed 100644 --- a/tests/test_multi_institution_crawl.py +++ b/tests/test_multi_institution_crawl.py @@ -1,207 +1,207 @@ -""" -Test multi-institution crawling functionality -Tests spider initialization and basic crawling for multiple institutions -""" - -import os -import sys - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from uraas.config.institutions import get_registry -from uraas.spiders.sources.openalex_spider import OpenAlexSpider - - -def test_spider_initialization(): - """Test that spiders can be initialized with different institutions""" - print("\n" + "=" * 60) - print("TEST: Spider Initialization") - print("=" * 60) - - registry = get_registry() - institutions = ["unilag", "ui", "oau", "unn", "abu"] - - results = {} - - for inst in institutions: - try: - config = registry.get(inst) - if not config: - print(f"\n✗ {inst}: Configuration not found") - results[inst] = False - continue - - # Try to initialize spider - spider = OpenAlexSpider(institution=inst) - - print(f"\n✓ {inst}: {spider.institution_name}") - print(f" ROR: {spider.ror_id}") - print(f" ROR Short: {spider.ror_short}") - print(f" Staff count: {len(config.staff_names)}") - - results[inst] = True - - except Exception as e: - print(f"\n✗ {inst}: Failed to initialize - {e}") - results[inst] = False - - # Summary - print("\n" + "=" * 60) - print("INITIALIZATION SUMMARY") - print("=" * 60) - - passed = sum(1 for v in results.values() if v) - total = len(results) - - print(f"\nPassed: {passed}/{total}") - - for inst, success in results.items(): - status = "✓" if success else "✗" - print(f" {status} {inst}") - - return passed == total - - -def test_affiliation_filter(): - """Test affiliation filter with multi-institution support""" - print("\n" + "=" * 60) - print("TEST: Affiliation Filter") - print("=" * 60) - - from uraas.config.institutions import get_registry - from uraas.pipelines.affiliation_filter import AffiliationFilterPipeline - - # Create mock spider for each institution - class MockSpider: - def __init__(self, institution): - self.institution = institution - self.logger = MockLogger() - - class MockLogger: - def info(self, msg): - pass - - def warning(self, msg): - pass - - def error(self, msg): - pass - - registry = get_registry() - institutions = ["unilag", "ui"] - - for inst in institutions: - config = registry.get(inst) - if not config: - continue - - print(f"\n{config.name}:") - - # Create pipeline - pipeline = AffiliationFilterPipeline() - spider = MockSpider(inst) - pipeline.open_spider(spider) - - print(f" Institution: {pipeline.current_institution.name}") - print(f" Staff count: {len(pipeline.current_validator.staff_names)}") - print(f" Patterns: {len(pipeline.current_patterns)}") - - # Test affiliation matching - test_texts = [ - (f"{config.name}, Nigeria", True), - (f"Department of Physics, {config.name}", True), - ("Random University", False), - ] - - print(f" Affiliation matching:") - for text, expected in test_texts: - result = pipeline.is_institution_affiliated(text) - status = "✓" if result == expected else "✗" - print(f" {status} '{text}' → {result}") - - return True - - -def test_ror_extraction(): - """Test ROR ID extraction from URLs""" - print("\n" + "=" * 60) - print("TEST: ROR ID Extraction") - print("=" * 60) - - test_cases = [ - ("https://ror.org/03qcnxw14", "03qcnxw14"), - ("https://ror.org/01js2sh04", "01js2sh04"), - ("https://ror.org/03yp73w09", "03yp73w09"), - ] - - all_passed = True - - for ror_url, expected_short in test_cases: - short = ror_url.split("/")[-1] - passed = short == expected_short - status = "✓" if passed else "✗" - print(f" {status} {ror_url} → {short}") - - if not passed: - all_passed = False - - return all_passed - - -def main(): - """Run all tests""" - print("\n" + "=" * 60) - print("MULTI-INSTITUTION CRAWL TEST SUITE") - print("=" * 60) - - tests = [ - ("Spider Initialization", test_spider_initialization), - ("Affiliation Filter", test_affiliation_filter), - ("ROR Extraction", test_ror_extraction), - ] - - results = {} - - for test_name, test_func in tests: - try: - result = test_func() - results[test_name] = result - except Exception as e: - print(f"\n✗ {test_name} FAILED: {e}") - import traceback - - traceback.print_exc() - results[test_name] = False - - # Final summary - print("\n" + "=" * 60) - print("FINAL SUMMARY") - print("=" * 60) - - passed = sum(1 for v in results.values() if v) - total = len(results) - - print(f"\nTests passed: {passed}/{total}\n") - - for test_name, success in results.items(): - status = "✓ PASS" if success else "✗ FAIL" - print(f" {status}: {test_name}") - - if passed == total: - print("\n✓ ALL TESTS PASSED") - print("\nReady for production crawling!") - print("\nNext steps:") - print( - " 1. Run: python crawl_multi_institution.py --institutions unilag,ui --target 10" - ) - print(" 2. Monitor database for new papers with ROR tags") - print(" 3. Verify multi-institution comparison in dashboard") - return 0 - else: - print("\n✗ SOME TESTS FAILED") - print("\nPlease fix issues before proceeding.") - return 1 - - -if __name__ == "__main__": - sys.exit(main()) +""" +Test multi-institution crawling functionality +Tests spider initialization and basic crawling for multiple institutions +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from uraas.config.institutions import get_registry +from uraas.spiders.sources.openalex_spider import OpenAlexSpider + + +def test_spider_initialization(): + """Test that spiders can be initialized with different institutions""" + print("\n" + "=" * 60) + print("TEST: Spider Initialization") + print("=" * 60) + + registry = get_registry() + institutions = ["unilag", "ui", "oau", "unn", "abu"] + + results = {} + + for inst in institutions: + try: + config = registry.get(inst) + if not config: + print(f"\n✗ {inst}: Configuration not found") + results[inst] = False + continue + + # Try to initialize spider + spider = OpenAlexSpider(institution=inst) + + print(f"\n✓ {inst}: {spider.institution_name}") + print(f" ROR: {spider.ror_id}") + print(f" ROR Short: {spider.ror_short}") + print(f" Staff count: {len(config.staff_names)}") + + results[inst] = True + + except Exception as e: + print(f"\n✗ {inst}: Failed to initialize - {e}") + results[inst] = False + + # Summary + print("\n" + "=" * 60) + print("INITIALIZATION SUMMARY") + print("=" * 60) + + passed = sum(1 for v in results.values() if v) + total = len(results) + + print(f"\nPassed: {passed}/{total}") + + for inst, success in results.items(): + status = "✓" if success else "✗" + print(f" {status} {inst}") + + return passed == total + + +def test_affiliation_filter(): + """Test affiliation filter with multi-institution support""" + print("\n" + "=" * 60) + print("TEST: Affiliation Filter") + print("=" * 60) + + from uraas.config.institutions import get_registry + from uraas.pipelines.affiliation_filter import AffiliationFilterPipeline + + # Create mock spider for each institution + class MockSpider: + def __init__(self, institution): + self.institution = institution + self.logger = MockLogger() + + class MockLogger: + def info(self, msg): + pass + + def warning(self, msg): + pass + + def error(self, msg): + pass + + registry = get_registry() + institutions = ["unilag", "ui"] + + for inst in institutions: + config = registry.get(inst) + if not config: + continue + + print(f"\n{config.name}:") + + # Create pipeline + pipeline = AffiliationFilterPipeline() + spider = MockSpider(inst) + pipeline.open_spider(spider) + + print(f" Institution: {pipeline.current_institution.name}") + print(f" Staff count: {len(pipeline.current_validator.staff_names)}") + print(f" Patterns: {len(pipeline.current_patterns)}") + + # Test affiliation matching + test_texts = [ + (f"{config.name}, Nigeria", True), + (f"Department of Physics, {config.name}", True), + ("Random University", False), + ] + + print(f" Affiliation matching:") + for text, expected in test_texts: + result = pipeline.is_institution_affiliated(text) + status = "✓" if result == expected else "✗" + print(f" {status} '{text}' → {result}") + + return True + + +def test_ror_extraction(): + """Test ROR ID extraction from URLs""" + print("\n" + "=" * 60) + print("TEST: ROR ID Extraction") + print("=" * 60) + + test_cases = [ + ("https://ror.org/03qcnxw14", "03qcnxw14"), + ("https://ror.org/01js2sh04", "01js2sh04"), + ("https://ror.org/03yp73w09", "03yp73w09"), + ] + + all_passed = True + + for ror_url, expected_short in test_cases: + short = ror_url.split("/")[-1] + passed = short == expected_short + status = "✓" if passed else "✗" + print(f" {status} {ror_url} → {short}") + + if not passed: + all_passed = False + + return all_passed + + +def main(): + """Run all tests""" + print("\n" + "=" * 60) + print("MULTI-INSTITUTION CRAWL TEST SUITE") + print("=" * 60) + + tests = [ + ("Spider Initialization", test_spider_initialization), + ("Affiliation Filter", test_affiliation_filter), + ("ROR Extraction", test_ror_extraction), + ] + + results = {} + + for test_name, test_func in tests: + try: + result = test_func() + results[test_name] = result + except Exception as e: + print(f"\n✗ {test_name} FAILED: {e}") + import traceback + + traceback.print_exc() + results[test_name] = False + + # Final summary + print("\n" + "=" * 60) + print("FINAL SUMMARY") + print("=" * 60) + + passed = sum(1 for v in results.values() if v) + total = len(results) + + print(f"\nTests passed: {passed}/{total}\n") + + for test_name, success in results.items(): + status = "✓ PASS" if success else "✗ FAIL" + print(f" {status}: {test_name}") + + if passed == total: + print("\n✓ ALL TESTS PASSED") + print("\nReady for production crawling!") + print("\nNext steps:") + print( + " 1. Run: python crawl_multi_institution.py --institutions unilag,ui --target 10" + ) + print(" 2. Monitor database for new papers with ROR tags") + print(" 3. Verify multi-institution comparison in dashboard") + return 0 + else: + print("\n✗ SOME TESTS FAILED") + print("\nPlease fix issues before proceeding.") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_new_features.py b/tests/test_new_features.py index c44c9fa3bca9c096d245186503cd9bbcc131a8d8..4451c9449c4729e2b06ec56d51cbc21b19954817 100644 --- a/tests/test_new_features.py +++ b/tests/test_new_features.py @@ -1,208 +1,208 @@ -""" -Test script for new Scopus-competitive features: -1. Citation tracking -2. H-index calculation -3. Advanced search with Boolean operators -""" - -import sys -import time - -from uraas.database import Author, Base, Item, SessionLocal, engine -from uraas.services.advanced_search import SearchQuery -from uraas.services.citation_tracker import ( - AuthorMetrics, - Citation, - CitationMetrics, - CitationTracker, - get_author_bibliometrics, - get_paper_citations, -) - -if __name__ == "__main__": - # Create new tables - print("=" * 70) - print("Creating citation tracking tables...") - print("=" * 70) - Base.metadata.create_all(bind=engine) - print("✓ Tables created\n") - - # Test 1: Citation Tracking - print("=" * 70) - print("TEST 1: Citation Tracking") - print("=" * 70) - - session = SessionLocal() - - # Find a paper with DOI - paper = session.query(Item).filter(Item.doi.isnot(None)).first() - - if paper: - print(f"\nTesting with paper: {paper.title[:60]}...") - print(f"DOI: {paper.doi}") - - print("\nFetching citations from OpenAlex...") - success = CitationTracker.update_paper_citations(paper.id) - - if success: - print("✓ Citations fetched successfully") - - # Get citation data - cite_data = get_paper_citations(paper.id) - print(f"\nCitation count: {cite_data['citation_count']}") - print(f"Citing papers in our DB: {len(cite_data['citing_papers'])}") - - if cite_data["citing_papers"]: - print("\nSample citing papers:") - for cite in cite_data["citing_papers"][:3]: - print(f" - {cite['title'][:60]}... ({cite['year']})") - else: - print("⚠ Citation fetch failed (paper may not be in OpenAlex)") - else: - print("⚠ No papers with DOI found in database") - - session.close() - - # Test 2: H-index Calculation - print("\n" + "=" * 70) - print("TEST 2: H-index Calculation") - print("=" * 70) - - session = SessionLocal() - - # Test h-index calculation - test_citations = [100, 50, 30, 20, 15, 10, 8, 5, 3, 2, 1, 1, 0, 0] - h_index = CitationTracker.calculate_h_index(test_citations) - print(f"\nTest citation counts: {test_citations}") - print(f"Calculated h-index: {h_index}") - print(f"Expected: 10 (10 papers with ≥10 citations)") - - # Find an author and calculate their metrics - author = session.query(Author).join(Author.items).first() - - if author: - print(f"\nTesting with author: {author.name}") - - # Update author metrics - success = CitationTracker.update_author_metrics(author.id) - - if success: - metrics = get_author_bibliometrics(author.id) - print(f"\nAuthor Bibliometrics:") - print(f" Total papers: {metrics.get('total_papers', 0)}") - print(f" Total citations: {metrics.get('total_citations', 0)}") - print(f" H-index: {metrics.get('h_index', 0)}") - print(f" i10-index: {metrics.get('i10_index', 0)}") - print(f" Citations per paper: {metrics.get('citations_per_paper', 0)}") - else: - print("⚠ Author metrics calculation failed (papers may lack citation data)") - - session.close() - - # Test 3: Advanced Search - print("\n" + "=" * 70) - print("TEST 3: Advanced Search with Boolean Operators") - print("=" * 70) - - # Test query parsing - test_queries = [ - "machine learning", - '"machine learning" AND author:smith', - "title:cancer NOT lung", - "author:okonkwo AND year:2020", - "(covid OR pandemic) AND faculty:medicine", - ] - - print("\nQuery Parsing Tests:") - for query in test_queries: - parsed = SearchQuery.parse_boolean_query(query) - print(f"\nQuery: {query}") - print(f"Parsed: {parsed}") - - # Test actual search execution - print("\n" + "=" * 70) - print("Search Execution Tests:") - print("=" * 70) - - # Test 1: Simple keyword search - print("\n1. Simple keyword search: 'health'") - results = SearchQuery.execute_search("health", limit=5) - print(f" Found {results['total']} papers in {results['took_ms']}ms") - if results["results"]: - print(f" Top result: {results['results'][0]['title'][:60]}...") - - # Test 2: Field-specific search - print("\n2. Field-specific search: 'year:2020'") - results = SearchQuery.execute_search("year:2020", limit=5) - print(f" Found {results['total']} papers from 2020") - - # Test 3: Boolean AND - print("\n3. Boolean AND: 'health AND education'") - results = SearchQuery.execute_search("health AND education", limit=5) - print(f" Found {results['total']} papers") - - # Test 4: Phrase search - print("\n4. Phrase search: '\"machine learning\"'") - results = SearchQuery.execute_search('"machine learning"', limit=5) - print(f" Found {results['total']} papers") - - # Test 5: Complex query - print("\n5. Complex query: 'author:okonkwo AND faculty:science'") - results = SearchQuery.execute_search("author:okonkwo AND faculty:science", limit=5) - print(f" Found {results['total']} papers") - - # Test 6: Sort by date - print("\n6. Sort by date: 'health' sorted by publication date") - results = SearchQuery.execute_search("health", limit=5, sort_by="date") - print(f" Found {results['total']} papers") - if results["results"]: - print( - f" Most recent: {results['results'][0]['title'][:60]}... ({results['results'][0]['year']})" - ) - - # Test autocomplete - print("\n" + "=" * 70) - print("Autocomplete Suggestions:") - print("=" * 70) - - test_partials = ["health", "machine", "science"] - for partial in test_partials: - suggestions = SearchQuery.get_search_suggestions(partial) - print(f"\n'{partial}' → {len(suggestions)} suggestions") - for sug in suggestions[:5]: - print(f" - {sug}") - - # Summary - print("\n" + "=" * 70) - print("FEATURE COMPARISON SUMMARY") - print("=" * 70) - - print("\n✓ IMPLEMENTED:") - print(" 1. Citation tracking (OpenAlex + Crossref APIs)") - print(" 2. H-index calculation (standard algorithm)") - print(" 3. i10-index (papers with 10+ citations)") - print(" 4. Author bibliometrics (total citations, papers, indices)") - print(" 5. Advanced search with Boolean operators (AND, OR, NOT)") - print(" 6. Field-specific queries (title:, author:, year:, faculty:, etc.)") - print(" 7. Phrase searches with quotes") - print(" 8. Multiple sort options (relevance, date, citations, title)") - print(" 9. Autocomplete suggestions") - print(" 10. Pagination support") - - print("\n⚠ LIMITATIONS vs Scopus:") - print(" - Scale: ~1K papers vs 80M+ (institutional focus)") - print(" - Citation data: Depends on OpenAlex coverage") - print(" - Update frequency: Weekly vs daily (configurable)") - print(" - Journal metrics: Not included (focus on institutional output)") - - print("\n✓ ADVANTAGES over Scopus:") - print(" - Zero false positives (staff validation)") - print(" - Free (no $40K/year subscription)") - print(" - Customizable (open source)") - print(" - African focus (indigenous knowledge metrics)") - print(" - Local PDF storage") - print(" - DocID™ persistent identifiers") - - print("\n" + "=" * 70) - print("Tests complete!") - print("=" * 70) +""" +Test script for new Scopus-competitive features: +1. Citation tracking +2. H-index calculation +3. Advanced search with Boolean operators +""" + +import sys +import time + +from uraas.database import Author, Base, Item, SessionLocal, engine +from uraas.services.advanced_search import SearchQuery +from uraas.services.citation_tracker import ( + AuthorMetrics, + Citation, + CitationMetrics, + CitationTracker, + get_author_bibliometrics, + get_paper_citations, +) + +if __name__ == "__main__": + # Create new tables + print("=" * 70) + print("Creating citation tracking tables...") + print("=" * 70) + Base.metadata.create_all(bind=engine) + print("✓ Tables created\n") + + # Test 1: Citation Tracking + print("=" * 70) + print("TEST 1: Citation Tracking") + print("=" * 70) + + session = SessionLocal() + + # Find a paper with DOI + paper = session.query(Item).filter(Item.doi.isnot(None)).first() + + if paper: + print(f"\nTesting with paper: {paper.title[:60]}...") + print(f"DOI: {paper.doi}") + + print("\nFetching citations from OpenAlex...") + success = CitationTracker.update_paper_citations(paper.id) + + if success: + print("✓ Citations fetched successfully") + + # Get citation data + cite_data = get_paper_citations(paper.id) + print(f"\nCitation count: {cite_data['citation_count']}") + print(f"Citing papers in our DB: {len(cite_data['citing_papers'])}") + + if cite_data["citing_papers"]: + print("\nSample citing papers:") + for cite in cite_data["citing_papers"][:3]: + print(f" - {cite['title'][:60]}... ({cite['year']})") + else: + print("⚠ Citation fetch failed (paper may not be in OpenAlex)") + else: + print("⚠ No papers with DOI found in database") + + session.close() + + # Test 2: H-index Calculation + print("\n" + "=" * 70) + print("TEST 2: H-index Calculation") + print("=" * 70) + + session = SessionLocal() + + # Test h-index calculation + test_citations = [100, 50, 30, 20, 15, 10, 8, 5, 3, 2, 1, 1, 0, 0] + h_index = CitationTracker.calculate_h_index(test_citations) + print(f"\nTest citation counts: {test_citations}") + print(f"Calculated h-index: {h_index}") + print(f"Expected: 10 (10 papers with ≥10 citations)") + + # Find an author and calculate their metrics + author = session.query(Author).join(Author.items).first() + + if author: + print(f"\nTesting with author: {author.name}") + + # Update author metrics + success = CitationTracker.update_author_metrics(author.id) + + if success: + metrics = get_author_bibliometrics(author.id) + print(f"\nAuthor Bibliometrics:") + print(f" Total papers: {metrics.get('total_papers', 0)}") + print(f" Total citations: {metrics.get('total_citations', 0)}") + print(f" H-index: {metrics.get('h_index', 0)}") + print(f" i10-index: {metrics.get('i10_index', 0)}") + print(f" Citations per paper: {metrics.get('citations_per_paper', 0)}") + else: + print("⚠ Author metrics calculation failed (papers may lack citation data)") + + session.close() + + # Test 3: Advanced Search + print("\n" + "=" * 70) + print("TEST 3: Advanced Search with Boolean Operators") + print("=" * 70) + + # Test query parsing + test_queries = [ + "machine learning", + '"machine learning" AND author:smith', + "title:cancer NOT lung", + "author:okonkwo AND year:2020", + "(covid OR pandemic) AND faculty:medicine", + ] + + print("\nQuery Parsing Tests:") + for query in test_queries: + parsed = SearchQuery.parse_boolean_query(query) + print(f"\nQuery: {query}") + print(f"Parsed: {parsed}") + + # Test actual search execution + print("\n" + "=" * 70) + print("Search Execution Tests:") + print("=" * 70) + + # Test 1: Simple keyword search + print("\n1. Simple keyword search: 'health'") + results = SearchQuery.execute_search("health", limit=5) + print(f" Found {results['total']} papers in {results['took_ms']}ms") + if results["results"]: + print(f" Top result: {results['results'][0]['title'][:60]}...") + + # Test 2: Field-specific search + print("\n2. Field-specific search: 'year:2020'") + results = SearchQuery.execute_search("year:2020", limit=5) + print(f" Found {results['total']} papers from 2020") + + # Test 3: Boolean AND + print("\n3. Boolean AND: 'health AND education'") + results = SearchQuery.execute_search("health AND education", limit=5) + print(f" Found {results['total']} papers") + + # Test 4: Phrase search + print("\n4. Phrase search: '\"machine learning\"'") + results = SearchQuery.execute_search('"machine learning"', limit=5) + print(f" Found {results['total']} papers") + + # Test 5: Complex query + print("\n5. Complex query: 'author:okonkwo AND faculty:science'") + results = SearchQuery.execute_search("author:okonkwo AND faculty:science", limit=5) + print(f" Found {results['total']} papers") + + # Test 6: Sort by date + print("\n6. Sort by date: 'health' sorted by publication date") + results = SearchQuery.execute_search("health", limit=5, sort_by="date") + print(f" Found {results['total']} papers") + if results["results"]: + print( + f" Most recent: {results['results'][0]['title'][:60]}... ({results['results'][0]['year']})" + ) + + # Test autocomplete + print("\n" + "=" * 70) + print("Autocomplete Suggestions:") + print("=" * 70) + + test_partials = ["health", "machine", "science"] + for partial in test_partials: + suggestions = SearchQuery.get_search_suggestions(partial) + print(f"\n'{partial}' → {len(suggestions)} suggestions") + for sug in suggestions[:5]: + print(f" - {sug}") + + # Summary + print("\n" + "=" * 70) + print("FEATURE COMPARISON SUMMARY") + print("=" * 70) + + print("\n✓ IMPLEMENTED:") + print(" 1. Citation tracking (OpenAlex + Crossref APIs)") + print(" 2. H-index calculation (standard algorithm)") + print(" 3. i10-index (papers with 10+ citations)") + print(" 4. Author bibliometrics (total citations, papers, indices)") + print(" 5. Advanced search with Boolean operators (AND, OR, NOT)") + print(" 6. Field-specific queries (title:, author:, year:, faculty:, etc.)") + print(" 7. Phrase searches with quotes") + print(" 8. Multiple sort options (relevance, date, citations, title)") + print(" 9. Autocomplete suggestions") + print(" 10. Pagination support") + + print("\n⚠ LIMITATIONS vs Scopus:") + print(" - Scale: ~1K papers vs 80M+ (institutional focus)") + print(" - Citation data: Depends on OpenAlex coverage") + print(" - Update frequency: Weekly vs daily (configurable)") + print(" - Journal metrics: Not included (focus on institutional output)") + + print("\n✓ ADVANTAGES over Scopus:") + print(" - Zero false positives (staff validation)") + print(" - Free (no $40K/year subscription)") + print(" - Customizable (open source)") + print(" - African focus (indigenous knowledge metrics)") + print(" - Local PDF storage") + print(" - DocID™ persistent identifiers") + + print("\n" + "=" * 70) + print("Tests complete!") + print("=" * 70) diff --git a/tests/test_production_ready.py b/tests/test_production_ready.py index 45fc513bf195ade1b40265f0ae83a9de82c14668..50406843833028b1739ae82d51c294d97a24736f 100644 --- a/tests/test_production_ready.py +++ b/tests/test_production_ready.py @@ -1,375 +1,375 @@ -""" -URAAS Production Readiness Test Suite -Comprehensive tests ensuring zero defects before deployment -""" - -import sys -from pathlib import Path - -import pytest - -sys.path.insert(0, str(Path(__file__).parent.parent)) - -from uraas.database import Author, Item, SessionLocal -from uraas.utils.ai_keyword_extractor import ai_extractor -from uraas.utils.staff_validator import staff_validator - - -class TestAIKeywordExtractor: - """Test AI keyword extraction system""" - - def test_keyword_extraction_basic(self): - """Test basic keyword extraction""" - text = "machine learning deep neural networks artificial intelligence" - keywords = ai_extractor.extract_keywords(text, top_n=5) - - assert len(keywords) > 0, "Should extract keywords" - assert all( - isinstance(k, tuple) and len(k) == 2 for k in keywords - ), "Keywords should be (word, score) tuples" - assert all( - 0 <= score <= 1 for _, score in keywords - ), "Scores should be between 0 and 1" - - def test_keyword_extraction_empty(self): - """Test with empty text""" - keywords = ai_extractor.extract_keywords("", top_n=5) - assert keywords == [], "Empty text should return no keywords" - - def test_domain_classification(self): - """Test domain classification""" - text = "algorithm data structure programming software engineering" - domains = ai_extractor.classify_domain(text) - - assert len(domains) > 0, "Should classify domains" - assert ( - domains[0][0] == "computer_science" - ), "Should identify computer science domain" - - def test_entity_extraction(self): - """Test entity extraction""" - text = "The study was conducted at University of Lagos using 50 mg of compound" - entities = ai_extractor.extract_entities(text) - - assert "organizations" in entities, "Should extract organizations" - assert "measurements" in entities, "Should extract measurements" - - def test_paper_scoring(self): - """Test paper quality scoring""" - title = "Machine Learning Applications in Medical Diagnosis" - abstract = ( - "This study investigates the application of machine learning " - "algorithms for medical diagnosis. We developed a novel approach " - "using deep neural networks to classify medical images. Our results " - "show significant improvement over existing methods." - ) - - score = ai_extractor.score_paper(title, abstract) - - assert "quality_score" in score, "Should have quality score" - assert 0 <= score["quality_score"] <= 1, "Quality score should be 0-1" - assert "keywords" in score, "Should have keywords" - assert "domains" in score, "Should have domains" - - -class TestDashboardUI: - """Test dashboard UI for emoji removal""" - - def test_dashboard_no_emojis(self): - """Verify dashboard HTML has no emojis in critical UI text""" - dashboard_path = Path("uraas/dashboard/templates/index.html") - assert dashboard_path.exists(), "Dashboard file should exist" - - content = dashboard_path.read_text(encoding="utf-8") - - # Check for common emoji patterns in non-optgroup areas (optgroups intentionally use flag emojis) - import re - - # Strip out optgroup labels before checking - stripped = re.sub(r']*label="[^"]*"[^>]*>', "", content) - emoji_pattern = r"[\U0001F680-\U0001F9FF]" # Rockets, charts, etc. - matches = re.findall(emoji_pattern, stripped) - assert len(matches) == 0, f"Found unexpected emojis: {matches[:5]}" - - def test_dashboard_minimalist_design(self): - """Verify dashboard uses consistent design tokens""" - dashboard_path = Path("uraas/dashboard/templates/index.html") - content = dashboard_path.read_text(encoding="utf-8") - - # Check for design system elements present in index.html - assert "Inter" in content, "Should use Inter font" - assert ( - "gradient-text" in content or "var(--accent)" in content - ), "Should use CSS design tokens" - - def test_dashboard_accessibility(self): - """Verify dashboard has accessibility features""" - dashboard_path = Path("uraas/dashboard/templates/index.html") - content = dashboard_path.read_text(encoding="utf-8") - - # Check for accessibility features - assert ( - "aria-" in content or "role=" in content or "title=" in content - ), "Should have ARIA attributes or title attributes" - assert "lang=" in content, "HTML element should have lang attribute" - - -class TestMetadataExtraction: - """Test metadata extraction accuracy""" - - def test_paper_metadata_completeness(self): - """Test that papers have complete metadata""" - session = SessionLocal() - try: - papers = session.query(Item).limit(10).all() - - for paper in papers: - assert paper.title, "Paper should have title" - assert paper.dc_title, "Paper should have Dublin Core title" - - # dc_identifier_doi stores the repository handle (OAI/DocID), - # while paper.doi stores the scholarly DOI — these are distinct fields. - # Both can coexist; just verify dc_identifier_doi is non-empty when doi is set. - if paper.doi and paper.dc_identifier_doi: - assert ( - len(paper.dc_identifier_doi) > 0 - ), "dc_identifier_doi should be non-empty when present" - finally: - session.close() - - def test_author_extraction_accuracy(self): - """Test author extraction""" - session = SessionLocal() - try: - papers = session.query(Item).filter(Item.authors.any()).limit(5).all() - - for paper in papers: - assert len(paper.authors) > 0, "Paper should have authors" - - for author in paper.authors: - assert author.name, "Author should have name" - assert author.normalized_name, "Author should have normalized name" - assert ( - author.normalized_name == author.name.lower().strip() - ), "Normalized name should be lowercase" - finally: - session.close() - - -class TestStaffValidation: - """Test staff validation accuracy""" - - def test_staff_cache_loaded(self): - """Verify staff cache can be loaded from the data directory""" - import os - - # The data file is always relative to the project root - staff_path = os.path.join( - os.path.dirname(__file__), "..", "data", "unilag_staff.json" - ) - assert os.path.exists( - staff_path - ), f"Staff data file should exist at {staff_path}" - # If staff_validator already loaded correctly, that is the best evidence - if len(staff_validator.staff_names) == 0: - # Reload using absolute path so tests pass regardless of working dir - from uraas.utils.staff_validator import StaffValidator - - sv = StaffValidator(staff_cache_path=os.path.abspath(staff_path)) - assert ( - len(sv.staff_names) > 0 - ), "Staff cache should load from data/unilag_staff.json" - - def test_exact_staff_match(self): - """Test exact staff matching""" - if staff_validator.staff_names: - test_name = list(staff_validator.staff_names)[0] - assert staff_validator.is_staff_member( - test_name, fuzzy_threshold=100 - ), "Exact match should work" - - def test_fuzzy_staff_match(self): - """Test fuzzy staff matching""" - if staff_validator.staff_names: - test_name = list(staff_validator.staff_names)[0] - # Slightly modify the name - modified = test_name.replace("a", "e", 1) if "a" in test_name else test_name - result = staff_validator.is_staff_member(modified, fuzzy_threshold=75) - assert isinstance(result, bool), "Should return boolean" - - -class TestAPIEndpoints: - """Test API endpoints""" - - @pytest.fixture - def client(self): - """Create test client""" - from uraas.dashboard.app import app - - app.config["TESTING"] = True - with app.test_client() as client: - yield client - - def test_dashboard_loads(self, client): - """Test dashboard loads""" - response = client.get("/") - assert response.status_code == 200, "Dashboard should load" - - def test_analytics_overview(self, client): - """Test analytics overview endpoint""" - response = client.get("/api/analytics/overview") - assert response.status_code == 200, "Analytics overview should work" - - data = response.get_json() - assert "total_papers" in data, "Should have total papers" - assert "total_authors" in data, "Should have total authors" - - def test_search_endpoint(self, client): - """Test search endpoint""" - response = client.get("/api/analytics/search?q=test&limit=10") - assert response.status_code == 200, "Search should work" - - data = response.get_json() - assert isinstance(data, list), "Search should return list" - - -class TestPerformance: - """Test performance requirements""" - - def test_keyword_extraction_speed(self): - """Test keyword extraction is fast""" - import time - - text = "machine learning deep neural networks artificial intelligence " * 10 - - start = time.time() - for _ in range(100): - ai_extractor.extract_keywords(text, top_n=10) - duration = time.time() - start - - avg_time = duration / 100 - assert ( - avg_time < 0.05 - ), f"Keyword extraction took {avg_time}s, should be < 0.05s" - - def test_domain_classification_speed(self): - """Test domain classification is fast""" - import time - - text = "algorithm data structure programming software engineering" * 5 - - start = time.time() - for _ in range(100): - ai_extractor.classify_domain(text) - duration = time.time() - start - - avg_time = duration / 100 - assert avg_time < 0.01, f"Classification took {avg_time}s, should be < 0.01s" - - -class TestSecurity: - """Test security measures""" - - def test_no_sql_injection_in_search(self): - """Test SQL injection protection""" - from uraas.dashboard.app import app - - app.config["TESTING"] = True - - with app.test_client() as client: - malicious_query = "'; DROP TABLE items; --" - response = client.get(f"/api/analytics/search?q={malicious_query}") - - # Should not crash - assert response.status_code in [200, 400], "Should handle malicious input" - - def test_xss_protection(self): - """Test XSS protection""" - session = SessionLocal() - try: - malicious_title = "" - item = Item( - title=malicious_title, dc_title=malicious_title, doi="10.test/xss.001" - ) - session.add(item) - session.commit() - - # Retrieve and verify - retrieved = session.query(Item).filter_by(doi="10.test/xss.001").first() - assert retrieved.title == malicious_title, "Should store as-is" - - # Cleanup - session.delete(retrieved) - session.commit() - finally: - session.close() - - -class TestCodeQuality: - """Test code quality""" - - def test_no_print_statements(self): - """Verify no debug print statements in production code""" - import os - import re - - production_dirs = ["uraas/dashboard", "uraas/utils", "uraas/pipelines"] - - # Files that legitimately use print() for IPC/stdout protocols - ALLOWED_FILES = { - os.path.normpath( - "uraas/pipelines/database.py" - ), # URAAS_DOWNLOAD: stdout IPC - } - - for dir_path in production_dirs: - for root, dirs, files in os.walk(dir_path): - for file in files: - if file.endswith(".py"): - filepath = os.path.join(root, file) - if os.path.normpath(filepath) in ALLOWED_FILES: - continue # Skip intentional stdout IPC files - with open( - filepath, "r", encoding="utf-8", errors="replace" - ) as f: - content = f.read() - # Check for debug print statements (not logging) - debug_prints = re.findall( - r"^\s*print\(", content, re.MULTILINE - ) - assert ( - len(debug_prints) == 0 - ), f"Found debug print statements in {filepath}" - - def test_imports_organized(self): - """Verify imports are organized""" - import os - - for root, dirs, files in os.walk("uraas"): - for file in files: - if file.endswith(".py"): - filepath = os.path.join(root, file) - with open(filepath, "r") as f: - lines = f.readlines() - - # Check that imports are at the top - import_section_ended = False - for i, line in enumerate(lines[:50]): - if line.strip() and not line.startswith( - ("import", "from", "#", '"""', "'''") - ): - import_section_ended = True - elif import_section_ended and line.startswith( - ("import", "from") - ): - # Imports after code is bad - pass # Allow for now - - -def run_all_tests(): - """Run all tests""" - pytest.main([__file__, "-v", "--tb=short", "--color=yes"]) - - -if __name__ == "__main__": - run_all_tests() +""" +URAAS Production Readiness Test Suite +Comprehensive tests ensuring zero defects before deployment +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from uraas.database import Author, Item, SessionLocal +from uraas.utils.ai_keyword_extractor import ai_extractor +from uraas.utils.staff_validator import staff_validator + + +class TestAIKeywordExtractor: + """Test AI keyword extraction system""" + + def test_keyword_extraction_basic(self): + """Test basic keyword extraction""" + text = "machine learning deep neural networks artificial intelligence" + keywords = ai_extractor.extract_keywords(text, top_n=5) + + assert len(keywords) > 0, "Should extract keywords" + assert all( + isinstance(k, tuple) and len(k) == 2 for k in keywords + ), "Keywords should be (word, score) tuples" + assert all( + 0 <= score <= 1 for _, score in keywords + ), "Scores should be between 0 and 1" + + def test_keyword_extraction_empty(self): + """Test with empty text""" + keywords = ai_extractor.extract_keywords("", top_n=5) + assert keywords == [], "Empty text should return no keywords" + + def test_domain_classification(self): + """Test domain classification""" + text = "algorithm data structure programming software engineering" + domains = ai_extractor.classify_domain(text) + + assert len(domains) > 0, "Should classify domains" + assert ( + domains[0][0] == "computer_science" + ), "Should identify computer science domain" + + def test_entity_extraction(self): + """Test entity extraction""" + text = "The study was conducted at University of Lagos using 50 mg of compound" + entities = ai_extractor.extract_entities(text) + + assert "organizations" in entities, "Should extract organizations" + assert "measurements" in entities, "Should extract measurements" + + def test_paper_scoring(self): + """Test paper quality scoring""" + title = "Machine Learning Applications in Medical Diagnosis" + abstract = ( + "This study investigates the application of machine learning " + "algorithms for medical diagnosis. We developed a novel approach " + "using deep neural networks to classify medical images. Our results " + "show significant improvement over existing methods." + ) + + score = ai_extractor.score_paper(title, abstract) + + assert "quality_score" in score, "Should have quality score" + assert 0 <= score["quality_score"] <= 1, "Quality score should be 0-1" + assert "keywords" in score, "Should have keywords" + assert "domains" in score, "Should have domains" + + +class TestDashboardUI: + """Test dashboard UI for emoji removal""" + + def test_dashboard_no_emojis(self): + """Verify dashboard HTML has no emojis in critical UI text""" + dashboard_path = Path("uraas/dashboard/templates/index.html") + assert dashboard_path.exists(), "Dashboard file should exist" + + content = dashboard_path.read_text(encoding="utf-8") + + # Check for common emoji patterns in non-optgroup areas (optgroups intentionally use flag emojis) + import re + + # Strip out optgroup labels before checking + stripped = re.sub(r']*label="[^"]*"[^>]*>', "", content) + emoji_pattern = r"[\U0001F680-\U0001F9FF]" # Rockets, charts, etc. + matches = re.findall(emoji_pattern, stripped) + assert len(matches) == 0, f"Found unexpected emojis: {matches[:5]}" + + def test_dashboard_minimalist_design(self): + """Verify dashboard uses consistent design tokens""" + dashboard_path = Path("uraas/dashboard/templates/index.html") + content = dashboard_path.read_text(encoding="utf-8") + + # Check for design system elements present in index.html + assert "Inter" in content, "Should use Inter font" + assert ( + "gradient-text" in content or "var(--accent)" in content + ), "Should use CSS design tokens" + + def test_dashboard_accessibility(self): + """Verify dashboard has accessibility features""" + dashboard_path = Path("uraas/dashboard/templates/index.html") + content = dashboard_path.read_text(encoding="utf-8") + + # Check for accessibility features + assert ( + "aria-" in content or "role=" in content or "title=" in content + ), "Should have ARIA attributes or title attributes" + assert "lang=" in content, "HTML element should have lang attribute" + + +class TestMetadataExtraction: + """Test metadata extraction accuracy""" + + def test_paper_metadata_completeness(self): + """Test that papers have complete metadata""" + session = SessionLocal() + try: + papers = session.query(Item).limit(10).all() + + for paper in papers: + assert paper.title, "Paper should have title" + assert paper.dc_title, "Paper should have Dublin Core title" + + # dc_identifier_doi stores the repository handle (OAI/DocID), + # while paper.doi stores the scholarly DOI — these are distinct fields. + # Both can coexist; just verify dc_identifier_doi is non-empty when doi is set. + if paper.doi and paper.dc_identifier_doi: + assert ( + len(paper.dc_identifier_doi) > 0 + ), "dc_identifier_doi should be non-empty when present" + finally: + session.close() + + def test_author_extraction_accuracy(self): + """Test author extraction""" + session = SessionLocal() + try: + papers = session.query(Item).filter(Item.authors.any()).limit(5).all() + + for paper in papers: + assert len(paper.authors) > 0, "Paper should have authors" + + for author in paper.authors: + assert author.name, "Author should have name" + assert author.normalized_name, "Author should have normalized name" + assert ( + author.normalized_name == author.name.lower().strip() + ), "Normalized name should be lowercase" + finally: + session.close() + + +class TestStaffValidation: + """Test staff validation accuracy""" + + def test_staff_cache_loaded(self): + """Verify staff cache can be loaded from the data directory""" + import os + + # The data file is always relative to the project root + staff_path = os.path.join( + os.path.dirname(__file__), "..", "data", "unilag_staff.json" + ) + assert os.path.exists( + staff_path + ), f"Staff data file should exist at {staff_path}" + # If staff_validator already loaded correctly, that is the best evidence + if len(staff_validator.staff_names) == 0: + # Reload using absolute path so tests pass regardless of working dir + from uraas.utils.staff_validator import StaffValidator + + sv = StaffValidator(staff_cache_path=os.path.abspath(staff_path)) + assert ( + len(sv.staff_names) > 0 + ), "Staff cache should load from data/unilag_staff.json" + + def test_exact_staff_match(self): + """Test exact staff matching""" + if staff_validator.staff_names: + test_name = list(staff_validator.staff_names)[0] + assert staff_validator.is_staff_member( + test_name, fuzzy_threshold=100 + ), "Exact match should work" + + def test_fuzzy_staff_match(self): + """Test fuzzy staff matching""" + if staff_validator.staff_names: + test_name = list(staff_validator.staff_names)[0] + # Slightly modify the name + modified = test_name.replace("a", "e", 1) if "a" in test_name else test_name + result = staff_validator.is_staff_member(modified, fuzzy_threshold=75) + assert isinstance(result, bool), "Should return boolean" + + +class TestAPIEndpoints: + """Test API endpoints""" + + @pytest.fixture + def client(self): + """Create test client""" + from uraas.dashboard.app import app + + app.config["TESTING"] = True + with app.test_client() as client: + yield client + + def test_dashboard_loads(self, client): + """Test dashboard loads""" + response = client.get("/") + assert response.status_code == 200, "Dashboard should load" + + def test_analytics_overview(self, client): + """Test analytics overview endpoint""" + response = client.get("/api/analytics/overview") + assert response.status_code == 200, "Analytics overview should work" + + data = response.get_json() + assert "total_papers" in data, "Should have total papers" + assert "total_authors" in data, "Should have total authors" + + def test_search_endpoint(self, client): + """Test search endpoint""" + response = client.get("/api/analytics/search?q=test&limit=10") + assert response.status_code == 200, "Search should work" + + data = response.get_json() + assert isinstance(data, list), "Search should return list" + + +class TestPerformance: + """Test performance requirements""" + + def test_keyword_extraction_speed(self): + """Test keyword extraction is fast""" + import time + + text = "machine learning deep neural networks artificial intelligence " * 10 + + start = time.time() + for _ in range(100): + ai_extractor.extract_keywords(text, top_n=10) + duration = time.time() - start + + avg_time = duration / 100 + assert ( + avg_time < 0.05 + ), f"Keyword extraction took {avg_time}s, should be < 0.05s" + + def test_domain_classification_speed(self): + """Test domain classification is fast""" + import time + + text = "algorithm data structure programming software engineering" * 5 + + start = time.time() + for _ in range(100): + ai_extractor.classify_domain(text) + duration = time.time() - start + + avg_time = duration / 100 + assert avg_time < 0.01, f"Classification took {avg_time}s, should be < 0.01s" + + +class TestSecurity: + """Test security measures""" + + def test_no_sql_injection_in_search(self): + """Test SQL injection protection""" + from uraas.dashboard.app import app + + app.config["TESTING"] = True + + with app.test_client() as client: + malicious_query = "'; DROP TABLE items; --" + response = client.get(f"/api/analytics/search?q={malicious_query}") + + # Should not crash + assert response.status_code in [200, 400], "Should handle malicious input" + + def test_xss_protection(self): + """Test XSS protection""" + session = SessionLocal() + try: + malicious_title = "" + item = Item( + title=malicious_title, dc_title=malicious_title, doi="10.test/xss.001" + ) + session.add(item) + session.commit() + + # Retrieve and verify + retrieved = session.query(Item).filter_by(doi="10.test/xss.001").first() + assert retrieved.title == malicious_title, "Should store as-is" + + # Cleanup + session.delete(retrieved) + session.commit() + finally: + session.close() + + +class TestCodeQuality: + """Test code quality""" + + def test_no_print_statements(self): + """Verify no debug print statements in production code""" + import os + import re + + production_dirs = ["uraas/dashboard", "uraas/utils", "uraas/pipelines"] + + # Files that legitimately use print() for IPC/stdout protocols + ALLOWED_FILES = { + os.path.normpath( + "uraas/pipelines/database.py" + ), # URAAS_DOWNLOAD: stdout IPC + } + + for dir_path in production_dirs: + for root, dirs, files in os.walk(dir_path): + for file in files: + if file.endswith(".py"): + filepath = os.path.join(root, file) + if os.path.normpath(filepath) in ALLOWED_FILES: + continue # Skip intentional stdout IPC files + with open( + filepath, "r", encoding="utf-8", errors="replace" + ) as f: + content = f.read() + # Check for debug print statements (not logging) + debug_prints = re.findall( + r"^\s*print\(", content, re.MULTILINE + ) + assert ( + len(debug_prints) == 0 + ), f"Found debug print statements in {filepath}" + + def test_imports_organized(self): + """Verify imports are organized""" + import os + + for root, dirs, files in os.walk("uraas"): + for file in files: + if file.endswith(".py"): + filepath = os.path.join(root, file) + with open(filepath, "r") as f: + lines = f.readlines() + + # Check that imports are at the top + import_section_ended = False + for i, line in enumerate(lines[:50]): + if line.strip() and not line.startswith( + ("import", "from", "#", '"""', "'''") + ): + import_section_ended = True + elif import_section_ended and line.startswith( + ("import", "from") + ): + # Imports after code is bad + pass # Allow for now + + +def run_all_tests(): + """Run all tests""" + pytest.main([__file__, "-v", "--tb=short", "--color=yes"]) + + +if __name__ == "__main__": + run_all_tests() diff --git a/tests/test_staff_res.py b/tests/test_staff_res.py index 2f8c6b9bbff2d443fe3d89d026aeca0c68e14865..bbb4e7da2dd08335b79eff71b7a6ed779dcbdbce 100644 --- a/tests/test_staff_res.py +++ b/tests/test_staff_res.py @@ -1,12 +1,12 @@ -import os - -from uraas.config.institutions import get_registry - -if __name__ == "__main__": - registry = get_registry() - for inst in registry.list_all(): - print( - f"{inst.short_name}: {len(inst.staff_names)} names, resolved path: {inst._resolve_staff_file()}" - ) - if not os.path.exists(inst._resolve_staff_file()): - print(f" ERROR: Path does not exist!") +import os + +from uraas.config.institutions import get_registry + +if __name__ == "__main__": + registry = get_registry() + for inst in registry.list_all(): + print( + f"{inst.short_name}: {len(inst.staff_names)} names, resolved path: {inst._resolve_staff_file()}" + ) + if not os.path.exists(inst._resolve_staff_file()): + print(f" ERROR: Path does not exist!") diff --git a/uraas/analytics/engine.py b/uraas/analytics/engine.py index 939134beaaf0eeba2b24d5ca51f811a932f2b997..cab8444be8b13b3ad1f5c1e1b03d474807fdb482 100644 --- a/uraas/analytics/engine.py +++ b/uraas/analytics/engine.py @@ -1,1981 +1,1981 @@ -""" -URAAS Analytics Engine -Implements all APA Intelligence & Analytics Platform metrics: - - Standard repository analytics (papers, authors, faculties, OA) - - TK Vitality Score (indigenous knowledge health index) - - Linguistic Diversity Index (African vs colonial language output) - - Patent-to-Paper Velocity (innovation lifecycle timing) - - Multi-institution Comparator (ROR-based benchmarking) - - SDG Alignment (UN Sustainable Development Goals) — AI-powered via spaCy - - Keyword Cloud (AI-extracted terms) - - Collaboration Network (D3 force graph data) - - Special Collections (African Literature, Indigenous Knowledge, etc.) -""" - -import itertools -import logging -import re -from collections import defaultdict -from datetime import datetime -from typing import Dict, List, Optional, Tuple - -from sqlalchemy import desc, extract, func, or_ -from sqlalchemy.orm import aliased, joinedload - -from uraas.config.institutions import get_registry -from uraas.database import ( - Author, - Collection, - Community, - File, - Item, - SessionLocal, - db_year, - db_year_month, -) -from uraas.services.sc_engine import SC_FILTER, category_breakdown, is_special_collection -from uraas.utils.ai_classifier import ( - SPECIAL_COLLECTIONS, - extract_keywords, - extract_trends_from_corpus, -) -from uraas.utils.analytics_cache import analytics_cache -from uraas.utils.unilag_classifier import classifier - -logger = logging.getLogger(__name__) - -# African language codes (kept here for Linguistic Diversity Index) -AFRICAN_LANG_CODES = { - "yo": "Yoruba", - "ig": "Igbo", - "ha": "Hausa", - "sw": "Swahili", - "am": "Amharic", - "so": "Somali", - "rw": "Kinyarwanda", - "sn": "Shona", - "zu": "Zulu", - "xh": "Xhosa", - "af": "Afrikaans", - "st": "Sesotho", - "tn": "Setswana", - "ts": "Tsonga", - "ss": "Swati", - "ve": "Venda", - "nr": "Ndebele", - "ff": "Fula", - "wo": "Wolof", - "bm": "Bambara", - "ln": "Lingala", - "kg": "Kongo", - "lua": "Luba", - "om": "Oromo", -} - -# Content type weights for TK Vitality Score -TK_WEIGHTS = { - "indigenous_knowledge": 3.0, - "cultural_heritage": 2.5, - "oral_tradition": 2.5, - "grey_literature": 1.5, - "thesis": 1.2, - "dataset": 1.2, - "patent": 1.0, - "research_paper": 0.5, -} - -# Common stop words for legacy keyword code -STOP_WORDS = { - "the", - "and", - "for", - "with", - "this", - "that", - "from", - "have", - "been", - "were", - "their", - "which", - "these", - "about", - "other", - "into", - "than", - "more", - "such", - "some", - "what", - "when", - "where", - "there", - "also", - "using", - "used", - "study", - "show", - "paper", - "research", - "analysis", - "findings", - "results", - "between", - "effect", - "impact", - "based", - "data", - "method", - "approach", - "model", - "system", - "review", - "case", - "report", - "among", - "within", - "across", - "during", - "after", - "before", - "through", - "while", - "both", - "each", - "only", - "very", - "well", - "high", - "low", - "new", - "large", - "small", - "significant", - "different", - "similar", - "total", - "however", - "therefore", - "thus", - "hence", - "although", - "despite", -} - - -class URAASAnalyticsEngine: - """ - Observer Engine for the APA Intelligence & Analytics Platform. - All methods return plain dicts/lists no ORM objects leak out. - """ - - # Helpers - - def _resolve_institution_name(self, identifier: Optional[str]) -> Optional[str]: - """Maps short name or ROR to full institution name from registry.""" - if not identifier: - return None - reg = get_registry() - inst = reg.get(identifier) - return inst.name if inst else identifier - - @staticmethod - def _is_oa(item: Item) -> bool: - return "openAccess" in (item.dc_rights or "") - - @staticmethod - def _year(item: Item) -> Optional[int]: - return item.publication_date.year if item.publication_date else None - - def _get_sc_item_ids(self, session, institution: Optional[str] = None) -> List[int]: - """ - Returns a list of Item IDs that are Special Collections. - - The authoritative SC signal is the stored ``special_collection_score`` - column (computed by the SC decision engine at crawl time / via the - re-classify script). This is a cheap indexed query — no per-row - re-classification — so the dashboard count updates immediately after a - crawl or prune (once the analytics cache is flushed). - """ - inst_name = self._resolve_institution_name(institution) - cache_key = f"sc_item_ids_{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - q = session.query(Item.id).filter(SC_FILTER) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - valid_ids = [row[0] for row in q.all()] - - analytics_cache.set(cache_key, valid_ids, ttl=3600) # cache for 1 hour - return valid_ids - - # Standard repository analytics - - def get_top_authors( - self, - limit: int = 15, - community_id: Optional[int] = None, - institution: Optional[str] = None, - ) -> List[Dict]: - inst_name = self._resolve_institution_name(institution) - session = SessionLocal() - try: - q = session.query( - Author.name, - Author.orcid, - Author.ror, - func.count(Item.id).label("count"), - ).join(Author.items).filter(SC_FILTER) - if community_id: - q = ( - q.join(Item.collections) - .join(Collection.community) - .filter(Community.id == community_id) - ) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - rows = ( - q.group_by(Author.name, Author.orcid, Author.ror) - .order_by(desc("count")) - .limit(limit) - .all() - ) - return [ - {"author": r[0], "orcid": r[1] or "", "ror": r[2] or "", "count": r[3]} - for r in rows - ] - except Exception as e: - logger.error("get_top_authors: %s", e) - return [] - finally: - session.close() - - def get_department_collaboration_network( - self, institution: Optional[str] = None - ) -> List[Dict]: - inst_name = self._resolve_institution_name(institution) - session = SessionLocal() - try: - q = session.query(Item).filter(SC_FILTER) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - docs = ( - q - .options(joinedload(Item.collections)) - .all() - ) - edges: Dict[Tuple, int] = {} - for doc in docs: - colls = sorted(c.name for c in doc.collections if c and c.name) - for pair in itertools.combinations(colls, 2): - edges[pair] = edges.get(pair, 0) + 1 - return [ - {"source": k[0], "target": k[1], "weight": v} for k, v in edges.items() - ] - except Exception as e: - logger.error("get_department_collaboration_network: %s", e) - return [] - finally: - session.close() - - def get_papers_by_faculty_and_department( - self, institution: Optional[str] = None - ) -> Dict: - inst_name = self._resolve_institution_name(institution) - session = SessionLocal() - try: - communities = ( - session.query(Community) - .options(joinedload(Community.collections)) - .all() - ) - tree: Dict = {} - seen: set = set() - - # Batch-fetch all File records once to avoid an O(N) query per paper. - file_map: Dict[int, object] = { - row.item_id: row - for row in session.query(File).all() - } - - for comm in communities: - dept_map: Dict = {} - for coll in comm.collections: - q = ( - session.query(Item) - .join(Item.collections) - .filter(Collection.id == coll.id, SC_FILTER) - ) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - - papers = q.all() - paper_list = [] - for p in papers: - seen.add(p.id) - f = file_map.get(p.id) - paper_list.append( - { - "id": p.id, - "title": p.title or "Untitled", - "doi": p.doi or "", - "url": p.url or "", - "docid": p.docid or "", - "has_local_pdf": f is not None, - "access_policy": f.access_policy if f else None, - "download_url": ( - f"/api/papers/{p.id}/download" if f else None - ), - } - ) - if paper_list: - dept_map[coll.name] = paper_list - if dept_map: - tree[comm.name] = dept_map - - # Unclassified bucket - unclassified_q = ( - session.query(Item).filter(SC_FILTER, ~Item.id.in_(seen)) - if seen - else session.query(Item).filter(SC_FILTER) - ) - if inst_name: - unclassified_q = unclassified_q.filter( - Item.institution.ilike(f"%{inst_name}%") - ) - unclassified = unclassified_q.all() - if unclassified: - tree["Unclassified"] = { - "General": [ - { - "id": p.id, - "title": p.title or "Untitled", - "doi": p.doi or "", - "url": p.url or "", - "docid": p.docid or "", - "has_local_pdf": False, - } - for p in unclassified - ] - } - return tree - except Exception as e: - logger.error("get_papers_by_faculty_and_department: %s", e) - return {} - finally: - session.close() - - - def get_publications_by_year(self, institution: Optional[str] = None) -> List[Dict]: - inst_name = self._resolve_institution_name(institution) - session = SessionLocal() - try: - q = session.query(db_year(Item.publication_date), func.count(Item.id)).filter(SC_FILTER) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - q = ( - q.filter(Item.publication_date.isnot(None)) - .group_by(db_year(Item.publication_date)) - .order_by(db_year(Item.publication_date)) - ) - return [{"year": int(r[0]) if r[0] else 0, "count": r[1]} for r in q.all()] - finally: - session.close() - - def get_papers_by_year_faculty( - self, institution: Optional[str] = None - ) -> List[Dict]: - inst_name = self._resolve_institution_name(institution) - session = SessionLocal() - try: - q = ( - session.query( - db_year(Item.publication_date), Community.name, func.count(Item.id) - ) - .join(Item.collections) - .join(Collection.community) - .filter(SC_FILTER) - ) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - q = q.filter(Item.publication_date.isnot(None)).group_by( - db_year(Item.publication_date), Community.name - ) - return [ - {"year": int(r[0]) if r[0] else 0, "faculty": r[1], "count": r[2]} - for r in q.all() - ] - finally: - session.close() - - def get_papers_by_faculty(self, institution: Optional[str] = None) -> List[Dict]: - inst_name = self._resolve_institution_name(institution) - session = SessionLocal() - try: - q = ( - session.query(Community.name, func.count(Item.id)) - .join(Item.collections) - .join(Collection.community) - .filter(SC_FILTER) - ) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - q = q.group_by(Community.name).order_by(desc(func.count(Item.id))) - return [{"faculty": r[0], "count": r[1]} for r in q.all()] - finally: - session.close() - - def get_open_access_breakdown( - self, institution: Optional[str] = None - ) -> List[Dict]: - inst_name = self._resolve_institution_name(institution) - session = SessionLocal() - try: - q = session.query(Item.dc_rights).filter(SC_FILTER) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - items = q.all() - counts = {"Open Access": 0, "Restricted": 0} - for it in items: - if "openAccess" in (it[0] or ""): - counts["Open Access"] += 1 - else: - counts["Restricted"] += 1 - return [{"label": k, "value": v} for k, v in counts.items()] - finally: - session.close() - - def get_authors_by_papers( - self, limit: int = 10, institution: Optional[str] = None - ) -> List[Dict]: - inst_name = self._resolve_institution_name(institution) - session = SessionLocal() - try: - q = session.query(Author.name, func.count(Item.id)).join(Author.items).filter(SC_FILTER) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - q = q.group_by(Author.name).order_by(desc(func.count(Item.id))).limit(limit) - return [{"author": r[0], "count": r[1]} for r in q.all()] - finally: - session.close() - - def get_faculty_oa_breakdown(self, institution: Optional[str] = None) -> List[Dict]: - inst_name = self._resolve_institution_name(institution) - session = SessionLocal() - try: - q = ( - session.query(Community.name, Item.dc_rights) - .join(Item.collections) - .join(Collection.community) - .filter(SC_FILTER) - ) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - rows = q.all() - facs = defaultdict(lambda: {"oa": 0, "restricted": 0}) - for fac, rights in rows: - if "openAccess" in (rights or ""): - facs[fac]["oa"] += 1 - else: - facs[fac]["restricted"] += 1 - return [ - {"faculty": k, "oa": v["oa"], "restricted": v["restricted"]} - for k, v in facs.items() - ] - finally: - session.close() - - def get_institutional_growth(self, institution: Optional[str] = None) -> List[Dict]: - inst_name = self._resolve_institution_name(institution) - session = SessionLocal() - try: - q = session.query(db_year_month(Item.created_at), func.count(Item.id)).filter(SC_FILTER) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - q = q.group_by(db_year_month(Item.created_at)).order_by( - db_year_month(Item.created_at) - ) - return [{"month": r[0], "count": r[1]} for r in q.all()] - finally: - session.close() - - def get_timeline_data(self, institution: Optional[str] = None) -> List[Dict]: - inst_name = self._resolve_institution_name(institution) - session = SessionLocal() - try: - q = session.query(func.date(Item.created_at), func.count(Item.id)).filter(SC_FILTER) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - q = q.group_by(func.date(Item.created_at)).order_by( - func.date(Item.created_at) - ) - res = [] - total = 0 - for r in q.all(): - total += r[1] - res.append({"date": r[0], "count": r[1], "total": total}) - return res - finally: - session.close() - - # SDG Alignment (AI-powered, cached) - - def get_sdg_alignment(self, institution: Optional[str] = None) -> List[Dict]: - """ - Score every paper against all 17 SDGs using AI. - Results are cached for 30 minutes per institution. - """ - inst_name = self._resolve_institution_name(institution) - cache_key = f"sdg_alignment:{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - session = SessionLocal() - try: - q = session.query(Item.id, Item.title, Item.abstract).filter(SC_FILTER) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - items = q.all() - - # Pre-populate buckets for SDG 1 to 17 - sdg_buckets = {n: [] for n in range(1, 18)} - sdg_names_full = {} - for item_id, title, abstract in items: - text_corpus = f"{title or ''} {abstract or ''}" - hits = classifier.detect_sdg_alignment(text_corpus) - for hit in hits: - sdg_str = hit["sdg"] # e.g. "SDG 1 — No Poverty" - try: - num = int(re.search(r"SDG (\d+)", sdg_str).group(1)) - sdg_names_full[num] = sdg_str - sdg_buckets[num].append( - { - "id": item_id, - "title": title, - "score": hit["score"], - "keywords": hit["matched_keywords"], - } - ) - except (AttributeError, ValueError): - continue - - result = [] - for sdg_num, papers in sdg_buckets.items(): - if papers: - papers.sort(key=lambda x: -x["score"]) - result.append( - { - "sdg": sdg_names_full.get(sdg_num, f"SDG {sdg_num}"), - "sdg_number": sdg_num, - "count": len(papers), - "papers": papers[:10], - } - ) - result.sort(key=lambda x: -x["count"]) - analytics_cache.set(cache_key, result) - return result - except Exception as e: - logger.error("get_sdg_alignment: %s", e) - return [] - finally: - session.close() - - def get_sdg_csv_data(self) -> List[List]: - """ - Returns SDG alignment data as rows for CSV export. - Header: [SDG Number, SDG Name, Paper Count, Paper Title, Score, Matched Keywords] - """ - rows = [ - [ - "SDG Number", - "SDG Name", - "Paper Count", - "Paper Title", - "Score", - "Matched Keywords", - ] - ] - try: - alignment = self.get_sdg_alignment() - for entry in alignment: - sdg_num = entry.get("sdg_number", "") - sdg_name_full = entry.get("sdg", "") - count = entry.get("count", 0) - for paper in entry.get("papers", []): - rows.append( - [ - sdg_num, - sdg_name_full, - count, - paper.get("title", ""), - paper.get("score", 0), - "; ".join(paper.get("keywords", [])), - ] - ) - except Exception as e: - logger.error("get_sdg_csv_data: %s", e) - return rows - - # Keyword Cloud (corpus-level TF-IDF, cached) - - def get_keyword_cloud( - self, top_n: int = 60, institution: Optional[str] = None - ) -> List[Dict]: - """Extract top keywords using corpus-level TF-IDF + spaCy NER. Cached 30 min.""" - inst_name = self._resolve_institution_name(institution) - cache_key = f"keyword_cloud:{inst_name or 'all'}:{top_n}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - session = SessionLocal() - try: - q = session.query(Item.title, Item.abstract).filter(SC_FILTER) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - items = q.all() - - # Build corpus for IDF calculation - all_texts = [f"{t or ''} {a or ''}" for t, a in items] - combined_title = " ".join(t or "" for t, _ in items) - combined_abstract = " ".join(a or "" for _, a in items) - - keywords = extract_keywords( - combined_title, combined_abstract, top_n=top_n, all_texts=all_texts - ) - result = [ - {"word": k["word"], "count": k["count"], "score": k["score"]} - for k in keywords - ] - analytics_cache.set(cache_key, result) - return result - except Exception as e: - logger.error("get_keyword_cloud: %s", e) - return [] - finally: - session.close() - - # APA Novel Metrics - - def get_institution_leaderboard(self) -> List[Dict]: - """Cross-institution leaderboard ranked across key metrics.""" - cache_key = "institution_leaderboard" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - session = SessionLocal() - try: - # Get distinct institutions in the DB - inst_rows = ( - session.query(Item.institution, func.count(Item.id).label("total")) - .filter(Item.institution.isnot(None)) - .group_by(Item.institution) - .all() - ) - - leaderboard = [] - for inst_name, total in inst_rows: - oa = ( - session.query(Item) - .filter( - Item.institution == inst_name, - Item.dc_rights.like("%openAccess%"), - ) - .count() - ) - authors = ( - session.query(func.count(func.distinct(Author.id))) - .join(Author.items) - .filter(Item.institution == inst_name) - .scalar() - or 0 - ) - - oa_rate = round(oa / total * 100, 1) if total else 0 - leaderboard.append( - { - "institution": inst_name, - "total_papers": total, - "open_access": oa, - "oa_rate": oa_rate, - "unique_authors": authors, - "score": round(total * 0.4 + oa_rate * 0.4 + authors * 0.2, 1), - } - ) - - leaderboard.sort(key=lambda x: -x["score"]) - for i, inst in enumerate(leaderboard): - inst["rank"] = i + 1 - - analytics_cache.set(cache_key, leaderboard) - return leaderboard - except Exception as e: - logger.error("get_institution_leaderboard: %s", e) - return [] - finally: - session.close() - - def get_tk_vitality_score(self, institution: Optional[str] = None) -> Dict: - """ - TK Vitality Score measures how well the institution is digitising - indigenous knowledge and cultural heritage. - - Score = weighted sum of content types / total items * 100 - Max theoretical score = 100 (all items are indigenous knowledge) - """ - inst_name = self._resolve_institution_name(institution) - cache_key = f"tk_vitality:{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - session = SessionLocal() - try: - q = session.query(Item.content_type, Item.tk_label, Item.dc_type).filter(SC_FILTER) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - items = q.all() - total = len(items) - if total == 0: - return {"score": 0, "breakdown": {}, "total_items": 0, "tk_items": 0} - - type_counts: Dict[str, int] = defaultdict(int) - weighted_sum = 0.0 - tk_items = 0 - - for content_type, tk_label, dc_type in items: - ct = content_type or "research_paper" - # Upgrade type if TK label is present - if tk_label: - ct = "indigenous_knowledge" - tk_items += 1 - elif dc_type and "cultural" in (dc_type or "").lower(): - ct = "cultural_heritage" - tk_items += 1 - - type_counts[ct] += 1 - weighted_sum += TK_WEIGHTS.get(ct, 0.5) - - max_possible = total * TK_WEIGHTS["indigenous_knowledge"] - score = round((weighted_sum / max_possible) * 100, 1) if max_possible else 0 - - result = { - "score": score, - "breakdown": dict(type_counts), - "total_items": total, - "tk_items": tk_items, - "tk_percentage": round(tk_items / total * 100, 1) if total else 0, - "interpretation": ( - "Excellent" - if score >= 60 - else "Good" if score >= 30 else "Developing" - ), - } - analytics_cache.set(cache_key, result) - return result - except Exception as e: - logger.error("get_tk_vitality_score: %s", e) - return {"score": 0, "breakdown": {}, "total_items": 0, "tk_items": 0} - finally: - session.close() - - def get_linguistic_diversity_index(self) -> Dict: - """ - Linguistic Diversity Index % of outputs in African languages vs English/French. - Supports the decolonisation of knowledge mission. - """ - session = SessionLocal() - try: - items = session.query( - Item.language_code, Item.is_african_language, Item.dc_language - ).all() - total = len(items) - if total == 0: - return {"index": 0, "african_count": 0, "total": 0, "breakdown": {}} - - lang_counts: Dict[str, int] = defaultdict(int) - african_count = 0 - - for lang_code, is_african, dc_lang in items: - code = lang_code or dc_lang or "en" - lang_counts[code] += 1 - if is_african or code in AFRICAN_LANG_CODES: - african_count += 1 - - index = round(african_count / total * 100, 1) - - # Build human-readable breakdown - breakdown = {} - for code, count in sorted(lang_counts.items(), key=lambda x: -x[1]): - label = AFRICAN_LANG_CODES.get(code, code.upper()) - breakdown[label] = count - - return { - "index": index, - "african_count": african_count, - "colonial_count": total - african_count, - "total": total, - "breakdown": breakdown, - "top_african_languages": [ - {"language": AFRICAN_LANG_CODES.get(c, c), "code": c, "count": n} - for c, n in sorted(lang_counts.items(), key=lambda x: -x[1]) - if c in AFRICAN_LANG_CODES - ][:10], - } - except Exception as e: - logger.error("get_linguistic_diversity_index: %s", e) - return {"index": 0, "african_count": 0, "total": 0, "breakdown": {}} - finally: - session.close() - - def get_special_collections_metrics( - self, institution: Optional[str] = None - ) -> Dict: - """ - Special Collections Metrics using AI classifier. Cached 30 min. - """ - inst_name = self._resolve_institution_name(institution) - cache_key = f"special_collections:{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - session = SessionLocal() - try: - q = session.query(Item.id, Item.title, Item.abstract, Item.dc_subject) - if inst_name: - q = q.filter(Item.institution.ilike(f"%{inst_name}%")) - sc_ids = self._get_sc_item_ids(session, institution) - if sc_ids: - q = q.filter(Item.id.in_(sc_ids)) - else: - return { - "summary": [], - "total_special_items": 0, - "total_repository_items": q.count(), - } - items = q.all() - results: Dict[str, List] = {cat: [] for cat in SPECIAL_COLLECTIONS} - - for item_id, title, abstract, dc_subject in items: - cats = category_breakdown( - title or "", abstract or "", dc_subject or "" - ) - for cat_result in cats: - cat = cat_result["category"] - if cat in results: - results[cat].append( - { - "id": item_id, - "title": title, - "matches": cat_result["matched_keywords"], - "count": cat_result["score"], - } - ) - - summary = [] - for category, papers in results.items(): - papers.sort(key=lambda x: -x["count"]) - summary.append( - { - "category": category, - "count": len(papers), - "top_papers": papers[:10], - } - ) - - summary.sort(key=lambda x: -x["count"]) - result = { - "summary": summary, - "total_special_items": sum(len(p) for p in results.values()), - "total_repository_items": len(items), - } - analytics_cache.set(cache_key, result) - return result - except Exception as e: - logger.error("get_special_collections_metrics: %s", e) - return { - "summary": [], - "total_special_items": 0, - "total_repository_items": 0, - } - finally: - session.close() - - def get_special_collections_overview( - self, institution: Optional[str] = None - ) -> Dict: - """ - Special Collections Overview analytics — purpose-built for indigenous - knowledge / African literature / cultural-heritage collections rather - than the citation-prestige lens of commercial bibliometric platforms. - - Returns thematic composition + theme co-occurrence, knowledge - sovereignty (African custodianship), SDG/development alignment, top - institutional custodians, a cultural lexicon, and the most influential - works. All metrics are restricted to genuine Special Collections via the - stored ``special_collection_score`` gate (``_get_sc_item_ids``) and are - computed only over fields populated for SC items. Cached 30 min. - """ - from uraas.config.african_countries import COUNTRY_NAMES - - inst_name = self._resolve_institution_name(institution) - cache_key = f"sc_overview:{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - canonical = list(SPECIAL_COLLECTIONS.keys()) - empty = { - "kpis": { - "total_items": 0, - "repo_total": 0, - "pct_of_repo": 0, - "themes_represented": 0, - "total_citations": 0, - "intra_african_pct": 0, - }, - "themes": [{"category": c, "count": 0} for c in canonical], - "co_occurrence": {"labels": canonical, "matrix": [[0] * len(canonical) for _ in canonical]}, - "countries": [], - "sdgs": [], - "keywords": [], - "custodians": [], - "influential": [], - } - - # Build institution name → ISO2 code for country-fallback when - # coauthor_countries is empty (papers with no cross-institution data). - _name_to_iso2 = {v.lower(): k for k, v in COUNTRY_NAMES.items()} - try: - from uraas.config import get_registry as _get_reg - _inst_country: Dict[str, str] = { - cfg.name: _name_to_iso2.get(cfg.country.lower(), "") - for cfg in _get_reg().list_all() - } - except Exception: - _inst_country = {} - - session = SessionLocal() - try: - repo_q = session.query(func.count(Item.id)) - if inst_name: - repo_q = repo_q.filter(Item.institution.ilike(f"%{inst_name}%")) - repo_total = repo_q.scalar() or 0 - - sc_ids = self._get_sc_item_ids(session, institution) - if not sc_ids: - empty["kpis"]["repo_total"] = repo_total - return empty - - rows = ( - session.query( - Item.id, - Item.title, - Item.cited_by_count, - Item.special_collection_categories, - Item.coauthor_countries, - Item.is_intra_african, - Item.sdg_tags, - Item.ai_keywords, - Item.institution, - Item.special_collection_score, - ) - .filter(Item.id.in_(sc_ids)) - .all() - ) - - cat_idx = {c: i for i, c in enumerate(canonical)} - theme_counts = {c: 0 for c in canonical} - matrix = [[0] * len(canonical) for _ in canonical] - country_counts: Dict[str, int] = defaultdict(int) - sdg_counts: Dict[str, int] = defaultdict(int) - kw_counts: Dict[str, int] = defaultdict(int) - custodian_counts: Dict[str, int] = defaultdict(int) - total_citations = 0 - intra_african = 0 - influential: List[Dict] = [] - - for ( - item_id, - title, - cited, - categories, - countries, - is_intra, - sdgs, - keywords, - inst, - sc_score, - ) in rows: - cited = int(cited or 0) - total_citations += cited - if is_intra: - intra_african += 1 - - # Themes (canonical, multi-valued) + co-occurrence matrix. - item_cats = [ - c.strip() - for c in (categories or "").split(",") - if c.strip() in cat_idx - ] - item_cats = sorted(set(item_cats)) - for c in item_cats: - theme_counts[c] += 1 - if len(item_cats) == 1: - i = cat_idx[item_cats[0]] - matrix[i][i] += 1 - else: - for a, b in itertools.combinations(item_cats, 2): - ia, ib = cat_idx[a], cat_idx[b] - matrix[ia][ib] += 1 - matrix[ib][ia] += 1 - - # Knowledge sovereignty — contributing African countries. - _any_country = False - for code in (countries or "").split(","): - code = code.strip().upper() - if code in COUNTRY_NAMES: - country_counts[code] += 1 - _any_country = True - if not _any_country and inst: - _fallback = _inst_country.get(inst, "") - if _fallback: - country_counts[_fallback] += 1 - - # SDG alignment. - for sdg in (sdgs or "").split(","): - sdg = sdg.strip() - if sdg: - sdg_counts[sdg] += 1 - - # Cultural lexicon. - for kw in (keywords or "").split(","): - kw = kw.strip().lower() - if len(kw) > 2: - kw_counts[kw] += 1 - - # Custodian institutions. - if inst: - custodian_counts[inst] += 1 - - # Collect all SC works for the influential list. - # Ranked by cited_by_count first; sc_score is the tiebreaker - # so the list is always populated even before citation backfill. - influential.append( - { - "id": item_id, - "title": title or "Untitled", - "citations": cited, - "sc_score": float(sc_score or 0), - "categories": item_cats, - } - ) - - total_items = len(rows) - themes_represented = sum(1 for v in theme_counts.values() if v > 0) - influential.sort(key=lambda x: (-x["citations"], -x["sc_score"])) - - result = { - "kpis": { - "total_items": total_items, - "repo_total": repo_total, - "pct_of_repo": round(total_items / repo_total * 100, 1) - if repo_total - else 0, - "themes_represented": themes_represented, - "total_citations": total_citations, - "intra_african_pct": round(intra_african / total_items * 100, 1) - if total_items - else 0, - }, - "themes": [ - {"category": c, "count": theme_counts[c]} for c in canonical - ], - "co_occurrence": {"labels": canonical, "matrix": matrix}, - "countries": [ - {"code": code, "name": COUNTRY_NAMES.get(code, code), "papers": n} - for code, n in sorted(country_counts.items(), key=lambda x: -x[1]) - ], - "sdgs": [ - {"sdg": sdg, "count": n} - for sdg, n in sorted( - sdg_counts.items(), - key=lambda x: -x[1], - ) - ], - "keywords": [ - {"word": w, "count": n} - for w, n in sorted(kw_counts.items(), key=lambda x: -x[1])[:30] - ], - "custodians": [ - {"institution": inst, "count": n} - for inst, n in sorted( - custodian_counts.items(), key=lambda x: -x[1] - )[:10] - ], - "influential": influential[:15], - } - # Sort themes descending by count for display. - result["themes"].sort(key=lambda x: -x["count"]) - analytics_cache.set(cache_key, result) - return result - except Exception as e: - logger.error("get_special_collections_overview: %s", e) - return empty - finally: - session.close() - - def get_special_collections_csv_data(self) -> List[List]: - """Returns special collections data as rows for CSV export.""" - rows = [["Category", "Paper Count", "Paper Title", "Score", "Matched Keywords"]] - try: - metrics = self.get_special_collections_metrics() - for entry in metrics.get("summary", []): - for paper in entry.get("top_papers", []): - rows.append( - [ - entry.get("category", ""), - entry.get("count", 0), - paper.get("title", ""), - paper.get("count", 0), - "; ".join(paper.get("matches", [])), - ] - ) - except Exception as e: - logger.error("get_special_collections_csv_data: %s", e) - return rows - - def get_author_network( - self, author_name: Optional[str] = None, limit: int = 30 - ) -> Dict: - """ - Collaboration network for D3 force graph. - If author_name given: ego-network for that researcher. - Otherwise: top-N authors by paper count. - """ - session = SessionLocal() - try: - if author_name: - items = ( - session.query(Item) - .join(Item.authors) - .filter(Author.name == author_name) - .options(joinedload(Item.authors)) - .all() - ) - edges: Dict[Tuple, int] = {} - for item in items: - names = [a.name for a in item.authors] - if author_name not in names: - continue - for name in names: - if name != author_name: - key = tuple(sorted([author_name, name])) - edges[key] = edges.get(key, 0) + 1 - - edge_list = sorted( - [ - {"source": k[0], "target": k[1], "weight": v} - for k, v in edges.items() - ], - key=lambda x: -x["weight"], - )[:15] - - collaborators = {e["source"] for e in edge_list} | { - e["target"] for e in edge_list - } - collaborators.discard(author_name) - - return { - "nodes": [{"id": author_name, "count": len(items)}] - + [{"id": c, "count": 0} for c in collaborators], - "edges": edge_list, - } - else: - # Global top-N network - top_rows = ( - session.query(Author.name, func.count(Item.id).label("cnt")) - .join(Author.items) - .group_by(Author.name) - .order_by(desc("cnt")) - .limit(limit) - .all() - ) - top_names = {r[0] for r in top_rows} - node_counts = {r[0]: r[1] for r in top_rows} - - items = session.query(Item).options(joinedload(Item.authors)).all() - edges: Dict[Tuple, int] = {} - for item in items: - names = [a.name for a in item.authors if a.name in top_names] - for pair in itertools.combinations(sorted(names), 2): - edges[pair] = edges.get(pair, 0) + 1 - - return self._annotate_network( - { - "nodes": [ - {"id": n, "count": node_counts[n]} for n in top_names - ], - "edges": [ - {"source": k[0], "target": k[1], "weight": v} - for k, v in edges.items() - ], - } - ) - except Exception as e: - logger.error("get_author_network: %s", e) - return {"nodes": [], "edges": []} - finally: - session.close() - - @staticmethod - def _annotate_network(graph: Dict) -> Dict: - """Add Louvain community + degree centrality to network nodes. - - Gephi/VOSviewer-standard visual grammar: color by community, size by - centrality. Deterministic (seed=42) so the layout story is stable.""" - try: - import networkx as nx - - G = nx.Graph() - G.add_nodes_from(n["id"] for n in graph["nodes"]) - G.add_weighted_edges_from( - (e["source"], e["target"], e["weight"]) for e in graph["edges"] - ) - if G.number_of_edges(): - communities = nx.community.louvain_communities( - G, weight="weight", seed=42 - ) - membership = { - node: idx for idx, comm in enumerate(communities) for node in comm - } - centrality = nx.degree_centrality(G) - for n in graph["nodes"]: - n["community"] = membership.get(n["id"], 0) - n["centrality"] = round(centrality.get(n["id"], 0.0), 3) - except Exception as e: - logger.warning("_annotate_network: %s", e) - return graph - - # Intra-African collaboration analytics - - # Continental baseline: share of African co-publications involving >=2 - # African countries, 2002-2019 (Research Policy, 2022). - INTRA_AFRICAN_BASELINE_PCT = 8.4 - - def get_collaboration_overview(self, institution: Optional[str] = None) -> Dict: - """Intra-African Collaboration Index — Scimago-compatible definition - (works whose affiliations span >=2 distinct African countries), - benchmarked against the continental average.""" - inst_name = self._resolve_institution_name(institution) - cache_key = f"collab_overview_{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - session = SessionLocal() - try: - sc_ids = self._get_sc_item_ids(session, institution) - if not sc_ids: - return {"total_papers": 0, "papers_with_affiliation_data": 0} - - from uraas.config.african_countries import COUNTRY_NAMES - from uraas.database import ItemAffiliation - - covered = { - i - for (i,) in session.query(ItemAffiliation.item_id) - .filter(ItemAffiliation.item_id.in_(sc_ids)) - .distinct() - } - intra_count = ( - session.query(func.count(Item.id)) - .filter(Item.id.in_(sc_ids), Item.is_intra_african.is_(True)) - .scalar() - or 0 - ) - denominator = len(covered) - pct = round(intra_count / denominator * 100, 1) if denominator else 0.0 - - # Per-country paper counts (partners), excluding none - country_rows = ( - session.query( - ItemAffiliation.country_code, - func.count(func.distinct(ItemAffiliation.item_id)), - ) - .filter( - ItemAffiliation.item_id.in_(sc_ids), - ItemAffiliation.country_code.in_(list(COUNTRY_NAMES)), - ) - .group_by(ItemAffiliation.country_code) - .all() - ) - home_codes = set() - if inst_name: - # Home country dominates counts; surface partners separately. - reg_inst = get_registry().get(institution or "") - if reg_inst: - home_codes = { - code - for code, name in COUNTRY_NAMES.items() - if name.lower() == reg_inst.country.lower() - } - partners = sorted( - ( - {"code": c, "name": COUNTRY_NAMES.get(c, c), "count": n} - for c, n in country_rows - if c not in home_codes - ), - key=lambda r: -r["count"], - ) - - result = { - "total_papers": len(sc_ids), - "papers_with_affiliation_data": denominator, - "intra_african_count": intra_count, - "intra_african_pct": pct, - "baseline_pct": self.INTRA_AFRICAN_BASELINE_PCT, - "ratio_vs_baseline": ( - round(pct / self.INTRA_AFRICAN_BASELINE_PCT, 2) if pct else 0.0 - ), - "country_count": len(country_rows), - "top_partner_countries": partners[:10], - } - analytics_cache.set(cache_key, result, ttl=1800) - return result - except Exception as e: - logger.error("get_collaboration_overview: %s", e) - return {"total_papers": 0, "papers_with_affiliation_data": 0} - finally: - session.close() - - def get_country_pair_matrix(self, institution: Optional[str] = None) -> Dict: - """Unordered African country-pair co-publication counts (full counting).""" - inst_name = self._resolve_institution_name(institution) - cache_key = f"collab_pairs_{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - session = SessionLocal() - try: - sc_ids = self._get_sc_item_ids(session, institution) - if not sc_ids: - return {"pairs": [], "countries": []} - - from uraas.config.african_countries import COUNTRY_NAMES - from uraas.database import ItemAffiliation - - a1 = aliased(ItemAffiliation) - a2 = aliased(ItemAffiliation) - rows = ( - session.query( - a1.country_code, - a2.country_code, - func.count(func.distinct(a1.item_id)), - ) - .join(a2, a1.item_id == a2.item_id) - .filter( - a1.country_code < a2.country_code, - a1.country_code.in_(list(COUNTRY_NAMES)), - a2.country_code.in_(list(COUNTRY_NAMES)), - a1.item_id.in_(sc_ids), - ) - .group_by(a1.country_code, a2.country_code) - .all() - ) - pairs = sorted( - ( - { - "source": s, - "target": t, - "source_name": COUNTRY_NAMES.get(s, s), - "target_name": COUNTRY_NAMES.get(t, t), - "count": n, - } - for s, t, n in rows - ), - key=lambda p: -p["count"], - ) - countries = sorted({p["source"] for p in pairs} | {p["target"] for p in pairs}) - result = {"pairs": pairs, "countries": countries} - analytics_cache.set(cache_key, result, ttl=1800) - return result - except Exception as e: - logger.error("get_country_pair_matrix: %s", e) - return {"pairs": [], "countries": []} - finally: - session.close() - - def get_country_aggregates(self, institution: Optional[str] = None) -> List[Dict]: - """Per-African-country paper counts for the choropleth.""" - inst_name = self._resolve_institution_name(institution) - cache_key = f"collab_countries_{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - session = SessionLocal() - try: - sc_ids = self._get_sc_item_ids(session, institution) - if not sc_ids: - return [] - - from uraas.config.african_countries import COUNTRY_NAMES - from uraas.database import ItemAffiliation - - rows = ( - session.query( - ItemAffiliation.country_code, - func.count(func.distinct(ItemAffiliation.item_id)), - ) - .filter( - ItemAffiliation.item_id.in_(sc_ids), - ItemAffiliation.country_code.in_(list(COUNTRY_NAMES)), - ) - .group_by(ItemAffiliation.country_code) - .all() - ) - intra = dict( - session.query( - ItemAffiliation.country_code, - func.count(func.distinct(ItemAffiliation.item_id)), - ) - .join(Item, Item.id == ItemAffiliation.item_id) - .filter( - ItemAffiliation.item_id.in_(sc_ids), - Item.is_intra_african.is_(True), - ItemAffiliation.country_code.in_(list(COUNTRY_NAMES)), - ) - .group_by(ItemAffiliation.country_code) - .all() - ) - result = sorted( - ( - { - "code": c, - "name": COUNTRY_NAMES.get(c, c), - "papers": n, - "intra_african": intra.get(c, 0), - } - for c, n in rows - ), - key=lambda r: -r["papers"], - ) - analytics_cache.set(cache_key, result, ttl=1800) - return result - except Exception as e: - logger.error("get_country_aggregates: %s", e) - return [] - finally: - session.close() - - def get_citation_velocity(self, institution: Optional[str] = None) -> Dict: - """Repository-level citation velocity from stored counts_by_year, plus - citation-weighted Pan-African citation share.""" - import json as _json - - inst_name = self._resolve_institution_name(institution) - cache_key = f"citation_velocity_{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - session = SessionLocal() - try: - sc_ids = self._get_sc_item_ids(session, institution) - if not sc_ids: - return {"by_year": [], "items_covered": 0} - - rows = ( - session.query( - Item.counts_by_year, - Item.cited_by_count, - Item.african_citation_share, - Item.publication_date, - ) - .filter(Item.id.in_(sc_ids), Item.counts_by_year.isnot(None)) - .all() - ) - by_year: Dict[int, int] = {} - first2y: List[int] = [] - share_weighted = share_weight = 0.0 - share_covered = 0 - for counts_json, cited, share, pub_date in rows: - try: - counts = _json.loads(counts_json) - except Exception: - continue - year_map = { - c["year"]: c.get("cited_by_count", 0) - for c in counts - if isinstance(c, dict) and "year" in c - } - for y, n in year_map.items(): - by_year[y] = by_year.get(y, 0) + n - if pub_date: - first2y.append( - year_map.get(pub_date.year, 0) - + year_map.get(pub_date.year + 1, 0) - ) - if share is not None: - share_covered += 1 - weight = max(cited or 0, 1) - share_weighted += share * weight - share_weight += weight - - result = { - "by_year": [ - {"year": y, "citations": by_year[y]} for y in sorted(by_year) - ], - "items_covered": len(rows), - "avg_first2y": ( - round(sum(first2y) / len(first2y), 1) if first2y else 0.0 - ), - "pan_african_share_pct": ( - round(share_weighted / share_weight, 1) if share_weight else None - ), - "pan_african_share_items": share_covered, - } - analytics_cache.set(cache_key, result, ttl=1800) - return result - except Exception as e: - logger.error("get_citation_velocity: %s", e) - return {"by_year": [], "items_covered": 0} - finally: - session.close() - - def get_collaboration_arcs(self, institution: Optional[str] = None) -> Dict: - """GeoJSON FeatureCollection of great-circle arcs for country pairs.""" - from uraas.config.african_countries import COUNTRY_CENTROIDS - from uraas.utils.geo import great_circle_arc - - matrix = self.get_country_pair_matrix(institution) - features = [] - for pair in matrix["pairs"]: - src = COUNTRY_CENTROIDS.get(pair["source"]) - dst = COUNTRY_CENTROIDS.get(pair["target"]) - if not (src and dst): - continue - features.append( - { - "type": "Feature", - "properties": { - "source": pair["source"], - "target": pair["target"], - "source_name": pair["source_name"], - "target_name": pair["target_name"], - "count": pair["count"], - }, - "geometry": { - "type": "LineString", - "coordinates": great_circle_arc( - src[0], src[1], dst[0], dst[1] - ), - }, - } - ) - return {"type": "FeatureCollection", "features": features} - - - def get_pid_coverage(self, institution: Optional[str] = None) -> Dict: - """Return PID coverage statistics — key metric for PID Alliance audiences. - - Reports the fraction of Special Collections items carrying each - persistent identifier type: DOI, ORCID (via any author), ARK, ROR. - """ - session = SessionLocal() - try: - sc_ids = self._get_sc_item_ids(session, institution) - total = len(sc_ids) - if not total: - return { - "total": 0, - "doi_count": 0, "doi_pct": 0.0, - "orcid_count": 0, "orcid_pct": 0.0, - "ark_count": 0, "ark_pct": 0.0, - "ror_count": 0, "ror_pct": 0.0, - } - - doi_count = ( - session.query(func.count(Item.id)) - .filter(Item.id.in_(sc_ids), Item.doi.isnot(None), Item.doi != "") - .scalar() - ) or 0 - - ark_count = ( - session.query(func.count(Item.id)) - .filter(Item.id.in_(sc_ids), Item.ark.isnot(None), Item.ark != "") - .scalar() - ) or 0 - - ror_count = ( - session.query(func.count(Item.id)) - .filter(Item.id.in_(sc_ids), Item.ror.isnot(None), Item.ror != "") - .scalar() - ) or 0 - - # Items where at least one author has an ORCID - orcid_item_ids = ( - session.query(Item.id) - .join(Item.authors) - .filter( - Item.id.in_(sc_ids), - Author.orcid.isnot(None), - Author.orcid != "", - ) - .distinct() - .all() - ) - orcid_count = len(orcid_item_ids) - - def _pct(n: int) -> float: - return round(100.0 * n / total, 1) if total else 0.0 - - return { - "total": total, - "doi_count": doi_count, "doi_pct": _pct(doi_count), - "orcid_count": orcid_count, "orcid_pct": _pct(orcid_count), - "ark_count": ark_count, "ark_pct": _pct(ark_count), - "ror_count": ror_count, "ror_pct": _pct(ror_count), - } - except Exception as e: - logger.error("get_pid_coverage: %s", e) - return {"total": 0, "doi_count": 0, "doi_pct": 0.0, - "orcid_count": 0, "orcid_pct": 0.0, - "ark_count": 0, "ark_pct": 0.0, - "ror_count": 0, "ror_pct": 0.0} - finally: - session.close() - - - # ── Knowledge Repatriation Index ───────────────────────────────────────── - - def get_knowledge_repatriation(self, institution: Optional[str] = None) -> Dict: - """Knowledge Repatriation Index — measures whether Africa leads its own - research rather than being a subject of externally-led studies. - - Segments the corpus into three collaboration profiles: - • Africa-Led : only African institutions on the paper (sole or pan-African) - • North-South : African institution + ≥1 non-African institution - • Unclassified : no affiliation data available - - The repatriation score (0-100) = Africa-Led / (Africa-Led + North-South) × 100. - Higher = Africa owns more of its own knowledge production. - No other analytics platform surfaces this metric. Cached 30 min. - """ - from uraas.config.african_countries import COUNTRY_NAMES - from uraas.database import ItemAffiliation - - inst_name = self._resolve_institution_name(institution) - cache_key = f"knowledge_repatriation:{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - session = SessionLocal() - try: - sc_ids = self._get_sc_item_ids(session, institution) - total = len(sc_ids) - if not total: - return {"score": 0, "africa_led": 0, "north_south": 0, - "unclassified": 0, "total": 0, "trend": []} - - african_codes = set(COUNTRY_NAMES.keys()) - - # Items that have any affiliation record - aff_items = { - item_id - for (item_id,) in session.query(ItemAffiliation.item_id) - .filter(ItemAffiliation.item_id.in_(sc_ids)) - .distinct() - } - - africa_led = 0 - north_south = 0 - - for item_id in aff_items: - countries = { - row[0].upper() - for row in session.query(ItemAffiliation.country_code) - .filter(ItemAffiliation.item_id == item_id, - ItemAffiliation.country_code.isnot(None)) - .all() - if row[0] - } - if not countries: - continue - non_african = countries - african_codes - if non_african: - north_south += 1 - else: - africa_led += 1 - - # Fall back to coauthor_countries for items without affiliation records - no_aff = [i for i in sc_ids if i not in aff_items] - no_country_ids = set() - for row in (session.query(Item.id, Item.coauthor_countries) - .filter(Item.id.in_(no_aff)) - .all()): - codes_raw = (row[1] or "").strip() - if not codes_raw: - no_country_ids.add(row[0]) - continue - codes = {c.strip().upper() for c in codes_raw.split(",") if c.strip()} - if not codes: - no_country_ids.add(row[0]) - continue - if codes - african_codes: - north_south += 1 - else: - africa_led += 1 - - # Items with no coauthor_countries and no ItemAffiliation were crawled - # from African institutions — classify as Africa-Led (best-effort). - africa_led += len(no_country_ids) - - classified = africa_led + north_south - unclassified = total - classified - score = round(africa_led / classified * 100, 1) if classified else 0.0 - - # Year-over-year trend (last 5 years of classified papers) - trend_rows = ( - session.query(db_year(Item.publication_date), - Item.is_intra_african, - func.count(Item.id)) - .filter(Item.id.in_(sc_ids), - Item.publication_date.isnot(None), - Item.coauthor_countries.isnot(None)) - .group_by(db_year(Item.publication_date), Item.is_intra_african) - .order_by(db_year(Item.publication_date)) - .all() - ) - by_year: Dict[str, Dict] = {} - for yr, intra, cnt in trend_rows: - if not yr: - continue - yr = str(yr) - by_year.setdefault(yr, {"year": yr, "africa_led": 0, "north_south": 0}) - if intra: - by_year[yr]["africa_led"] += cnt - else: - by_year[yr]["north_south"] += cnt - trend = sorted(by_year.values(), key=lambda x: x["year"])[-8:] - - result = { - "score": score, - "africa_led": africa_led, - "north_south": north_south, - "unclassified": unclassified, - "total": total, - "trend": trend, - "interpretation": ( - "Strong" if score >= 60 - else "Moderate" if score >= 35 - else "Dependent" - ), - } - analytics_cache.set(cache_key, result, ttl=1800) - return result - except Exception as e: - logger.error("get_knowledge_repatriation: %s", e) - return {"score": 0, "africa_led": 0, "north_south": 0, - "unclassified": 0, "total": 0, "trend": []} - finally: - session.close() - - # ── Research Portfolio Diversity Score ──────────────────────────────────── - - def get_research_diversity(self, institution: Optional[str] = None) -> Dict: - """Research Portfolio Diversity Score — Shannon entropy across SDGs. - - A diversified research portfolio is more resilient to funding shifts - and more likely to support cross-cutting African development goals. - - Score (0-100): 100 = perfectly uniform across all 17 SDGs. - Compared against a continental peer baseline of ~45 (estimated from - OpenAlex African institution SDG distributions). - No competitor surfaces this as an institutional resilience metric. - Cached 30 min. - """ - import math - - inst_name = self._resolve_institution_name(institution) - cache_key = f"research_diversity:{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - session = SessionLocal() - try: - sc_ids = self._get_sc_item_ids(session, institution) - if not sc_ids: - return {"score": 0, "sdg_distribution": [], "dominant_sdg": None, - "gap_sdgs": [], "interpretation": "No data"} - - rows = (session.query(Item.sdg_tags) - .filter(Item.id.in_(sc_ids), Item.sdg_tags.isnot(None)) - .all()) - - sdg_counts: Dict[str, int] = defaultdict(int) - for (tags,) in rows: - for tag in (tags or "").split(","): - tag = tag.strip() - if tag: - sdg_counts[tag] += 1 - - if not sdg_counts: - return {"score": 0, "sdg_distribution": [], "dominant_sdg": None, - "gap_sdgs": [], "interpretation": "No SDG data"} - - # Shannon entropy normalised to 0-100 - total_tags = sum(sdg_counts.values()) - probs = [c / total_tags for c in sdg_counts.values()] - entropy = -sum(p * math.log(p) for p in probs if p > 0) - max_entropy = math.log(17) # perfectly uniform across 17 SDGs - score = round(entropy / max_entropy * 100, 1) - - # Gap SDGs: SDGs 1-17 not represented at all - represented = {int(k.split()[1]) for k in sdg_counts if k.startswith("SDG") and len(k.split()) > 1} - gap_sdgs = [f"SDG {i}" for i in range(1, 18) if i not in represented] - - dominant = max(sdg_counts, key=sdg_counts.get) if sdg_counts else None - distribution = sorted( - [{"sdg": k, "count": v} for k, v in sdg_counts.items()], - key=lambda x: -x["count"] - )[:17] - - result = { - "score": score, - "sdg_distribution": distribution, - "dominant_sdg": dominant, - "gap_sdgs": gap_sdgs, - "sdg_count": len(sdg_counts), - "baseline_score": 45.0, - "vs_baseline": round(score - 45.0, 1), - "interpretation": ( - "Highly Diversified" if score >= 70 - else "Diversified" if score >= 50 - else "Specialised" if score >= 30 - else "Narrow" - ), - } - analytics_cache.set(cache_key, result, ttl=1800) - return result - except Exception as e: - logger.error("get_research_diversity: %s", e) - return {"score": 0, "sdg_distribution": [], "dominant_sdg": None, - "gap_sdgs": [], "interpretation": "No data"} - finally: - session.close() - - # ── Open Science Health Score ───────────────────────────────────────────── - - def get_open_science_health(self, institution: Optional[str] = None) -> Dict: - """Open Science Health Score — composite OA + PID + reproducibility metric. - - Weighted composite (0-100): - 40 pts Open Access rate (% of papers openly accessible) - 25 pts DOI coverage (% of papers with a DOI) - 20 pts ORCID coverage (% of papers with ≥1 ORCID author) - 15 pts ARK/DocID coverage (% with a persistent institutional PID) - - Provides a single, actionable number for funder compliance, AU Open - Science Policy alignment, and benchmarking against Plan S requirements. - Compared against continental average estimate of ~38/100. Cached 30 min. - """ - inst_name = self._resolve_institution_name(institution) - cache_key = f"open_science_health:{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached is not None: - return cached - - session = SessionLocal() - try: - sc_ids = self._get_sc_item_ids(session, institution) - total = len(sc_ids) - if not total: - return {"score": 0, "components": {}, "total": 0, - "grade": "F", "interpretation": "No data"} - - oa_count = ( - session.query(func.count(Item.id)) - .filter(Item.id.in_(sc_ids), - Item.dc_rights.like("%openAccess%")) - .scalar() - ) or 0 - - doi_count = ( - session.query(func.count(Item.id)) - .filter(Item.id.in_(sc_ids), - Item.doi.isnot(None), Item.doi != "") - .scalar() - ) or 0 - - ark_count = ( - session.query(func.count(Item.id)) - .filter(Item.id.in_(sc_ids), - Item.ark.isnot(None), Item.ark != "") - .scalar() - ) or 0 - - orcid_count = len( - session.query(Item.id) - .join(Item.authors) - .filter(Item.id.in_(sc_ids), - Author.orcid.isnot(None), Author.orcid != "") - .distinct().all() - ) - - def pct(n): return round(n / total * 100, 1) if total else 0.0 - - oa_pct = pct(oa_count) - doi_pct = pct(doi_count) - orcid_pct = pct(orcid_count) - ark_pct = pct(ark_count) - - score = round( - (oa_pct * 0.40) + - (doi_pct * 0.25) + - (orcid_pct * 0.20) + - (ark_pct * 0.15), - 1 - ) - - grade = "A" if score >= 75 else "B" if score >= 55 else "C" if score >= 35 else "D" if score >= 20 else "F" - - result = { - "score": score, - "grade": grade, - "total": total, - "components": { - "open_access": {"value": oa_pct, "weight": 40, "count": oa_count}, - "doi_coverage": {"value": doi_pct, "weight": 25, "count": doi_count}, - "orcid_coverage": {"value": orcid_pct, "weight": 20, "count": orcid_count}, - "ark_coverage": {"value": ark_pct, "weight": 15, "count": ark_count}, - }, - "continental_baseline": 38.0, - "vs_baseline": round(score - 38.0, 1), - "interpretation": ( - "Excellent — Plan S compliant" if score >= 75 - else "Good — approaching open science standards" if score >= 55 - else "Developing — action needed on OA and PIDs" if score >= 35 - else "Critical — significant gaps in open science" - ), - } - analytics_cache.set(cache_key, result, ttl=1800) - return result - except Exception as e: - logger.error("get_open_science_health: %s", e) - return {"score": 0, "components": {}, "total": 0, "grade": "F", - "interpretation": "No data"} - finally: - session.close() - - -analytics = URAASAnalyticsEngine() +""" +URAAS Analytics Engine +Implements all APA Intelligence & Analytics Platform metrics: + - Standard repository analytics (papers, authors, faculties, OA) + - TK Vitality Score (indigenous knowledge health index) + - Linguistic Diversity Index (African vs colonial language output) + - Patent-to-Paper Velocity (innovation lifecycle timing) + - Multi-institution Comparator (ROR-based benchmarking) + - SDG Alignment (UN Sustainable Development Goals) — AI-powered via spaCy + - Keyword Cloud (AI-extracted terms) + - Collaboration Network (D3 force graph data) + - Special Collections (African Literature, Indigenous Knowledge, etc.) +""" + +import itertools +import logging +import re +from collections import defaultdict +from datetime import datetime +from typing import Dict, List, Optional, Tuple + +from sqlalchemy import desc, extract, func, or_ +from sqlalchemy.orm import aliased, joinedload + +from uraas.config.institutions import get_registry +from uraas.database import ( + Author, + Collection, + Community, + File, + Item, + SessionLocal, + db_year, + db_year_month, +) +from uraas.services.sc_engine import SC_FILTER, category_breakdown, is_special_collection +from uraas.utils.ai_classifier import ( + SPECIAL_COLLECTIONS, + extract_keywords, + extract_trends_from_corpus, +) +from uraas.utils.analytics_cache import analytics_cache +from uraas.utils.unilag_classifier import classifier + +logger = logging.getLogger(__name__) + +# African language codes (kept here for Linguistic Diversity Index) +AFRICAN_LANG_CODES = { + "yo": "Yoruba", + "ig": "Igbo", + "ha": "Hausa", + "sw": "Swahili", + "am": "Amharic", + "so": "Somali", + "rw": "Kinyarwanda", + "sn": "Shona", + "zu": "Zulu", + "xh": "Xhosa", + "af": "Afrikaans", + "st": "Sesotho", + "tn": "Setswana", + "ts": "Tsonga", + "ss": "Swati", + "ve": "Venda", + "nr": "Ndebele", + "ff": "Fula", + "wo": "Wolof", + "bm": "Bambara", + "ln": "Lingala", + "kg": "Kongo", + "lua": "Luba", + "om": "Oromo", +} + +# Content type weights for TK Vitality Score +TK_WEIGHTS = { + "indigenous_knowledge": 3.0, + "cultural_heritage": 2.5, + "oral_tradition": 2.5, + "grey_literature": 1.5, + "thesis": 1.2, + "dataset": 1.2, + "patent": 1.0, + "research_paper": 0.5, +} + +# Common stop words for legacy keyword code +STOP_WORDS = { + "the", + "and", + "for", + "with", + "this", + "that", + "from", + "have", + "been", + "were", + "their", + "which", + "these", + "about", + "other", + "into", + "than", + "more", + "such", + "some", + "what", + "when", + "where", + "there", + "also", + "using", + "used", + "study", + "show", + "paper", + "research", + "analysis", + "findings", + "results", + "between", + "effect", + "impact", + "based", + "data", + "method", + "approach", + "model", + "system", + "review", + "case", + "report", + "among", + "within", + "across", + "during", + "after", + "before", + "through", + "while", + "both", + "each", + "only", + "very", + "well", + "high", + "low", + "new", + "large", + "small", + "significant", + "different", + "similar", + "total", + "however", + "therefore", + "thus", + "hence", + "although", + "despite", +} + + +class URAASAnalyticsEngine: + """ + Observer Engine for the APA Intelligence & Analytics Platform. + All methods return plain dicts/lists no ORM objects leak out. + """ + + # Helpers + + def _resolve_institution_name(self, identifier: Optional[str]) -> Optional[str]: + """Maps short name or ROR to full institution name from registry.""" + if not identifier: + return None + reg = get_registry() + inst = reg.get(identifier) + return inst.name if inst else identifier + + @staticmethod + def _is_oa(item: Item) -> bool: + return "openAccess" in (item.dc_rights or "") + + @staticmethod + def _year(item: Item) -> Optional[int]: + return item.publication_date.year if item.publication_date else None + + def _get_sc_item_ids(self, session, institution: Optional[str] = None) -> List[int]: + """ + Returns a list of Item IDs that are Special Collections. + + The authoritative SC signal is the stored ``special_collection_score`` + column (computed by the SC decision engine at crawl time / via the + re-classify script). This is a cheap indexed query — no per-row + re-classification — so the dashboard count updates immediately after a + crawl or prune (once the analytics cache is flushed). + """ + inst_name = self._resolve_institution_name(institution) + cache_key = f"sc_item_ids_{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + q = session.query(Item.id).filter(SC_FILTER) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + valid_ids = [row[0] for row in q.all()] + + analytics_cache.set(cache_key, valid_ids, ttl=3600) # cache for 1 hour + return valid_ids + + # Standard repository analytics + + def get_top_authors( + self, + limit: int = 15, + community_id: Optional[int] = None, + institution: Optional[str] = None, + ) -> List[Dict]: + inst_name = self._resolve_institution_name(institution) + session = SessionLocal() + try: + q = session.query( + Author.name, + Author.orcid, + Author.ror, + func.count(Item.id).label("count"), + ).join(Author.items).filter(SC_FILTER) + if community_id: + q = ( + q.join(Item.collections) + .join(Collection.community) + .filter(Community.id == community_id) + ) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + rows = ( + q.group_by(Author.name, Author.orcid, Author.ror) + .order_by(desc("count")) + .limit(limit) + .all() + ) + return [ + {"author": r[0], "orcid": r[1] or "", "ror": r[2] or "", "count": r[3]} + for r in rows + ] + except Exception as e: + logger.error("get_top_authors: %s", e) + return [] + finally: + session.close() + + def get_department_collaboration_network( + self, institution: Optional[str] = None + ) -> List[Dict]: + inst_name = self._resolve_institution_name(institution) + session = SessionLocal() + try: + q = session.query(Item).filter(SC_FILTER) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + docs = ( + q + .options(joinedload(Item.collections)) + .all() + ) + edges: Dict[Tuple, int] = {} + for doc in docs: + colls = sorted(c.name for c in doc.collections if c and c.name) + for pair in itertools.combinations(colls, 2): + edges[pair] = edges.get(pair, 0) + 1 + return [ + {"source": k[0], "target": k[1], "weight": v} for k, v in edges.items() + ] + except Exception as e: + logger.error("get_department_collaboration_network: %s", e) + return [] + finally: + session.close() + + def get_papers_by_faculty_and_department( + self, institution: Optional[str] = None + ) -> Dict: + inst_name = self._resolve_institution_name(institution) + session = SessionLocal() + try: + communities = ( + session.query(Community) + .options(joinedload(Community.collections)) + .all() + ) + tree: Dict = {} + seen: set = set() + + # Batch-fetch all File records once to avoid an O(N) query per paper. + file_map: Dict[int, object] = { + row.item_id: row + for row in session.query(File).all() + } + + for comm in communities: + dept_map: Dict = {} + for coll in comm.collections: + q = ( + session.query(Item) + .join(Item.collections) + .filter(Collection.id == coll.id, SC_FILTER) + ) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + + papers = q.all() + paper_list = [] + for p in papers: + seen.add(p.id) + f = file_map.get(p.id) + paper_list.append( + { + "id": p.id, + "title": p.title or "Untitled", + "doi": p.doi or "", + "url": p.url or "", + "docid": p.docid or "", + "has_local_pdf": f is not None, + "access_policy": f.access_policy if f else None, + "download_url": ( + f"/api/papers/{p.id}/download" if f else None + ), + } + ) + if paper_list: + dept_map[coll.name] = paper_list + if dept_map: + tree[comm.name] = dept_map + + # Unclassified bucket + unclassified_q = ( + session.query(Item).filter(SC_FILTER, ~Item.id.in_(seen)) + if seen + else session.query(Item).filter(SC_FILTER) + ) + if inst_name: + unclassified_q = unclassified_q.filter( + Item.institution.ilike(f"%{inst_name}%") + ) + unclassified = unclassified_q.all() + if unclassified: + tree["Unclassified"] = { + "General": [ + { + "id": p.id, + "title": p.title or "Untitled", + "doi": p.doi or "", + "url": p.url or "", + "docid": p.docid or "", + "has_local_pdf": False, + } + for p in unclassified + ] + } + return tree + except Exception as e: + logger.error("get_papers_by_faculty_and_department: %s", e) + return {} + finally: + session.close() + + + def get_publications_by_year(self, institution: Optional[str] = None) -> List[Dict]: + inst_name = self._resolve_institution_name(institution) + session = SessionLocal() + try: + q = session.query(db_year(Item.publication_date), func.count(Item.id)).filter(SC_FILTER) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + q = ( + q.filter(Item.publication_date.isnot(None)) + .group_by(db_year(Item.publication_date)) + .order_by(db_year(Item.publication_date)) + ) + return [{"year": int(r[0]) if r[0] else 0, "count": r[1]} for r in q.all()] + finally: + session.close() + + def get_papers_by_year_faculty( + self, institution: Optional[str] = None + ) -> List[Dict]: + inst_name = self._resolve_institution_name(institution) + session = SessionLocal() + try: + q = ( + session.query( + db_year(Item.publication_date), Community.name, func.count(Item.id) + ) + .join(Item.collections) + .join(Collection.community) + .filter(SC_FILTER) + ) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + q = q.filter(Item.publication_date.isnot(None)).group_by( + db_year(Item.publication_date), Community.name + ) + return [ + {"year": int(r[0]) if r[0] else 0, "faculty": r[1], "count": r[2]} + for r in q.all() + ] + finally: + session.close() + + def get_papers_by_faculty(self, institution: Optional[str] = None) -> List[Dict]: + inst_name = self._resolve_institution_name(institution) + session = SessionLocal() + try: + q = ( + session.query(Community.name, func.count(Item.id)) + .join(Item.collections) + .join(Collection.community) + .filter(SC_FILTER) + ) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + q = q.group_by(Community.name).order_by(desc(func.count(Item.id))) + return [{"faculty": r[0], "count": r[1]} for r in q.all()] + finally: + session.close() + + def get_open_access_breakdown( + self, institution: Optional[str] = None + ) -> List[Dict]: + inst_name = self._resolve_institution_name(institution) + session = SessionLocal() + try: + q = session.query(Item.dc_rights).filter(SC_FILTER) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + items = q.all() + counts = {"Open Access": 0, "Restricted": 0} + for it in items: + if "openAccess" in (it[0] or ""): + counts["Open Access"] += 1 + else: + counts["Restricted"] += 1 + return [{"label": k, "value": v} for k, v in counts.items()] + finally: + session.close() + + def get_authors_by_papers( + self, limit: int = 10, institution: Optional[str] = None + ) -> List[Dict]: + inst_name = self._resolve_institution_name(institution) + session = SessionLocal() + try: + q = session.query(Author.name, func.count(Item.id)).join(Author.items).filter(SC_FILTER) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + q = q.group_by(Author.name).order_by(desc(func.count(Item.id))).limit(limit) + return [{"author": r[0], "count": r[1]} for r in q.all()] + finally: + session.close() + + def get_faculty_oa_breakdown(self, institution: Optional[str] = None) -> List[Dict]: + inst_name = self._resolve_institution_name(institution) + session = SessionLocal() + try: + q = ( + session.query(Community.name, Item.dc_rights) + .join(Item.collections) + .join(Collection.community) + .filter(SC_FILTER) + ) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + rows = q.all() + facs = defaultdict(lambda: {"oa": 0, "restricted": 0}) + for fac, rights in rows: + if "openAccess" in (rights or ""): + facs[fac]["oa"] += 1 + else: + facs[fac]["restricted"] += 1 + return [ + {"faculty": k, "oa": v["oa"], "restricted": v["restricted"]} + for k, v in facs.items() + ] + finally: + session.close() + + def get_institutional_growth(self, institution: Optional[str] = None) -> List[Dict]: + inst_name = self._resolve_institution_name(institution) + session = SessionLocal() + try: + q = session.query(db_year_month(Item.created_at), func.count(Item.id)).filter(SC_FILTER) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + q = q.group_by(db_year_month(Item.created_at)).order_by( + db_year_month(Item.created_at) + ) + return [{"month": r[0], "count": r[1]} for r in q.all()] + finally: + session.close() + + def get_timeline_data(self, institution: Optional[str] = None) -> List[Dict]: + inst_name = self._resolve_institution_name(institution) + session = SessionLocal() + try: + q = session.query(func.date(Item.created_at), func.count(Item.id)).filter(SC_FILTER) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + q = q.group_by(func.date(Item.created_at)).order_by( + func.date(Item.created_at) + ) + res = [] + total = 0 + for r in q.all(): + total += r[1] + res.append({"date": r[0], "count": r[1], "total": total}) + return res + finally: + session.close() + + # SDG Alignment (AI-powered, cached) + + def get_sdg_alignment(self, institution: Optional[str] = None) -> List[Dict]: + """ + Score every paper against all 17 SDGs using AI. + Results are cached for 30 minutes per institution. + """ + inst_name = self._resolve_institution_name(institution) + cache_key = f"sdg_alignment:{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + session = SessionLocal() + try: + q = session.query(Item.id, Item.title, Item.abstract).filter(SC_FILTER) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + items = q.all() + + # Pre-populate buckets for SDG 1 to 17 + sdg_buckets = {n: [] for n in range(1, 18)} + sdg_names_full = {} + for item_id, title, abstract in items: + text_corpus = f"{title or ''} {abstract or ''}" + hits = classifier.detect_sdg_alignment(text_corpus) + for hit in hits: + sdg_str = hit["sdg"] # e.g. "SDG 1 — No Poverty" + try: + num = int(re.search(r"SDG (\d+)", sdg_str).group(1)) + sdg_names_full[num] = sdg_str + sdg_buckets[num].append( + { + "id": item_id, + "title": title, + "score": hit["score"], + "keywords": hit["matched_keywords"], + } + ) + except (AttributeError, ValueError): + continue + + result = [] + for sdg_num, papers in sdg_buckets.items(): + if papers: + papers.sort(key=lambda x: -x["score"]) + result.append( + { + "sdg": sdg_names_full.get(sdg_num, f"SDG {sdg_num}"), + "sdg_number": sdg_num, + "count": len(papers), + "papers": papers[:10], + } + ) + result.sort(key=lambda x: -x["count"]) + analytics_cache.set(cache_key, result) + return result + except Exception as e: + logger.error("get_sdg_alignment: %s", e) + return [] + finally: + session.close() + + def get_sdg_csv_data(self) -> List[List]: + """ + Returns SDG alignment data as rows for CSV export. + Header: [SDG Number, SDG Name, Paper Count, Paper Title, Score, Matched Keywords] + """ + rows = [ + [ + "SDG Number", + "SDG Name", + "Paper Count", + "Paper Title", + "Score", + "Matched Keywords", + ] + ] + try: + alignment = self.get_sdg_alignment() + for entry in alignment: + sdg_num = entry.get("sdg_number", "") + sdg_name_full = entry.get("sdg", "") + count = entry.get("count", 0) + for paper in entry.get("papers", []): + rows.append( + [ + sdg_num, + sdg_name_full, + count, + paper.get("title", ""), + paper.get("score", 0), + "; ".join(paper.get("keywords", [])), + ] + ) + except Exception as e: + logger.error("get_sdg_csv_data: %s", e) + return rows + + # Keyword Cloud (corpus-level TF-IDF, cached) + + def get_keyword_cloud( + self, top_n: int = 60, institution: Optional[str] = None + ) -> List[Dict]: + """Extract top keywords using corpus-level TF-IDF + spaCy NER. Cached 30 min.""" + inst_name = self._resolve_institution_name(institution) + cache_key = f"keyword_cloud:{inst_name or 'all'}:{top_n}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + session = SessionLocal() + try: + q = session.query(Item.title, Item.abstract).filter(SC_FILTER) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + items = q.all() + + # Build corpus for IDF calculation + all_texts = [f"{t or ''} {a or ''}" for t, a in items] + combined_title = " ".join(t or "" for t, _ in items) + combined_abstract = " ".join(a or "" for _, a in items) + + keywords = extract_keywords( + combined_title, combined_abstract, top_n=top_n, all_texts=all_texts + ) + result = [ + {"word": k["word"], "count": k["count"], "score": k["score"]} + for k in keywords + ] + analytics_cache.set(cache_key, result) + return result + except Exception as e: + logger.error("get_keyword_cloud: %s", e) + return [] + finally: + session.close() + + # APA Novel Metrics + + def get_institution_leaderboard(self) -> List[Dict]: + """Cross-institution leaderboard ranked across key metrics.""" + cache_key = "institution_leaderboard" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + session = SessionLocal() + try: + # Get distinct institutions in the DB + inst_rows = ( + session.query(Item.institution, func.count(Item.id).label("total")) + .filter(Item.institution.isnot(None)) + .group_by(Item.institution) + .all() + ) + + leaderboard = [] + for inst_name, total in inst_rows: + oa = ( + session.query(Item) + .filter( + Item.institution == inst_name, + Item.dc_rights.like("%openAccess%"), + ) + .count() + ) + authors = ( + session.query(func.count(func.distinct(Author.id))) + .join(Author.items) + .filter(Item.institution == inst_name) + .scalar() + or 0 + ) + + oa_rate = round(oa / total * 100, 1) if total else 0 + leaderboard.append( + { + "institution": inst_name, + "total_papers": total, + "open_access": oa, + "oa_rate": oa_rate, + "unique_authors": authors, + "score": round(total * 0.4 + oa_rate * 0.4 + authors * 0.2, 1), + } + ) + + leaderboard.sort(key=lambda x: -x["score"]) + for i, inst in enumerate(leaderboard): + inst["rank"] = i + 1 + + analytics_cache.set(cache_key, leaderboard) + return leaderboard + except Exception as e: + logger.error("get_institution_leaderboard: %s", e) + return [] + finally: + session.close() + + def get_tk_vitality_score(self, institution: Optional[str] = None) -> Dict: + """ + TK Vitality Score measures how well the institution is digitising + indigenous knowledge and cultural heritage. + + Score = weighted sum of content types / total items * 100 + Max theoretical score = 100 (all items are indigenous knowledge) + """ + inst_name = self._resolve_institution_name(institution) + cache_key = f"tk_vitality:{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + session = SessionLocal() + try: + q = session.query(Item.content_type, Item.tk_label, Item.dc_type).filter(SC_FILTER) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + items = q.all() + total = len(items) + if total == 0: + return {"score": 0, "breakdown": {}, "total_items": 0, "tk_items": 0} + + type_counts: Dict[str, int] = defaultdict(int) + weighted_sum = 0.0 + tk_items = 0 + + for content_type, tk_label, dc_type in items: + ct = content_type or "research_paper" + # Upgrade type if TK label is present + if tk_label: + ct = "indigenous_knowledge" + tk_items += 1 + elif dc_type and "cultural" in (dc_type or "").lower(): + ct = "cultural_heritage" + tk_items += 1 + + type_counts[ct] += 1 + weighted_sum += TK_WEIGHTS.get(ct, 0.5) + + max_possible = total * TK_WEIGHTS["indigenous_knowledge"] + score = round((weighted_sum / max_possible) * 100, 1) if max_possible else 0 + + result = { + "score": score, + "breakdown": dict(type_counts), + "total_items": total, + "tk_items": tk_items, + "tk_percentage": round(tk_items / total * 100, 1) if total else 0, + "interpretation": ( + "Excellent" + if score >= 60 + else "Good" if score >= 30 else "Developing" + ), + } + analytics_cache.set(cache_key, result) + return result + except Exception as e: + logger.error("get_tk_vitality_score: %s", e) + return {"score": 0, "breakdown": {}, "total_items": 0, "tk_items": 0} + finally: + session.close() + + def get_linguistic_diversity_index(self) -> Dict: + """ + Linguistic Diversity Index % of outputs in African languages vs English/French. + Supports the decolonisation of knowledge mission. + """ + session = SessionLocal() + try: + items = session.query( + Item.language_code, Item.is_african_language, Item.dc_language + ).all() + total = len(items) + if total == 0: + return {"index": 0, "african_count": 0, "total": 0, "breakdown": {}} + + lang_counts: Dict[str, int] = defaultdict(int) + african_count = 0 + + for lang_code, is_african, dc_lang in items: + code = lang_code or dc_lang or "en" + lang_counts[code] += 1 + if is_african or code in AFRICAN_LANG_CODES: + african_count += 1 + + index = round(african_count / total * 100, 1) + + # Build human-readable breakdown + breakdown = {} + for code, count in sorted(lang_counts.items(), key=lambda x: -x[1]): + label = AFRICAN_LANG_CODES.get(code, code.upper()) + breakdown[label] = count + + return { + "index": index, + "african_count": african_count, + "colonial_count": total - african_count, + "total": total, + "breakdown": breakdown, + "top_african_languages": [ + {"language": AFRICAN_LANG_CODES.get(c, c), "code": c, "count": n} + for c, n in sorted(lang_counts.items(), key=lambda x: -x[1]) + if c in AFRICAN_LANG_CODES + ][:10], + } + except Exception as e: + logger.error("get_linguistic_diversity_index: %s", e) + return {"index": 0, "african_count": 0, "total": 0, "breakdown": {}} + finally: + session.close() + + def get_special_collections_metrics( + self, institution: Optional[str] = None + ) -> Dict: + """ + Special Collections Metrics using AI classifier. Cached 30 min. + """ + inst_name = self._resolve_institution_name(institution) + cache_key = f"special_collections:{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + session = SessionLocal() + try: + q = session.query(Item.id, Item.title, Item.abstract, Item.dc_subject) + if inst_name: + q = q.filter(Item.institution.ilike(f"%{inst_name}%")) + sc_ids = self._get_sc_item_ids(session, institution) + if sc_ids: + q = q.filter(Item.id.in_(sc_ids)) + else: + return { + "summary": [], + "total_special_items": 0, + "total_repository_items": q.count(), + } + items = q.all() + results: Dict[str, List] = {cat: [] for cat in SPECIAL_COLLECTIONS} + + for item_id, title, abstract, dc_subject in items: + cats = category_breakdown( + title or "", abstract or "", dc_subject or "" + ) + for cat_result in cats: + cat = cat_result["category"] + if cat in results: + results[cat].append( + { + "id": item_id, + "title": title, + "matches": cat_result["matched_keywords"], + "count": cat_result["score"], + } + ) + + summary = [] + for category, papers in results.items(): + papers.sort(key=lambda x: -x["count"]) + summary.append( + { + "category": category, + "count": len(papers), + "top_papers": papers[:10], + } + ) + + summary.sort(key=lambda x: -x["count"]) + result = { + "summary": summary, + "total_special_items": sum(len(p) for p in results.values()), + "total_repository_items": len(items), + } + analytics_cache.set(cache_key, result) + return result + except Exception as e: + logger.error("get_special_collections_metrics: %s", e) + return { + "summary": [], + "total_special_items": 0, + "total_repository_items": 0, + } + finally: + session.close() + + def get_special_collections_overview( + self, institution: Optional[str] = None + ) -> Dict: + """ + Special Collections Overview analytics — purpose-built for indigenous + knowledge / African literature / cultural-heritage collections rather + than the citation-prestige lens of commercial bibliometric platforms. + + Returns thematic composition + theme co-occurrence, knowledge + sovereignty (African custodianship), SDG/development alignment, top + institutional custodians, a cultural lexicon, and the most influential + works. All metrics are restricted to genuine Special Collections via the + stored ``special_collection_score`` gate (``_get_sc_item_ids``) and are + computed only over fields populated for SC items. Cached 30 min. + """ + from uraas.config.african_countries import COUNTRY_NAMES + + inst_name = self._resolve_institution_name(institution) + cache_key = f"sc_overview:{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + canonical = list(SPECIAL_COLLECTIONS.keys()) + empty = { + "kpis": { + "total_items": 0, + "repo_total": 0, + "pct_of_repo": 0, + "themes_represented": 0, + "total_citations": 0, + "intra_african_pct": 0, + }, + "themes": [{"category": c, "count": 0} for c in canonical], + "co_occurrence": {"labels": canonical, "matrix": [[0] * len(canonical) for _ in canonical]}, + "countries": [], + "sdgs": [], + "keywords": [], + "custodians": [], + "influential": [], + } + + # Build institution name → ISO2 code for country-fallback when + # coauthor_countries is empty (papers with no cross-institution data). + _name_to_iso2 = {v.lower(): k for k, v in COUNTRY_NAMES.items()} + try: + from uraas.config import get_registry as _get_reg + _inst_country: Dict[str, str] = { + cfg.name: _name_to_iso2.get(cfg.country.lower(), "") + for cfg in _get_reg().list_all() + } + except Exception: + _inst_country = {} + + session = SessionLocal() + try: + repo_q = session.query(func.count(Item.id)) + if inst_name: + repo_q = repo_q.filter(Item.institution.ilike(f"%{inst_name}%")) + repo_total = repo_q.scalar() or 0 + + sc_ids = self._get_sc_item_ids(session, institution) + if not sc_ids: + empty["kpis"]["repo_total"] = repo_total + return empty + + rows = ( + session.query( + Item.id, + Item.title, + Item.cited_by_count, + Item.special_collection_categories, + Item.coauthor_countries, + Item.is_intra_african, + Item.sdg_tags, + Item.ai_keywords, + Item.institution, + Item.special_collection_score, + ) + .filter(Item.id.in_(sc_ids)) + .all() + ) + + cat_idx = {c: i for i, c in enumerate(canonical)} + theme_counts = {c: 0 for c in canonical} + matrix = [[0] * len(canonical) for _ in canonical] + country_counts: Dict[str, int] = defaultdict(int) + sdg_counts: Dict[str, int] = defaultdict(int) + kw_counts: Dict[str, int] = defaultdict(int) + custodian_counts: Dict[str, int] = defaultdict(int) + total_citations = 0 + intra_african = 0 + influential: List[Dict] = [] + + for ( + item_id, + title, + cited, + categories, + countries, + is_intra, + sdgs, + keywords, + inst, + sc_score, + ) in rows: + cited = int(cited or 0) + total_citations += cited + if is_intra: + intra_african += 1 + + # Themes (canonical, multi-valued) + co-occurrence matrix. + item_cats = [ + c.strip() + for c in (categories or "").split(",") + if c.strip() in cat_idx + ] + item_cats = sorted(set(item_cats)) + for c in item_cats: + theme_counts[c] += 1 + if len(item_cats) == 1: + i = cat_idx[item_cats[0]] + matrix[i][i] += 1 + else: + for a, b in itertools.combinations(item_cats, 2): + ia, ib = cat_idx[a], cat_idx[b] + matrix[ia][ib] += 1 + matrix[ib][ia] += 1 + + # Knowledge sovereignty — contributing African countries. + _any_country = False + for code in (countries or "").split(","): + code = code.strip().upper() + if code in COUNTRY_NAMES: + country_counts[code] += 1 + _any_country = True + if not _any_country and inst: + _fallback = _inst_country.get(inst, "") + if _fallback: + country_counts[_fallback] += 1 + + # SDG alignment. + for sdg in (sdgs or "").split(","): + sdg = sdg.strip() + if sdg: + sdg_counts[sdg] += 1 + + # Cultural lexicon. + for kw in (keywords or "").split(","): + kw = kw.strip().lower() + if len(kw) > 2: + kw_counts[kw] += 1 + + # Custodian institutions. + if inst: + custodian_counts[inst] += 1 + + # Collect all SC works for the influential list. + # Ranked by cited_by_count first; sc_score is the tiebreaker + # so the list is always populated even before citation backfill. + influential.append( + { + "id": item_id, + "title": title or "Untitled", + "citations": cited, + "sc_score": float(sc_score or 0), + "categories": item_cats, + } + ) + + total_items = len(rows) + themes_represented = sum(1 for v in theme_counts.values() if v > 0) + influential.sort(key=lambda x: (-x["citations"], -x["sc_score"])) + + result = { + "kpis": { + "total_items": total_items, + "repo_total": repo_total, + "pct_of_repo": round(total_items / repo_total * 100, 1) + if repo_total + else 0, + "themes_represented": themes_represented, + "total_citations": total_citations, + "intra_african_pct": round(intra_african / total_items * 100, 1) + if total_items + else 0, + }, + "themes": [ + {"category": c, "count": theme_counts[c]} for c in canonical + ], + "co_occurrence": {"labels": canonical, "matrix": matrix}, + "countries": [ + {"code": code, "name": COUNTRY_NAMES.get(code, code), "papers": n} + for code, n in sorted(country_counts.items(), key=lambda x: -x[1]) + ], + "sdgs": [ + {"sdg": sdg, "count": n} + for sdg, n in sorted( + sdg_counts.items(), + key=lambda x: -x[1], + ) + ], + "keywords": [ + {"word": w, "count": n} + for w, n in sorted(kw_counts.items(), key=lambda x: -x[1])[:30] + ], + "custodians": [ + {"institution": inst, "count": n} + for inst, n in sorted( + custodian_counts.items(), key=lambda x: -x[1] + )[:10] + ], + "influential": influential[:15], + } + # Sort themes descending by count for display. + result["themes"].sort(key=lambda x: -x["count"]) + analytics_cache.set(cache_key, result) + return result + except Exception as e: + logger.error("get_special_collections_overview: %s", e) + return empty + finally: + session.close() + + def get_special_collections_csv_data(self) -> List[List]: + """Returns special collections data as rows for CSV export.""" + rows = [["Category", "Paper Count", "Paper Title", "Score", "Matched Keywords"]] + try: + metrics = self.get_special_collections_metrics() + for entry in metrics.get("summary", []): + for paper in entry.get("top_papers", []): + rows.append( + [ + entry.get("category", ""), + entry.get("count", 0), + paper.get("title", ""), + paper.get("count", 0), + "; ".join(paper.get("matches", [])), + ] + ) + except Exception as e: + logger.error("get_special_collections_csv_data: %s", e) + return rows + + def get_author_network( + self, author_name: Optional[str] = None, limit: int = 30 + ) -> Dict: + """ + Collaboration network for D3 force graph. + If author_name given: ego-network for that researcher. + Otherwise: top-N authors by paper count. + """ + session = SessionLocal() + try: + if author_name: + items = ( + session.query(Item) + .join(Item.authors) + .filter(Author.name == author_name) + .options(joinedload(Item.authors)) + .all() + ) + edges: Dict[Tuple, int] = {} + for item in items: + names = [a.name for a in item.authors] + if author_name not in names: + continue + for name in names: + if name != author_name: + key = tuple(sorted([author_name, name])) + edges[key] = edges.get(key, 0) + 1 + + edge_list = sorted( + [ + {"source": k[0], "target": k[1], "weight": v} + for k, v in edges.items() + ], + key=lambda x: -x["weight"], + )[:15] + + collaborators = {e["source"] for e in edge_list} | { + e["target"] for e in edge_list + } + collaborators.discard(author_name) + + return { + "nodes": [{"id": author_name, "count": len(items)}] + + [{"id": c, "count": 0} for c in collaborators], + "edges": edge_list, + } + else: + # Global top-N network + top_rows = ( + session.query(Author.name, func.count(Item.id).label("cnt")) + .join(Author.items) + .group_by(Author.name) + .order_by(desc("cnt")) + .limit(limit) + .all() + ) + top_names = {r[0] for r in top_rows} + node_counts = {r[0]: r[1] for r in top_rows} + + items = session.query(Item).options(joinedload(Item.authors)).all() + edges: Dict[Tuple, int] = {} + for item in items: + names = [a.name for a in item.authors if a.name in top_names] + for pair in itertools.combinations(sorted(names), 2): + edges[pair] = edges.get(pair, 0) + 1 + + return self._annotate_network( + { + "nodes": [ + {"id": n, "count": node_counts[n]} for n in top_names + ], + "edges": [ + {"source": k[0], "target": k[1], "weight": v} + for k, v in edges.items() + ], + } + ) + except Exception as e: + logger.error("get_author_network: %s", e) + return {"nodes": [], "edges": []} + finally: + session.close() + + @staticmethod + def _annotate_network(graph: Dict) -> Dict: + """Add Louvain community + degree centrality to network nodes. + + Gephi/VOSviewer-standard visual grammar: color by community, size by + centrality. Deterministic (seed=42) so the layout story is stable.""" + try: + import networkx as nx + + G = nx.Graph() + G.add_nodes_from(n["id"] for n in graph["nodes"]) + G.add_weighted_edges_from( + (e["source"], e["target"], e["weight"]) for e in graph["edges"] + ) + if G.number_of_edges(): + communities = nx.community.louvain_communities( + G, weight="weight", seed=42 + ) + membership = { + node: idx for idx, comm in enumerate(communities) for node in comm + } + centrality = nx.degree_centrality(G) + for n in graph["nodes"]: + n["community"] = membership.get(n["id"], 0) + n["centrality"] = round(centrality.get(n["id"], 0.0), 3) + except Exception as e: + logger.warning("_annotate_network: %s", e) + return graph + + # Intra-African collaboration analytics + + # Continental baseline: share of African co-publications involving >=2 + # African countries, 2002-2019 (Research Policy, 2022). + INTRA_AFRICAN_BASELINE_PCT = 8.4 + + def get_collaboration_overview(self, institution: Optional[str] = None) -> Dict: + """Intra-African Collaboration Index — Scimago-compatible definition + (works whose affiliations span >=2 distinct African countries), + benchmarked against the continental average.""" + inst_name = self._resolve_institution_name(institution) + cache_key = f"collab_overview_{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + session = SessionLocal() + try: + sc_ids = self._get_sc_item_ids(session, institution) + if not sc_ids: + return {"total_papers": 0, "papers_with_affiliation_data": 0} + + from uraas.config.african_countries import COUNTRY_NAMES + from uraas.database import ItemAffiliation + + covered = { + i + for (i,) in session.query(ItemAffiliation.item_id) + .filter(ItemAffiliation.item_id.in_(sc_ids)) + .distinct() + } + intra_count = ( + session.query(func.count(Item.id)) + .filter(Item.id.in_(sc_ids), Item.is_intra_african.is_(True)) + .scalar() + or 0 + ) + denominator = len(covered) + pct = round(intra_count / denominator * 100, 1) if denominator else 0.0 + + # Per-country paper counts (partners), excluding none + country_rows = ( + session.query( + ItemAffiliation.country_code, + func.count(func.distinct(ItemAffiliation.item_id)), + ) + .filter( + ItemAffiliation.item_id.in_(sc_ids), + ItemAffiliation.country_code.in_(list(COUNTRY_NAMES)), + ) + .group_by(ItemAffiliation.country_code) + .all() + ) + home_codes = set() + if inst_name: + # Home country dominates counts; surface partners separately. + reg_inst = get_registry().get(institution or "") + if reg_inst: + home_codes = { + code + for code, name in COUNTRY_NAMES.items() + if name.lower() == reg_inst.country.lower() + } + partners = sorted( + ( + {"code": c, "name": COUNTRY_NAMES.get(c, c), "count": n} + for c, n in country_rows + if c not in home_codes + ), + key=lambda r: -r["count"], + ) + + result = { + "total_papers": len(sc_ids), + "papers_with_affiliation_data": denominator, + "intra_african_count": intra_count, + "intra_african_pct": pct, + "baseline_pct": self.INTRA_AFRICAN_BASELINE_PCT, + "ratio_vs_baseline": ( + round(pct / self.INTRA_AFRICAN_BASELINE_PCT, 2) if pct else 0.0 + ), + "country_count": len(country_rows), + "top_partner_countries": partners[:10], + } + analytics_cache.set(cache_key, result, ttl=1800) + return result + except Exception as e: + logger.error("get_collaboration_overview: %s", e) + return {"total_papers": 0, "papers_with_affiliation_data": 0} + finally: + session.close() + + def get_country_pair_matrix(self, institution: Optional[str] = None) -> Dict: + """Unordered African country-pair co-publication counts (full counting).""" + inst_name = self._resolve_institution_name(institution) + cache_key = f"collab_pairs_{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + session = SessionLocal() + try: + sc_ids = self._get_sc_item_ids(session, institution) + if not sc_ids: + return {"pairs": [], "countries": []} + + from uraas.config.african_countries import COUNTRY_NAMES + from uraas.database import ItemAffiliation + + a1 = aliased(ItemAffiliation) + a2 = aliased(ItemAffiliation) + rows = ( + session.query( + a1.country_code, + a2.country_code, + func.count(func.distinct(a1.item_id)), + ) + .join(a2, a1.item_id == a2.item_id) + .filter( + a1.country_code < a2.country_code, + a1.country_code.in_(list(COUNTRY_NAMES)), + a2.country_code.in_(list(COUNTRY_NAMES)), + a1.item_id.in_(sc_ids), + ) + .group_by(a1.country_code, a2.country_code) + .all() + ) + pairs = sorted( + ( + { + "source": s, + "target": t, + "source_name": COUNTRY_NAMES.get(s, s), + "target_name": COUNTRY_NAMES.get(t, t), + "count": n, + } + for s, t, n in rows + ), + key=lambda p: -p["count"], + ) + countries = sorted({p["source"] for p in pairs} | {p["target"] for p in pairs}) + result = {"pairs": pairs, "countries": countries} + analytics_cache.set(cache_key, result, ttl=1800) + return result + except Exception as e: + logger.error("get_country_pair_matrix: %s", e) + return {"pairs": [], "countries": []} + finally: + session.close() + + def get_country_aggregates(self, institution: Optional[str] = None) -> List[Dict]: + """Per-African-country paper counts for the choropleth.""" + inst_name = self._resolve_institution_name(institution) + cache_key = f"collab_countries_{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + session = SessionLocal() + try: + sc_ids = self._get_sc_item_ids(session, institution) + if not sc_ids: + return [] + + from uraas.config.african_countries import COUNTRY_NAMES + from uraas.database import ItemAffiliation + + rows = ( + session.query( + ItemAffiliation.country_code, + func.count(func.distinct(ItemAffiliation.item_id)), + ) + .filter( + ItemAffiliation.item_id.in_(sc_ids), + ItemAffiliation.country_code.in_(list(COUNTRY_NAMES)), + ) + .group_by(ItemAffiliation.country_code) + .all() + ) + intra = dict( + session.query( + ItemAffiliation.country_code, + func.count(func.distinct(ItemAffiliation.item_id)), + ) + .join(Item, Item.id == ItemAffiliation.item_id) + .filter( + ItemAffiliation.item_id.in_(sc_ids), + Item.is_intra_african.is_(True), + ItemAffiliation.country_code.in_(list(COUNTRY_NAMES)), + ) + .group_by(ItemAffiliation.country_code) + .all() + ) + result = sorted( + ( + { + "code": c, + "name": COUNTRY_NAMES.get(c, c), + "papers": n, + "intra_african": intra.get(c, 0), + } + for c, n in rows + ), + key=lambda r: -r["papers"], + ) + analytics_cache.set(cache_key, result, ttl=1800) + return result + except Exception as e: + logger.error("get_country_aggregates: %s", e) + return [] + finally: + session.close() + + def get_citation_velocity(self, institution: Optional[str] = None) -> Dict: + """Repository-level citation velocity from stored counts_by_year, plus + citation-weighted Pan-African citation share.""" + import json as _json + + inst_name = self._resolve_institution_name(institution) + cache_key = f"citation_velocity_{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + session = SessionLocal() + try: + sc_ids = self._get_sc_item_ids(session, institution) + if not sc_ids: + return {"by_year": [], "items_covered": 0} + + rows = ( + session.query( + Item.counts_by_year, + Item.cited_by_count, + Item.african_citation_share, + Item.publication_date, + ) + .filter(Item.id.in_(sc_ids), Item.counts_by_year.isnot(None)) + .all() + ) + by_year: Dict[int, int] = {} + first2y: List[int] = [] + share_weighted = share_weight = 0.0 + share_covered = 0 + for counts_json, cited, share, pub_date in rows: + try: + counts = _json.loads(counts_json) + except Exception: + continue + year_map = { + c["year"]: c.get("cited_by_count", 0) + for c in counts + if isinstance(c, dict) and "year" in c + } + for y, n in year_map.items(): + by_year[y] = by_year.get(y, 0) + n + if pub_date: + first2y.append( + year_map.get(pub_date.year, 0) + + year_map.get(pub_date.year + 1, 0) + ) + if share is not None: + share_covered += 1 + weight = max(cited or 0, 1) + share_weighted += share * weight + share_weight += weight + + result = { + "by_year": [ + {"year": y, "citations": by_year[y]} for y in sorted(by_year) + ], + "items_covered": len(rows), + "avg_first2y": ( + round(sum(first2y) / len(first2y), 1) if first2y else 0.0 + ), + "pan_african_share_pct": ( + round(share_weighted / share_weight, 1) if share_weight else None + ), + "pan_african_share_items": share_covered, + } + analytics_cache.set(cache_key, result, ttl=1800) + return result + except Exception as e: + logger.error("get_citation_velocity: %s", e) + return {"by_year": [], "items_covered": 0} + finally: + session.close() + + def get_collaboration_arcs(self, institution: Optional[str] = None) -> Dict: + """GeoJSON FeatureCollection of great-circle arcs for country pairs.""" + from uraas.config.african_countries import COUNTRY_CENTROIDS + from uraas.utils.geo import great_circle_arc + + matrix = self.get_country_pair_matrix(institution) + features = [] + for pair in matrix["pairs"]: + src = COUNTRY_CENTROIDS.get(pair["source"]) + dst = COUNTRY_CENTROIDS.get(pair["target"]) + if not (src and dst): + continue + features.append( + { + "type": "Feature", + "properties": { + "source": pair["source"], + "target": pair["target"], + "source_name": pair["source_name"], + "target_name": pair["target_name"], + "count": pair["count"], + }, + "geometry": { + "type": "LineString", + "coordinates": great_circle_arc( + src[0], src[1], dst[0], dst[1] + ), + }, + } + ) + return {"type": "FeatureCollection", "features": features} + + + def get_pid_coverage(self, institution: Optional[str] = None) -> Dict: + """Return PID coverage statistics — key metric for PID Alliance audiences. + + Reports the fraction of Special Collections items carrying each + persistent identifier type: DOI, ORCID (via any author), ARK, ROR. + """ + session = SessionLocal() + try: + sc_ids = self._get_sc_item_ids(session, institution) + total = len(sc_ids) + if not total: + return { + "total": 0, + "doi_count": 0, "doi_pct": 0.0, + "orcid_count": 0, "orcid_pct": 0.0, + "ark_count": 0, "ark_pct": 0.0, + "ror_count": 0, "ror_pct": 0.0, + } + + doi_count = ( + session.query(func.count(Item.id)) + .filter(Item.id.in_(sc_ids), Item.doi.isnot(None), Item.doi != "") + .scalar() + ) or 0 + + ark_count = ( + session.query(func.count(Item.id)) + .filter(Item.id.in_(sc_ids), Item.ark.isnot(None), Item.ark != "") + .scalar() + ) or 0 + + ror_count = ( + session.query(func.count(Item.id)) + .filter(Item.id.in_(sc_ids), Item.ror.isnot(None), Item.ror != "") + .scalar() + ) or 0 + + # Items where at least one author has an ORCID + orcid_item_ids = ( + session.query(Item.id) + .join(Item.authors) + .filter( + Item.id.in_(sc_ids), + Author.orcid.isnot(None), + Author.orcid != "", + ) + .distinct() + .all() + ) + orcid_count = len(orcid_item_ids) + + def _pct(n: int) -> float: + return round(100.0 * n / total, 1) if total else 0.0 + + return { + "total": total, + "doi_count": doi_count, "doi_pct": _pct(doi_count), + "orcid_count": orcid_count, "orcid_pct": _pct(orcid_count), + "ark_count": ark_count, "ark_pct": _pct(ark_count), + "ror_count": ror_count, "ror_pct": _pct(ror_count), + } + except Exception as e: + logger.error("get_pid_coverage: %s", e) + return {"total": 0, "doi_count": 0, "doi_pct": 0.0, + "orcid_count": 0, "orcid_pct": 0.0, + "ark_count": 0, "ark_pct": 0.0, + "ror_count": 0, "ror_pct": 0.0} + finally: + session.close() + + + # ── Knowledge Repatriation Index ───────────────────────────────────────── + + def get_knowledge_repatriation(self, institution: Optional[str] = None) -> Dict: + """Knowledge Repatriation Index — measures whether Africa leads its own + research rather than being a subject of externally-led studies. + + Segments the corpus into three collaboration profiles: + • Africa-Led : only African institutions on the paper (sole or pan-African) + • North-South : African institution + ≥1 non-African institution + • Unclassified : no affiliation data available + + The repatriation score (0-100) = Africa-Led / (Africa-Led + North-South) × 100. + Higher = Africa owns more of its own knowledge production. + No other analytics platform surfaces this metric. Cached 30 min. + """ + from uraas.config.african_countries import COUNTRY_NAMES + from uraas.database import ItemAffiliation + + inst_name = self._resolve_institution_name(institution) + cache_key = f"knowledge_repatriation:{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + session = SessionLocal() + try: + sc_ids = self._get_sc_item_ids(session, institution) + total = len(sc_ids) + if not total: + return {"score": 0, "africa_led": 0, "north_south": 0, + "unclassified": 0, "total": 0, "trend": []} + + african_codes = set(COUNTRY_NAMES.keys()) + + # Items that have any affiliation record + aff_items = { + item_id + for (item_id,) in session.query(ItemAffiliation.item_id) + .filter(ItemAffiliation.item_id.in_(sc_ids)) + .distinct() + } + + africa_led = 0 + north_south = 0 + + for item_id in aff_items: + countries = { + row[0].upper() + for row in session.query(ItemAffiliation.country_code) + .filter(ItemAffiliation.item_id == item_id, + ItemAffiliation.country_code.isnot(None)) + .all() + if row[0] + } + if not countries: + continue + non_african = countries - african_codes + if non_african: + north_south += 1 + else: + africa_led += 1 + + # Fall back to coauthor_countries for items without affiliation records + no_aff = [i for i in sc_ids if i not in aff_items] + no_country_ids = set() + for row in (session.query(Item.id, Item.coauthor_countries) + .filter(Item.id.in_(no_aff)) + .all()): + codes_raw = (row[1] or "").strip() + if not codes_raw: + no_country_ids.add(row[0]) + continue + codes = {c.strip().upper() for c in codes_raw.split(",") if c.strip()} + if not codes: + no_country_ids.add(row[0]) + continue + if codes - african_codes: + north_south += 1 + else: + africa_led += 1 + + # Items with no coauthor_countries and no ItemAffiliation were crawled + # from African institutions — classify as Africa-Led (best-effort). + africa_led += len(no_country_ids) + + classified = africa_led + north_south + unclassified = total - classified + score = round(africa_led / classified * 100, 1) if classified else 0.0 + + # Year-over-year trend (last 5 years of classified papers) + trend_rows = ( + session.query(db_year(Item.publication_date), + Item.is_intra_african, + func.count(Item.id)) + .filter(Item.id.in_(sc_ids), + Item.publication_date.isnot(None), + Item.coauthor_countries.isnot(None)) + .group_by(db_year(Item.publication_date), Item.is_intra_african) + .order_by(db_year(Item.publication_date)) + .all() + ) + by_year: Dict[str, Dict] = {} + for yr, intra, cnt in trend_rows: + if not yr: + continue + yr = str(yr) + by_year.setdefault(yr, {"year": yr, "africa_led": 0, "north_south": 0}) + if intra: + by_year[yr]["africa_led"] += cnt + else: + by_year[yr]["north_south"] += cnt + trend = sorted(by_year.values(), key=lambda x: x["year"])[-8:] + + result = { + "score": score, + "africa_led": africa_led, + "north_south": north_south, + "unclassified": unclassified, + "total": total, + "trend": trend, + "interpretation": ( + "Strong" if score >= 60 + else "Moderate" if score >= 35 + else "Dependent" + ), + } + analytics_cache.set(cache_key, result, ttl=1800) + return result + except Exception as e: + logger.error("get_knowledge_repatriation: %s", e) + return {"score": 0, "africa_led": 0, "north_south": 0, + "unclassified": 0, "total": 0, "trend": []} + finally: + session.close() + + # ── Research Portfolio Diversity Score ──────────────────────────────────── + + def get_research_diversity(self, institution: Optional[str] = None) -> Dict: + """Research Portfolio Diversity Score — Shannon entropy across SDGs. + + A diversified research portfolio is more resilient to funding shifts + and more likely to support cross-cutting African development goals. + + Score (0-100): 100 = perfectly uniform across all 17 SDGs. + Compared against a continental peer baseline of ~45 (estimated from + OpenAlex African institution SDG distributions). + No competitor surfaces this as an institutional resilience metric. + Cached 30 min. + """ + import math + + inst_name = self._resolve_institution_name(institution) + cache_key = f"research_diversity:{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + session = SessionLocal() + try: + sc_ids = self._get_sc_item_ids(session, institution) + if not sc_ids: + return {"score": 0, "sdg_distribution": [], "dominant_sdg": None, + "gap_sdgs": [], "interpretation": "No data"} + + rows = (session.query(Item.sdg_tags) + .filter(Item.id.in_(sc_ids), Item.sdg_tags.isnot(None)) + .all()) + + sdg_counts: Dict[str, int] = defaultdict(int) + for (tags,) in rows: + for tag in (tags or "").split(","): + tag = tag.strip() + if tag: + sdg_counts[tag] += 1 + + if not sdg_counts: + return {"score": 0, "sdg_distribution": [], "dominant_sdg": None, + "gap_sdgs": [], "interpretation": "No SDG data"} + + # Shannon entropy normalised to 0-100 + total_tags = sum(sdg_counts.values()) + probs = [c / total_tags for c in sdg_counts.values()] + entropy = -sum(p * math.log(p) for p in probs if p > 0) + max_entropy = math.log(17) # perfectly uniform across 17 SDGs + score = round(entropy / max_entropy * 100, 1) + + # Gap SDGs: SDGs 1-17 not represented at all + represented = {int(k.split()[1]) for k in sdg_counts if k.startswith("SDG") and len(k.split()) > 1} + gap_sdgs = [f"SDG {i}" for i in range(1, 18) if i not in represented] + + dominant = max(sdg_counts, key=sdg_counts.get) if sdg_counts else None + distribution = sorted( + [{"sdg": k, "count": v} for k, v in sdg_counts.items()], + key=lambda x: -x["count"] + )[:17] + + result = { + "score": score, + "sdg_distribution": distribution, + "dominant_sdg": dominant, + "gap_sdgs": gap_sdgs, + "sdg_count": len(sdg_counts), + "baseline_score": 45.0, + "vs_baseline": round(score - 45.0, 1), + "interpretation": ( + "Highly Diversified" if score >= 70 + else "Diversified" if score >= 50 + else "Specialised" if score >= 30 + else "Narrow" + ), + } + analytics_cache.set(cache_key, result, ttl=1800) + return result + except Exception as e: + logger.error("get_research_diversity: %s", e) + return {"score": 0, "sdg_distribution": [], "dominant_sdg": None, + "gap_sdgs": [], "interpretation": "No data"} + finally: + session.close() + + # ── Open Science Health Score ───────────────────────────────────────────── + + def get_open_science_health(self, institution: Optional[str] = None) -> Dict: + """Open Science Health Score — composite OA + PID + reproducibility metric. + + Weighted composite (0-100): + 40 pts Open Access rate (% of papers openly accessible) + 25 pts DOI coverage (% of papers with a DOI) + 20 pts ORCID coverage (% of papers with ≥1 ORCID author) + 15 pts ARK/DocID coverage (% with a persistent institutional PID) + + Provides a single, actionable number for funder compliance, AU Open + Science Policy alignment, and benchmarking against Plan S requirements. + Compared against continental average estimate of ~38/100. Cached 30 min. + """ + inst_name = self._resolve_institution_name(institution) + cache_key = f"open_science_health:{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached is not None: + return cached + + session = SessionLocal() + try: + sc_ids = self._get_sc_item_ids(session, institution) + total = len(sc_ids) + if not total: + return {"score": 0, "components": {}, "total": 0, + "grade": "F", "interpretation": "No data"} + + oa_count = ( + session.query(func.count(Item.id)) + .filter(Item.id.in_(sc_ids), + Item.dc_rights.like("%openAccess%")) + .scalar() + ) or 0 + + doi_count = ( + session.query(func.count(Item.id)) + .filter(Item.id.in_(sc_ids), + Item.doi.isnot(None), Item.doi != "") + .scalar() + ) or 0 + + ark_count = ( + session.query(func.count(Item.id)) + .filter(Item.id.in_(sc_ids), + Item.ark.isnot(None), Item.ark != "") + .scalar() + ) or 0 + + orcid_count = len( + session.query(Item.id) + .join(Item.authors) + .filter(Item.id.in_(sc_ids), + Author.orcid.isnot(None), Author.orcid != "") + .distinct().all() + ) + + def pct(n): return round(n / total * 100, 1) if total else 0.0 + + oa_pct = pct(oa_count) + doi_pct = pct(doi_count) + orcid_pct = pct(orcid_count) + ark_pct = pct(ark_count) + + score = round( + (oa_pct * 0.40) + + (doi_pct * 0.25) + + (orcid_pct * 0.20) + + (ark_pct * 0.15), + 1 + ) + + grade = "A" if score >= 75 else "B" if score >= 55 else "C" if score >= 35 else "D" if score >= 20 else "F" + + result = { + "score": score, + "grade": grade, + "total": total, + "components": { + "open_access": {"value": oa_pct, "weight": 40, "count": oa_count}, + "doi_coverage": {"value": doi_pct, "weight": 25, "count": doi_count}, + "orcid_coverage": {"value": orcid_pct, "weight": 20, "count": orcid_count}, + "ark_coverage": {"value": ark_pct, "weight": 15, "count": ark_count}, + }, + "continental_baseline": 38.0, + "vs_baseline": round(score - 38.0, 1), + "interpretation": ( + "Excellent — Plan S compliant" if score >= 75 + else "Good — approaching open science standards" if score >= 55 + else "Developing — action needed on OA and PIDs" if score >= 35 + else "Critical — significant gaps in open science" + ), + } + analytics_cache.set(cache_key, result, ttl=1800) + return result + except Exception as e: + logger.error("get_open_science_health: %s", e) + return {"score": 0, "components": {}, "total": 0, "grade": "F", + "interpretation": "No data"} + finally: + session.close() + + +analytics = URAASAnalyticsEngine() diff --git a/uraas/config.py b/uraas/config.py index c9198b35e44bccdd920dc5f6e51b1b9eb5cd5048..7146a7aae06ef298d4e96f02965fb5cb73cf39eb 100644 --- a/uraas/config.py +++ b/uraas/config.py @@ -1,29 +1,29 @@ -import os - -from dotenv import load_dotenv - -load_dotenv() - - -class Config: - # Database — defaults to local SQLite; Render overrides via DATABASE_URL. - DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///uraas.db") - - # Redis - REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0") - - # Storage - STORAGE_PATH = os.getenv("STORAGE_PATH", "./storage") - STORAGE_MIN_FREE_GB = float(os.getenv("STORAGE_MIN_FREE_GB", "10.0")) - - # Crawler - RATE_LIMIT_DELAY = float(os.getenv("RATE_LIMIT_DELAY", "2.0")) - MAX_DEPTH = int(os.getenv("MAX_DEPTH", "10")) - CONCURRENT_REQUESTS = int(os.getenv("CONCURRENT_REQUESTS", "16")) - - # Dashboard - DASHBOARD_PORT = int(os.getenv("DASHBOARD_PORT", "8080")) - DASHBOARD_SECRET_KEY = os.getenv("DASHBOARD_SECRET_KEY", "dev-secret-key") - - -config = Config() +import os + +from dotenv import load_dotenv + +load_dotenv() + + +class Config: + # Database — defaults to local SQLite; Render overrides via DATABASE_URL. + DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///uraas.db") + + # Redis + REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0") + + # Storage + STORAGE_PATH = os.getenv("STORAGE_PATH", "./storage") + STORAGE_MIN_FREE_GB = float(os.getenv("STORAGE_MIN_FREE_GB", "10.0")) + + # Crawler + RATE_LIMIT_DELAY = float(os.getenv("RATE_LIMIT_DELAY", "2.0")) + MAX_DEPTH = int(os.getenv("MAX_DEPTH", "10")) + CONCURRENT_REQUESTS = int(os.getenv("CONCURRENT_REQUESTS", "16")) + + # Dashboard + DASHBOARD_PORT = int(os.getenv("DASHBOARD_PORT", "8080")) + DASHBOARD_SECRET_KEY = os.getenv("DASHBOARD_SECRET_KEY", "dev-secret-key") + + +config = Config() diff --git a/uraas/config/__init__.py b/uraas/config/__init__.py index 8a1476eea3bdf941d03dfe7d0f81deafed452795..8037ecf2c753ead53bf547ddd2e8e3c273a13422 100644 --- a/uraas/config/__init__.py +++ b/uraas/config/__init__.py @@ -1,124 +1,124 @@ -""" -Configuration package for URAAS -""" - -# Import config from the root config module -import os - -from dotenv import load_dotenv - -from .institutions import InstitutionConfig, InstitutionRegistry, get_registry - -load_dotenv() - - -class Config: - # Database — defaults to SQLite for local dev. - # In any PostgreSQL deployment, DATABASE_URL must be explicitly set as an env var. - DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///uraas.db") - - # Redis - REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0") - - # Storage - STORAGE_PATH = os.getenv("STORAGE_PATH", "./storage") - STORAGE_MIN_FREE_GB = float(os.getenv("STORAGE_MIN_FREE_GB", "10.0")) - - # Crawler - RATE_LIMIT_DELAY = float(os.getenv("RATE_LIMIT_DELAY", "2.0")) - MAX_DEPTH = int(os.getenv("MAX_DEPTH", "10")) - CONCURRENT_REQUESTS = int(os.getenv("CONCURRENT_REQUESTS", "16")) - - # Dashboard - DASHBOARD_PORT = int(os.getenv("DASHBOARD_PORT", "8080")) - DASHBOARD_SECRET_KEY = os.getenv("DASHBOARD_SECRET_KEY", "dev-secret-key") - - # Authentication — credentials are env-provided; passwords are stored as - # Werkzeug hashes (generate with: - # python -c "from werkzeug.security import generate_password_hash as g; \ - # print(g('your-password'))" - # ). Two roles: admin (full control) and viewer (read + OA downloads only). - ADMIN_USERNAME = os.getenv("ADMIN_USERNAME", "admin") - ADMIN_PASSWORD_HASH = os.getenv("ADMIN_PASSWORD_HASH", "") - VIEWER_USERNAME = os.getenv("VIEWER_USERNAME", "viewer") - VIEWER_PASSWORD_HASH = os.getenv("VIEWER_PASSWORD_HASH", "") - - # Comma-separated list of allowed origins for the SocketIO handshake. - DASHBOARD_CORS_ORIGINS = os.getenv( - "DASHBOARD_CORS_ORIGINS", "http://localhost:8080,http://127.0.0.1:8080" - ) - - @staticmethod - def is_production() -> bool: - """Production when on Render OR explicitly flagged (e.g. a UNILAG DMZ host).""" - return os.getenv("RENDER") == "true" or os.getenv("URAAS_ENV") == "production" - - def validate(self) -> None: - """Fail fast on insecure production config. Called at app startup.""" - if self.is_production(): - if ( - not self.DASHBOARD_SECRET_KEY - or self.DASHBOARD_SECRET_KEY == "dev-secret-key" - ): - raise RuntimeError( - "DASHBOARD_SECRET_KEY must be set to a strong unique value in " - "production (the 'dev-secret-key' default is not allowed)." - ) - if not self.ADMIN_PASSWORD_HASH: - raise RuntimeError( - "ADMIN_PASSWORD_HASH must be set in production " - "(generate with werkzeug.security.generate_password_hash)." - ) - # Warn if CORS origins haven't been tightened for production. - # SocketIO will restrict to localhost-only origins which breaks the - # live dashboard for remote users connecting to the UNILAG server. - if "localhost" in self.DASHBOARD_CORS_ORIGINS: - import warnings - warnings.warn( - "DASHBOARD_CORS_ORIGINS still contains 'localhost' in production. " - "Set DASHBOARD_CORS_ORIGINS to the actual UNILAG HTTPS domain " - "(e.g. https://repository.unilag.edu.ng) in the environment.", - RuntimeWarning, - stacklevel=2, - ) - - def cors_origins(self) -> list: - return [o.strip() for o in self.DASHBOARD_CORS_ORIGINS.split(",") if o.strip()] - - # OpenAlex — free API key (openalex.org/settings/api); keyless access is - # being retired, so set this in production. - OPENALEX_API_KEY = os.getenv("OPENALEX_API_KEY", "") - OPENALEX_MAILTO = os.getenv("OPENALEX_MAILTO", "cokiki@unilag.edu.ng") - - # ARK persistent identifiers (Archival Resource Key) — 99999 is the - # official test NAAN until the Africa PID Alliance NAAN registration lands. - ARK_NAAN = os.getenv("ARK_NAAN", "99999") - # Shoulder must use the betanumeric alphabet (no vowels / no 'l'). - ARK_SHOULDER = os.getenv("ARK_SHOULDER", "z1") - - # ── Public dashboard URL (needed for approval email links) ──────────────── - # Set to the URL users reach the dashboard at (no trailing slash). - DASHBOARD_BASE_URL = os.getenv("DASHBOARD_BASE_URL", "http://localhost:8080").rstrip("/") - - # ── Live DSpace IR (api-ir.unilag.edu.ng) ──────────────────────────────── - # Backend API base — NOT the Angular frontend URL. - DSPACE_API_URL = os.getenv("DSPACE_API_URL", "https://api-ir.unilag.edu.ng/server").rstrip("/") - DSPACE_USERNAME = os.getenv("DSPACE_USERNAME", "") - DSPACE_PASSWORD = os.getenv("DSPACE_PASSWORD", "") - - # ── SMTP for batch approval emails ──────────────────────────────────────── - SMTP_HOST = os.getenv("SMTP_HOST", "") - SMTP_PORT = int(os.getenv("SMTP_PORT", "587")) - SMTP_USE_TLS = os.getenv("SMTP_USE_TLS", "true").lower() == "true" - SMTP_USER = os.getenv("SMTP_USER", "") - SMTP_PASSWORD = os.getenv("SMTP_PASSWORD", "") - SMTP_FROM = os.getenv("SMTP_FROM", "URAAS IR Deposit ") - S2_API_KEY = os.getenv("S2_API_KEY", "") - CORE_API_KEY = os.getenv("CORE_API_KEY", "") - NCBI_API_KEY = os.getenv("NCBI_API_KEY", "") - LENS_API_KEY = os.getenv("LENS_API_KEY", "") - - -config = Config() - -__all__ = ["InstitutionConfig", "InstitutionRegistry", "get_registry", "config"] +""" +Configuration package for URAAS +""" + +# Import config from the root config module +import os + +from dotenv import load_dotenv + +from .institutions import InstitutionConfig, InstitutionRegistry, get_registry + +load_dotenv() + + +class Config: + # Database — defaults to SQLite for local dev. + # In any PostgreSQL deployment, DATABASE_URL must be explicitly set as an env var. + DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///uraas.db") + + # Redis + REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0") + + # Storage + STORAGE_PATH = os.getenv("STORAGE_PATH", "./storage") + STORAGE_MIN_FREE_GB = float(os.getenv("STORAGE_MIN_FREE_GB", "10.0")) + + # Crawler + RATE_LIMIT_DELAY = float(os.getenv("RATE_LIMIT_DELAY", "2.0")) + MAX_DEPTH = int(os.getenv("MAX_DEPTH", "10")) + CONCURRENT_REQUESTS = int(os.getenv("CONCURRENT_REQUESTS", "16")) + + # Dashboard + DASHBOARD_PORT = int(os.getenv("DASHBOARD_PORT", "8080")) + DASHBOARD_SECRET_KEY = os.getenv("DASHBOARD_SECRET_KEY", "dev-secret-key") + + # Authentication — credentials are env-provided; passwords are stored as + # Werkzeug hashes (generate with: + # python -c "from werkzeug.security import generate_password_hash as g; \ + # print(g('your-password'))" + # ). Two roles: admin (full control) and viewer (read + OA downloads only). + ADMIN_USERNAME = os.getenv("ADMIN_USERNAME", "admin") + ADMIN_PASSWORD_HASH = os.getenv("ADMIN_PASSWORD_HASH", "") + VIEWER_USERNAME = os.getenv("VIEWER_USERNAME", "viewer") + VIEWER_PASSWORD_HASH = os.getenv("VIEWER_PASSWORD_HASH", "") + + # Comma-separated list of allowed origins for the SocketIO handshake. + DASHBOARD_CORS_ORIGINS = os.getenv( + "DASHBOARD_CORS_ORIGINS", "http://localhost:8080,http://127.0.0.1:8080" + ) + + @staticmethod + def is_production() -> bool: + """Production when on Render OR explicitly flagged (e.g. a UNILAG DMZ host).""" + return os.getenv("RENDER") == "true" or os.getenv("URAAS_ENV") == "production" + + def validate(self) -> None: + """Fail fast on insecure production config. Called at app startup.""" + if self.is_production(): + if ( + not self.DASHBOARD_SECRET_KEY + or self.DASHBOARD_SECRET_KEY == "dev-secret-key" + ): + raise RuntimeError( + "DASHBOARD_SECRET_KEY must be set to a strong unique value in " + "production (the 'dev-secret-key' default is not allowed)." + ) + if not self.ADMIN_PASSWORD_HASH: + raise RuntimeError( + "ADMIN_PASSWORD_HASH must be set in production " + "(generate with werkzeug.security.generate_password_hash)." + ) + # Warn if CORS origins haven't been tightened for production. + # SocketIO will restrict to localhost-only origins which breaks the + # live dashboard for remote users connecting to the UNILAG server. + if "localhost" in self.DASHBOARD_CORS_ORIGINS: + import warnings + warnings.warn( + "DASHBOARD_CORS_ORIGINS still contains 'localhost' in production. " + "Set DASHBOARD_CORS_ORIGINS to the actual UNILAG HTTPS domain " + "(e.g. https://repository.unilag.edu.ng) in the environment.", + RuntimeWarning, + stacklevel=2, + ) + + def cors_origins(self) -> list: + return [o.strip() for o in self.DASHBOARD_CORS_ORIGINS.split(",") if o.strip()] + + # OpenAlex — free API key (openalex.org/settings/api); keyless access is + # being retired, so set this in production. + OPENALEX_API_KEY = os.getenv("OPENALEX_API_KEY", "") + OPENALEX_MAILTO = os.getenv("OPENALEX_MAILTO", "cokiki@unilag.edu.ng") + + # ARK persistent identifiers (Archival Resource Key) — 99999 is the + # official test NAAN until the Africa PID Alliance NAAN registration lands. + ARK_NAAN = os.getenv("ARK_NAAN", "99999") + # Shoulder must use the betanumeric alphabet (no vowels / no 'l'). + ARK_SHOULDER = os.getenv("ARK_SHOULDER", "z1") + + # ── Public dashboard URL (needed for approval email links) ──────────────── + # Set to the URL users reach the dashboard at (no trailing slash). + DASHBOARD_BASE_URL = os.getenv("DASHBOARD_BASE_URL", "http://localhost:8080").rstrip("/") + + # ── Live DSpace IR (api-ir.unilag.edu.ng) ──────────────────────────────── + # Backend API base — NOT the Angular frontend URL. + DSPACE_API_URL = os.getenv("DSPACE_API_URL", "https://api-ir.unilag.edu.ng/server").rstrip("/") + DSPACE_USERNAME = os.getenv("DSPACE_USERNAME", "") + DSPACE_PASSWORD = os.getenv("DSPACE_PASSWORD", "") + + # ── SMTP for batch approval emails ──────────────────────────────────────── + SMTP_HOST = os.getenv("SMTP_HOST", "") + SMTP_PORT = int(os.getenv("SMTP_PORT", "587")) + SMTP_USE_TLS = os.getenv("SMTP_USE_TLS", "true").lower() == "true" + SMTP_USER = os.getenv("SMTP_USER", "") + SMTP_PASSWORD = os.getenv("SMTP_PASSWORD", "") + SMTP_FROM = os.getenv("SMTP_FROM", "URAAS IR Deposit ") + S2_API_KEY = os.getenv("S2_API_KEY", "") + CORE_API_KEY = os.getenv("CORE_API_KEY", "") + NCBI_API_KEY = os.getenv("NCBI_API_KEY", "") + LENS_API_KEY = os.getenv("LENS_API_KEY", "") + + +config = Config() + +__all__ = ["InstitutionConfig", "InstitutionRegistry", "get_registry", "config"] diff --git a/uraas/config/african_countries.py b/uraas/config/african_countries.py index 3cafae380babe9b96b5112885c1e53ee9379fc66..1aaaddc38bd5e7415b333170fc0e02dc7b282f12 100644 --- a/uraas/config/african_countries.py +++ b/uraas/config/african_countries.py @@ -1,136 +1,136 @@ -""" -African country reference data (AU member states). - -- AFRICAN_ISO2: ISO 3166-1 alpha-2 codes used to classify OpenAlex - authorship country_code values as African. -- COUNTRY_NAMES: ISO2 -> display name (matches Natural Earth admin-0 names - where possible so the choropleth join works both ways). -- COUNTRY_CENTROIDS: ISO2 -> (lat, lon) display anchors for collaboration - arcs and map labels. Approximate visual centroids — chosen to sit inside - each country's land area, not geodetic centroids. -""" - -COUNTRY_NAMES = { - "DZ": "Algeria", - "AO": "Angola", - "BJ": "Benin", - "BW": "Botswana", - "BF": "Burkina Faso", - "BI": "Burundi", - "CV": "Cabo Verde", - "CM": "Cameroon", - "CF": "Central African Republic", - "TD": "Chad", - "KM": "Comoros", - "CG": "Republic of the Congo", - "CD": "Democratic Republic of the Congo", - "CI": "Côte d'Ivoire", - "DJ": "Djibouti", - "EG": "Egypt", - "GQ": "Equatorial Guinea", - "ER": "Eritrea", - "SZ": "Eswatini", - "ET": "Ethiopia", - "GA": "Gabon", - "GM": "Gambia", - "GH": "Ghana", - "GN": "Guinea", - "GW": "Guinea-Bissau", - "KE": "Kenya", - "LS": "Lesotho", - "LR": "Liberia", - "LY": "Libya", - "MG": "Madagascar", - "MW": "Malawi", - "ML": "Mali", - "MR": "Mauritania", - "MU": "Mauritius", - "MA": "Morocco", - "MZ": "Mozambique", - "NA": "Namibia", - "NE": "Niger", - "NG": "Nigeria", - "RW": "Rwanda", - "ST": "São Tomé and Príncipe", - "SN": "Senegal", - "SC": "Seychelles", - "SL": "Sierra Leone", - "SO": "Somalia", - "ZA": "South Africa", - "SS": "South Sudan", - "SD": "Sudan", - "TZ": "Tanzania", - "TG": "Togo", - "TN": "Tunisia", - "UG": "Uganda", - "ZM": "Zambia", - "ZW": "Zimbabwe", - # AU member state via the Sahrawi Republic; Natural Earth has a geometry. - "EH": "Western Sahara", -} - -AFRICAN_ISO2 = frozenset(COUNTRY_NAMES) - -# (lat, lon) — display anchors for arcs/labels. -COUNTRY_CENTROIDS = { - "DZ": (28.0, 2.6), - "AO": (-12.3, 17.5), - "BJ": (9.6, 2.3), - "BW": (-22.2, 23.8), - "BF": (12.3, -1.7), - "BI": (-3.4, 29.9), - "CV": (15.1, -23.6), - "CM": (5.7, 12.7), - "CF": (6.6, 20.5), - "TD": (15.4, 18.7), - "KM": (-11.7, 43.3), - "CG": (-0.8, 15.2), - "CD": (-2.9, 23.6), - "CI": (7.6, -5.6), - "DJ": (11.7, 42.6), - "EG": (26.6, 29.8), - "GQ": (1.6, 10.4), - "ER": (15.4, 38.8), - "SZ": (-26.6, 31.5), - "ET": (8.6, 39.6), - "GA": (-0.6, 11.8), - "GM": (13.4, -15.4), - "GH": (7.9, -1.2), - "GN": (10.4, -11.0), - "GW": (12.0, -15.0), - "KE": (0.5, 37.9), - "LS": (-29.6, 28.2), - "LR": (6.4, -9.3), - "LY": (27.0, 17.3), - "MG": (-19.4, 46.7), - "MW": (-13.2, 34.3), - "ML": (17.4, -4.0), - "MR": (20.3, -10.4), - "MU": (-20.3, 57.6), - "MA": (31.9, -6.9), - "MZ": (-17.3, 35.5), - "NA": (-22.1, 17.2), - "NE": (17.4, 9.4), - "NG": (9.6, 8.1), - "RW": (-2.0, 29.9), - "ST": (0.2, 6.6), - "SN": (14.4, -14.5), - "SC": (-4.7, 55.5), - "SL": (8.6, -11.8), - "SO": (6.1, 45.9), - "ZA": (-29.0, 25.1), - "SS": (7.3, 30.3), - "SD": (16.0, 30.0), - "TZ": (-6.4, 34.8), - "TG": (8.5, 1.0), - "TN": (34.1, 9.6), - "UG": (1.3, 32.4), - "ZM": (-13.5, 27.8), - "ZW": (-19.0, 29.9), - "EH": (24.6, -13.1), -} - - -def african_countries_in(codes) -> list: - """Sorted list of distinct African ISO2 codes from an iterable.""" - return sorted({(c or "").upper() for c in codes} & AFRICAN_ISO2) +""" +African country reference data (AU member states). + +- AFRICAN_ISO2: ISO 3166-1 alpha-2 codes used to classify OpenAlex + authorship country_code values as African. +- COUNTRY_NAMES: ISO2 -> display name (matches Natural Earth admin-0 names + where possible so the choropleth join works both ways). +- COUNTRY_CENTROIDS: ISO2 -> (lat, lon) display anchors for collaboration + arcs and map labels. Approximate visual centroids — chosen to sit inside + each country's land area, not geodetic centroids. +""" + +COUNTRY_NAMES = { + "DZ": "Algeria", + "AO": "Angola", + "BJ": "Benin", + "BW": "Botswana", + "BF": "Burkina Faso", + "BI": "Burundi", + "CV": "Cabo Verde", + "CM": "Cameroon", + "CF": "Central African Republic", + "TD": "Chad", + "KM": "Comoros", + "CG": "Republic of the Congo", + "CD": "Democratic Republic of the Congo", + "CI": "Côte d'Ivoire", + "DJ": "Djibouti", + "EG": "Egypt", + "GQ": "Equatorial Guinea", + "ER": "Eritrea", + "SZ": "Eswatini", + "ET": "Ethiopia", + "GA": "Gabon", + "GM": "Gambia", + "GH": "Ghana", + "GN": "Guinea", + "GW": "Guinea-Bissau", + "KE": "Kenya", + "LS": "Lesotho", + "LR": "Liberia", + "LY": "Libya", + "MG": "Madagascar", + "MW": "Malawi", + "ML": "Mali", + "MR": "Mauritania", + "MU": "Mauritius", + "MA": "Morocco", + "MZ": "Mozambique", + "NA": "Namibia", + "NE": "Niger", + "NG": "Nigeria", + "RW": "Rwanda", + "ST": "São Tomé and Príncipe", + "SN": "Senegal", + "SC": "Seychelles", + "SL": "Sierra Leone", + "SO": "Somalia", + "ZA": "South Africa", + "SS": "South Sudan", + "SD": "Sudan", + "TZ": "Tanzania", + "TG": "Togo", + "TN": "Tunisia", + "UG": "Uganda", + "ZM": "Zambia", + "ZW": "Zimbabwe", + # AU member state via the Sahrawi Republic; Natural Earth has a geometry. + "EH": "Western Sahara", +} + +AFRICAN_ISO2 = frozenset(COUNTRY_NAMES) + +# (lat, lon) — display anchors for arcs/labels. +COUNTRY_CENTROIDS = { + "DZ": (28.0, 2.6), + "AO": (-12.3, 17.5), + "BJ": (9.6, 2.3), + "BW": (-22.2, 23.8), + "BF": (12.3, -1.7), + "BI": (-3.4, 29.9), + "CV": (15.1, -23.6), + "CM": (5.7, 12.7), + "CF": (6.6, 20.5), + "TD": (15.4, 18.7), + "KM": (-11.7, 43.3), + "CG": (-0.8, 15.2), + "CD": (-2.9, 23.6), + "CI": (7.6, -5.6), + "DJ": (11.7, 42.6), + "EG": (26.6, 29.8), + "GQ": (1.6, 10.4), + "ER": (15.4, 38.8), + "SZ": (-26.6, 31.5), + "ET": (8.6, 39.6), + "GA": (-0.6, 11.8), + "GM": (13.4, -15.4), + "GH": (7.9, -1.2), + "GN": (10.4, -11.0), + "GW": (12.0, -15.0), + "KE": (0.5, 37.9), + "LS": (-29.6, 28.2), + "LR": (6.4, -9.3), + "LY": (27.0, 17.3), + "MG": (-19.4, 46.7), + "MW": (-13.2, 34.3), + "ML": (17.4, -4.0), + "MR": (20.3, -10.4), + "MU": (-20.3, 57.6), + "MA": (31.9, -6.9), + "MZ": (-17.3, 35.5), + "NA": (-22.1, 17.2), + "NE": (17.4, 9.4), + "NG": (9.6, 8.1), + "RW": (-2.0, 29.9), + "ST": (0.2, 6.6), + "SN": (14.4, -14.5), + "SC": (-4.7, 55.5), + "SL": (8.6, -11.8), + "SO": (6.1, 45.9), + "ZA": (-29.0, 25.1), + "SS": (7.3, 30.3), + "SD": (16.0, 30.0), + "TZ": (-6.4, 34.8), + "TG": (8.5, 1.0), + "TN": (34.1, 9.6), + "UG": (1.3, 32.4), + "ZM": (-13.5, 27.8), + "ZW": (-19.0, 29.9), + "EH": (24.6, -13.1), +} + + +def african_countries_in(codes) -> list: + """Sorted list of distinct African ISO2 codes from an iterable.""" + return sorted({(c or "").upper() for c in codes} & AFRICAN_ISO2) diff --git a/uraas/config/alignment_frameworks.py b/uraas/config/alignment_frameworks.py index df84dc82bb5a31f461644324a5b4a4bbabdcbe47..100f2715221988fdb889e1e585a0cd47ff0b512c 100644 --- a/uraas/config/alignment_frameworks.py +++ b/uraas/config/alignment_frameworks.py @@ -1,1111 +1,1111 @@ -"""Alignment scoring taxonomy for continental and regional policy frameworks. - -This module defines the canonical taxonomy used by the alignment engine to score -African research output against three families of policy frameworks: - - 1. AU Charters (Banjul, Public Service, Cultural Renaissance, Malabo, Youth, ACDEG) - 2. AU Agenda 2063 (the seven aspirations, with their numbered goals) - 3. Regional bloc strategies (ECOWAS Vision 2050, SADC RISDP 2020-2030, EAC Vision 2050) - -Each framework is decomposed into thematic pillars. A pillar carries two layers: - - - ``description``: 2-3 sentences of prose written in the vocabulary of African - academic paper abstracts. These descriptions are embedded by a Model2Vec - sentence-embedding model and compared with paper abstracts via cosine - similarity to produce the semantic component of the alignment score. - - ``keywords``: a curated term list that forms the auditable evidence layer. - Keyword hits in titles/abstracts are surfaced to users as transparent - justification for why a paper was mapped to a pillar. - -Plain dicts only — this module must stay import-light and serialisable. -""" - -ALIGNMENT_VERSION = 1 - -# Minimum pillar coverage (percent) below which a framework pillar is flagged -# as a research gap for an institution or country. -GAP_THRESHOLD = 25 - -FRAMEWORKS = { - # ------------------------------------------------------------------ - # AU Charters - # ------------------------------------------------------------------ - "banjul": { - "name": "African Charter on Human and Peoples' Rights (Banjul Charter)", - "type": "au_charter", - "year": 1981, - "color": "#1d4ed8", # blue - "pillars": { - "civil_political_rights": { - "name": "Civil and Political Rights", - "description": ( - "Research addressing civil liberties, political participation, due " - "process, freedom of expression and assembly in African states. " - "Includes studies of fair trial rights, arbitrary detention, press " - "freedom, torture prevention and the rule of law. Examines how " - "constitutions, courts and security agencies protect or violate " - "fundamental freedoms." - ), - "keywords": [ - "human rights", "civil liberties", "freedom of expression", - "press freedom", "fair trial", "rule of law", - "arbitrary detention", "torture", "freedom of association", - "political rights", "right to life", "due process", - "fundamental freedoms", - ], - }, - "socioeconomic_cultural_rights": { - "name": "Socio-economic and Cultural Rights", - "description": ( - "Studies of the right to health, education, work, housing and " - "social protection in African countries. Covers access to " - "healthcare and schooling, labour rights, decent working " - "conditions and social justice for poor and marginalised " - "populations. Includes analyses of state obligations to " - "progressively realise socio-economic and cultural rights." - ), - "keywords": [ - "right to health", "right to education", "socio-economic rights", - "social protection", "labour rights", "right to work", - "housing rights", "cultural rights", "social justice", - "access to healthcare", "educational access", - ], - }, - "peoples_collective_rights": { - "name": "Peoples' and Collective Rights", - "description": ( - "Research on collective and peoples' rights, including " - "self-determination, the right to development and sovereignty " - "over natural resources. Covers environmental rights and " - "environmental justice, land rights, community rights and the " - "claims of indigenous peoples. Includes studies of resource " - "governance, extractive industries and communities affected by " - "mining, oil and land dispossession." - ), - "keywords": [ - "self-determination", "right to development", "collective rights", - "environmental rights", "indigenous peoples", - "natural resource sovereignty", "peoples' rights", - "right to peace", "land rights", "community rights", - "environmental justice", "resource governance", - ], - }, - "vulnerable_groups_protection": { - "name": "Protection of Vulnerable Groups", - "description": ( - "Research on the rights and protection of women, children, " - "persons with disabilities, older persons, refugees and " - "internally displaced persons in Africa. Covers gender " - "discrimination, gender-based violence, child labour, human " - "trafficking and minority rights. Includes studies of social " - "inclusion and legal frameworks safeguarding vulnerable and " - "marginalised populations." - ), - "keywords": [ - "women's rights", "child protection", "disability rights", - "rights of older persons", "gender discrimination", - "child labour", "human trafficking", "refugee rights", - "internally displaced persons", "minority rights", - "gender-based violence", "social inclusion", - ], - }, - "african_human_rights_system": { - "name": "African Human Rights System", - "description": ( - "Scholarship on the institutions and procedures of the African " - "human rights system, including the African Commission and the " - "African Court on Human and Peoples' Rights. Examines human " - "rights litigation, jurisprudence, state reporting, treaty " - "compliance and domestication of regional instruments. Includes " - "studies of transitional justice and accountability mechanisms " - "for past violations." - ), - "keywords": [ - "African Commission on Human and Peoples' Rights", - "African Court", "human rights litigation", "treaty compliance", - "state reporting", "regional human rights mechanisms", - "human rights jurisprudence", "domestication of treaties", - "transitional justice", "accountability", - ], - }, - }, - }, - "public_service": { - "name": ( - "African Charter on Values and Principles of Public Service " - "and Administration" - ), - "type": "au_charter", - "year": 2011, - "color": "#0f766e", # teal - "pillars": { - "service_delivery": { - "name": "Public Service Delivery", - "description": ( - "Research on the quality, efficiency and equity of public " - "service delivery in African countries. Covers citizen " - "satisfaction, local government services, public utilities and " - "public sector performance management. Includes studies of " - "service delivery innovation and administrative reforms aimed " - "at improving outcomes for citizens." - ), - "keywords": [ - "public service delivery", "service quality", - "citizen satisfaction", "local government services", - "public sector performance", "service delivery innovation", - "public utilities", "administrative efficiency", - "performance management", - ], - }, - "ethics_anticorruption": { - "name": "Ethics and Anti-corruption", - "description": ( - "Studies of corruption, bribery and integrity in the African " - "public sector. Covers anti-corruption agencies and reforms, " - "procurement fraud, illicit enrichment, conflict of interest, " - "asset declaration and whistleblowing. Includes research on " - "public sector ethics, codes of conduct, accountability and " - "citizens' trust in government." - ), - "keywords": [ - "corruption", "anti-corruption", "public sector ethics", - "accountability", "integrity", "bribery", - "conflict of interest", "asset declaration", "whistleblowing", - "procurement fraud", "illicit enrichment", "code of conduct", - "public trust", - ], - }, - "transparency_access_to_information": { - "name": "Transparency and Access to Information", - "description": ( - "Research on access to information, freedom of information laws " - "and open government in Africa. Covers transparency in public " - "budgets and fiscal management, open data initiatives and " - "disclosure of public records. Includes studies of the right to " - "information as a tool for accountability and citizen oversight." - ), - "keywords": [ - "access to information", "freedom of information", - "open government", "transparency", "open data", - "fiscal transparency", "budget transparency", "public records", - "right to information", "information disclosure", - ], - }, - "egovernment_modernisation": { - "name": "E-government and Administrative Modernisation", - "description": ( - "Research on e-government, digital government and the digital " - "transformation of public administration in Africa. Covers " - "digital public services, digital identity systems, civil " - "service and bureaucratic reform, and administrative " - "modernisation programmes. Includes studies of e-governance " - "adoption, implementation challenges and impacts on service " - "delivery." - ), - "keywords": [ - "e-government", "digital government", "digital public services", - "civil service reform", "public administration reform", - "digital identity", "administrative modernisation", - "bureaucratic reform", "digital transformation of government", - "e-governance", - ], - }, - "civil_service_hrm": { - "name": "Civil Service and Human Resource Management", - "description": ( - "Studies of civil service systems, public sector employment and " - "human resource management in African states. Covers " - "meritocracy, recruitment, training and capacity building of " - "public officials, motivation and working conditions of public " - "servants. Includes research on decentralisation and the " - "staffing and management of local governance structures." - ), - "keywords": [ - "civil service", "public servants", - "human resource management", "public sector employment", - "meritocracy", "capacity building", - "training of public officials", "public sector motivation", - "working conditions", "decentralisation", "local governance", - ], - }, - }, - }, - "cultural_renaissance": { - "name": "Charter for African Cultural Renaissance", - "type": "au_charter", - "year": 2006, - "color": "#b45309", # amber - "pillars": { - "heritage_preservation_restitution": { - "name": "Heritage Preservation and Restitution", - "description": ( - "Research on the preservation, conservation and digitisation of " - "African cultural heritage, including museums, archives, " - "monuments and world heritage sites. Covers archaeology and " - "heritage management as well as the restitution and " - "repatriation of cultural property and looted artefacts. " - "Includes debates on ownership, custodianship and access to " - "African cultural objects." - ), - "keywords": [ - "cultural heritage", "heritage preservation", "museums", - "archives", "restitution of cultural property", - "repatriation of artefacts", "world heritage sites", - "monuments", "archaeology", "heritage conservation", - "digitisation of heritage", "cultural property", - ], - }, - "african_languages": { - "name": "African Languages", - "description": ( - "Linguistic research on African and indigenous languages, " - "including documentation of endangered languages, orthography " - "development, sociolinguistics and translation. Covers language " - "policy, mother tongue education and multilingualism in African " - "classrooms and societies. Includes studies of major languages " - "such as Yoruba, Swahili, Hausa and Igbo." - ), - "keywords": [ - "African languages", "indigenous languages", "language policy", - "mother tongue education", "multilingualism", "linguistics", - "language documentation", "endangered languages", "Yoruba", - "Swahili", "Hausa", "Igbo", "orthography", "sociolinguistics", - "translation", - ], - }, - "creative_cultural_industries": { - "name": "Creative and Cultural Industries", - "description": ( - "Research on Africa's creative economy and cultural industries, " - "including film, music, literature, publishing, performing and " - "visual arts. Covers Nollywood and African cinema, festivals, " - "cultural tourism and cultural entrepreneurship. Includes " - "studies of the economic contribution, value chains and policy " - "environment of the creative sector." - ), - "keywords": [ - "creative industries", "creative economy", "Nollywood", - "African cinema", "African music", "performing arts", - "visual arts", "cultural entrepreneurship", "film industry", - "literature", "publishing", "festivals", "cultural tourism", - ], - }, - "identity_diversity": { - "name": "Cultural Identity and Diversity", - "description": ( - "Scholarship on African and cultural identity, pan-Africanism, " - "afrocentricity and cultural diversity. Covers oral tradition, " - "folklore, ethnicity, religion and cultural values, and " - "intercultural dialogue within and between African societies. " - "Includes studies of decolonisation of culture, thought and " - "representation." - ), - "keywords": [ - "cultural identity", "African identity", "pan-Africanism", - "cultural diversity", "intercultural dialogue", - "decolonisation", "afrocentricity", "oral tradition", - "folklore", "ethnicity", "religion and culture", - "cultural values", - ], - }, - "indigenous_knowledge_systems": { - "name": "Indigenous Knowledge Systems", - "description": ( - "Research on indigenous, traditional and endogenous knowledge " - "systems in Africa, including traditional medicine, " - "ethnobotany, ethnomedicine and traditional ecological " - "knowledge. Covers the documentation, validation and protection " - "of traditional knowledge and cultural expressions. Includes " - "debates on knowledge sovereignty, biopiracy and benefit " - "sharing." - ), - "keywords": [ - "indigenous knowledge", "traditional knowledge", - "endogenous knowledge", "traditional medicine", "ethnobotany", - "ethnomedicine", "indigenous knowledge systems", - "traditional ecological knowledge", "knowledge sovereignty", - "biopiracy", "traditional cultural expressions", - ], - }, - "diaspora_historical_memory": { - "name": "Diaspora and Historical Memory", - "description": ( - "Historical research on the African diaspora, the slave trade, " - "colonialism and their post-colonial legacies. Covers African " - "history, oral history, memory studies and the politics of " - "historical memory and reparations. Includes scholarship on the " - "African renaissance and connections between the continent and " - "its diaspora." - ), - "keywords": [ - "African diaspora", "slave trade", "historical memory", - "colonialism", "post-colonial", "African history", - "oral history", "memory studies", "reparations", - "African renaissance", - ], - }, - }, - }, - "malabo": { - "name": ( - "AU Convention on Cyber Security and Personal Data Protection " - "(Malabo Convention)" - ), - "type": "au_charter", - "year": 2014, - "color": "#6d28d9", # violet - "pillars": { - "electronic_transactions": { - "name": "Electronic Transactions and Digital Economy", - "description": ( - "Research on e-commerce, electronic transactions and the " - "digital economy in Africa. Covers digital payments, mobile " - "money, fintech, digital financial services and online " - "marketplaces. Includes studies of electronic contracts, " - "electronic signatures, payment systems and the regulation of " - "digital trade." - ), - "keywords": [ - "e-commerce", "electronic transactions", "digital payments", - "mobile money", "electronic contracts", "digital trade", - "fintech", "online marketplaces", "electronic signatures", - "digital economy", "payment systems", - "digital financial services", - ], - }, - "data_protection_privacy": { - "name": "Data Protection and Privacy", - "description": ( - "Studies of personal data protection, privacy law and data " - "governance in African countries. Covers data protection " - "authorities, consent, biometric data, cross-border data flows, " - "data sovereignty and data localisation. Includes analyses of " - "compliance with privacy frameworks and protection of " - "information privacy in digital systems." - ), - "keywords": [ - "data protection", "personal data", "privacy", "data privacy", - "data protection authority", "consent", "data governance", - "cross-border data flows", "data sovereignty", - "data localisation", "privacy law", "information privacy", - "biometric data", - ], - }, - "cybersecurity": { - "name": "Cybersecurity", - "description": ( - "Research on cybersecurity, network and information security in " - "African contexts. Covers protection of critical " - "infrastructure, cyber resilience, cyber threats, malware, " - "intrusion detection, encryption and vulnerability assessment. " - "Includes studies of national cybersecurity strategies, " - "capacity and incident response." - ), - "keywords": [ - "cybersecurity", "cyber security", "network security", - "information security", "critical infrastructure protection", - "cyber resilience", "cyber threats", "malware", - "intrusion detection", "encryption", - "vulnerability assessment", - ], - }, - "cybercrime": { - "name": "Cybercrime", - "description": ( - "Research on cybercrime and online criminality in Africa, " - "including computer fraud, phishing, identity theft, hacking " - "and ransomware. Covers cybercrime legislation, digital " - "forensics, electronic evidence and law enforcement responses " - "to internet fraud. Includes criminological studies of online " - "fraud networks and victimisation." - ), - "keywords": [ - "cybercrime", "computer fraud", "phishing", "online fraud", - "identity theft", "ransomware", "digital forensics", - "cybercrime legislation", "electronic evidence", - "internet fraud", "hacking", - ], - }, - "digital_rights_cooperation": { - "name": "Digital Rights and Internet Governance", - "description": ( - "Scholarship on digital rights, internet governance and online " - "freedom of expression in Africa. Covers surveillance, internet " - "shutdowns, platform regulation, misinformation and digital " - "inclusion. Includes studies of internet policy and the balance " - "between security, rights and access in digital spaces." - ), - "keywords": [ - "digital rights", "internet governance", - "online freedom of expression", "surveillance", - "internet shutdowns", "digital inclusion", "internet policy", - "platform regulation", "misinformation", - ], - }, - }, - }, - "youth": { - "name": "African Youth Charter", - "type": "au_charter", - "year": 2006, - "color": "#15803d", # green - "pillars": { - "education_skills": { - "name": "Youth Education and Skills", - "description": ( - "Research on education and skills development for African " - "youth, including technical and vocational education and " - "training (TVET), tertiary education and STEM education. Covers " - "literacy, digital skills, curriculum reform, graduate " - "employability skills and access to education. Includes studies " - "of out-of-school youth and barriers to educational attainment." - ), - "keywords": [ - "youth education", "technical and vocational education", "TVET", - "skills development", "tertiary education", - "out-of-school youth", "digital skills", "literacy", - "STEM education", "curriculum reform", "graduate skills", - "education access", - ], - }, - "employment_entrepreneurship": { - "name": "Youth Employment and Entrepreneurship", - "description": ( - "Studies of youth employment, unemployment and livelihoods in " - "African labour markets. Covers entrepreneurship and youth-led " - "start-ups, self-employment, the informal sector, the gig " - "economy and small and medium enterprises. Includes research on " - "job creation, decent work and policies to absorb young people " - "into productive employment." - ), - "keywords": [ - "youth employment", "youth unemployment", "entrepreneurship", - "youth entrepreneurship", "informal sector", "job creation", - "start-ups", "self-employment", "labour market", "decent work", - "gig economy", "small and medium enterprises", "livelihoods", - ], - }, - "youth_health_wellbeing": { - "name": "Youth Health and Wellbeing", - "description": ( - "Research on adolescent and youth health in Africa, including " - "sexual and reproductive health, HIV/AIDS, teenage pregnancy " - "and adolescent nutrition. Covers substance and drug abuse, " - "mental health and access to youth-friendly health services. " - "Includes studies of risk behaviours and interventions " - "promoting young people's wellbeing." - ), - "keywords": [ - "adolescent health", "youth health", "HIV/AIDS", - "sexual and reproductive health", "teenage pregnancy", - "substance abuse", "drug abuse", "mental health", - "adolescent nutrition", "youth-friendly services", - ], - }, - "participation_civic_engagement": { - "name": "Youth Participation and Civic Engagement", - "description": ( - "Scholarship on youth participation in politics, governance and " - "civic life in Africa. Covers youth policy, youth leadership " - "and representation, student activism, social movements, " - "volunteerism and digital activism. Includes studies of how " - "young people engage with, contest and shape political " - "processes." - ), - "keywords": [ - "youth participation", "civic engagement", "youth in politics", - "youth policy", "student activism", "youth leadership", - "youth representation", "social movements", "volunteerism", - "digital activism", "political participation", - ], - }, - "youth_peace_security": { - "name": "Youth, Peace and Security", - "description": ( - "Research on young people in conflict and peacebuilding in " - "Africa, including youth radicalisation, violent extremism, " - "gangs, cultism and youth violence. Covers the roles of youth " - "in post-conflict recovery and the youth, peace and security " - "agenda. Includes studies of drivers of youth recruitment into " - "armed groups and programmes for reintegration." - ), - "keywords": [ - "youth and conflict", "youth radicalisation", - "violent extremism", "youth in peacebuilding", "gangs", - "cultism", "youth violence", "post-conflict youth", - "youth peace and security", - ], - }, - }, - }, - "acdeg": { - "name": "African Charter on Democracy, Elections and Governance (ACDEG)", - "type": "au_charter", - "year": 2007, - "color": "#b91c1c", # red - "pillars": { - "democracy_rule_of_law": { - "name": "Democracy and Rule of Law", - "description": ( - "Research on democracy, democratisation and constitutionalism " - "in African states. Covers the rule of law, separation of " - "powers, judicial independence, constitutional reform, " - "political parties and civil society. Includes studies of " - "democratic consolidation, democratic backsliding, " - "authoritarianism and presidential term limits." - ), - "keywords": [ - "democracy", "democratisation", "constitutionalism", - "rule of law", "separation of powers", "judicial independence", - "constitutional reform", "civil society", "political parties", - "democratic consolidation", "authoritarianism", "term limits", - "democratic backsliding", - ], - }, - "elections": { - "name": "Elections and Electoral Integrity", - "description": ( - "Studies of elections and electoral processes in Africa, " - "including electoral commissions, voter registration, biometric " - "voter technologies and election observation. Covers electoral " - "violence, electoral reform, election petitions, campaign " - "finance and voter turnout. Includes research on the conduct " - "and integrity of free and fair elections." - ), - "keywords": [ - "elections", "electoral commission", "election observation", - "voter registration", "electoral violence", - "free and fair elections", "electoral reform", "voter turnout", - "election petition", "biometric voter", "electoral integrity", - "campaign finance", - ], - }, - "unconstitutional_change": { - "name": "Unconstitutional Changes of Government", - "description": ( - "Research on coups d'etat, military rule and unconstitutional " - "changes of government in Africa. Covers civil-military " - "relations, juntas, political transitions, power transfers and " - "third-term bids that subvert constitutional order. Includes " - "studies of regional and continental responses to military " - "takeovers." - ), - "keywords": [ - "coup d'etat", "military coup", - "unconstitutional change of government", "military rule", - "civil-military relations", "junta", "political transition", - "power transfer", "third-term bids", - ], - }, - "governance_institutions": { - "name": "Governance and Public Institutions", - "description": ( - "Scholarship on good governance and the performance of public " - "institutions in Africa. Covers anti-corruption, " - "accountability, transparency, the African Peer Review " - "Mechanism, decentralisation and local government. Includes " - "studies of state capacity, institutional reform and political " - "leadership." - ), - "keywords": [ - "good governance", "governance", "anti-corruption", - "accountability", "African Peer Review Mechanism", - "public institutions", "decentralisation", "local government", - "state capacity", "institutional reform", "transparency", - "leadership", - ], - }, - "participation_gender_equity": { - "name": "Participation, Gender and Equity", - "description": ( - "Research on political participation and inclusive governance " - "in Africa, with emphasis on women in politics, gender quotas " - "and women's political leadership. Covers citizen " - "participation, participatory democracy, civic education and " - "the representation of marginalised groups. Includes studies of " - "barriers to and strategies for equitable political inclusion." - ), - "keywords": [ - "political participation", "women in politics", "gender quota", - "citizen participation", "inclusive governance", - "marginalised groups", "representation", - "women's political leadership", "participatory democracy", - "civic education", - ], - }, - }, - }, - # ------------------------------------------------------------------ - # Agenda 2063 - # ------------------------------------------------------------------ - "agenda2063": { - "name": "AU Agenda 2063: The Africa We Want", - "type": "agenda2063", - "year": 2013, - "color": "#047857", # emerald green - "pillars": { - "aspiration_1_prosperity": { - "name": ( - "A Prosperous Africa (Inclusive Growth & Sustainable " - "Development)" - ), - "goals": [1, 2, 3, 4, 5, 6, 7], - "description": ( - "Research on poverty reduction, inclusive growth and " - "sustainable development in Africa. Covers food security and " - "agricultural productivity, public health including maternal " - "health and infectious diseases such as malaria, education " - "quality, nutrition and social protection. Includes studies of " - "renewable energy, climate change adaptation, water security, " - "the blue economy, industrialisation and economic " - "diversification." - ), - "keywords": [ - "poverty reduction", "inclusive growth", - "sustainable development", "food security", - "agricultural productivity", "climate-smart agriculture", - "public health", "maternal health", "infectious diseases", - "malaria", "education quality", "STEM education", - "renewable energy", "climate change adaptation", - "water security", "blue economy", "industrialisation", - "economic diversification", "social protection", "nutrition", - "universal health coverage", - ], - }, - "aspiration_2_integration": { - "name": ( - "An Integrated Continent (Pan-Africanism & African " - "Renaissance)" - ), - "goals": [8, 9, 10], - "description": ( - "Studies of continental and regional integration in Africa, " - "including the African Continental Free Trade Area (AfCFTA), " - "intra-African trade, customs unions and regional value " - "chains. Covers free movement of persons, the African passport, " - "monetary union and cross-border trade. Includes research on " - "infrastructure development, transport corridors and regional " - "connectivity in the spirit of pan-Africanism." - ), - "keywords": [ - "regional integration", "African Continental Free Trade Area", - "AfCFTA", "intra-African trade", "free movement of persons", - "African passport", "monetary union", - "infrastructure development", "transport corridors", - "regional connectivity", "cross-border trade", - "pan-Africanism", "customs union", "regional value chains", - ], - }, - "aspiration_3_governance": { - "name": "Good Governance, Democracy & Human Rights", - "description": ( - "Research on good governance, democracy, justice and human " - "rights across African states. Covers the rule of law, " - "anti-corruption, accountability, transparency and elections. " - "Includes studies of judicial reform, institutional capacity, " - "decentralisation and civic participation in democratic " - "governance." - ), - "goals": [11, 12], - "keywords": [ - "good governance", "democracy", "human rights", "rule of law", - "justice", "anti-corruption", "accountability", "elections", - "judicial reform", "institutional capacity", "transparency", - "decentralisation", "civic participation", - ], - }, - "aspiration_4_peace_security": { - "name": "A Peaceful and Secure Africa", - "goals": [13, 14, 15], - "description": ( - "Research on peace, security and conflict in Africa, including " - "armed conflict, terrorism, violent extremism, insurgency and " - "communal and farmer-herder conflicts. Covers peacebuilding, " - "conflict resolution, mediation, peacekeeping and post-conflict " - "reconstruction. Includes studies of security sector reform, " - "proliferation of small arms and human security." - ), - "keywords": [ - "peacebuilding", "conflict resolution", "armed conflict", - "terrorism", "violent extremism", "insurgency", "peacekeeping", - "security sector reform", "mediation", - "post-conflict reconstruction", "small arms", - "communal conflict", "farmer-herder conflict", - "human security", - ], - }, - "aspiration_5_culture": { - "name": "Strong Cultural Identity & Shared Values", - "goals": [16], - "description": ( - "Scholarship on African cultural identity, heritage and shared " - "values, including African languages, oral tradition and " - "indigenous knowledge. Covers the creative industries, arts and " - "culture, museums, heritage conservation and the cultural " - "economy. Includes studies of the African renaissance, " - "decolonising knowledge and African philosophy." - ), - "keywords": [ - "cultural identity", "cultural heritage", "African languages", - "creative industries", "African renaissance", - "indigenous knowledge", "oral tradition", "African values", - "arts and culture", "heritage conservation", - "cultural economy", "museums", "decolonising knowledge", - "African philosophy", - ], - }, - "aspiration_6_people_driven": { - "name": "People-Driven Development (Women & Youth)", - "goals": [17, 18], - "description": ( - "Research on gender equality, women's empowerment and youth " - "development in Africa. Covers gender-based violence, female " - "genital mutilation, child marriage, girls' education, women in " - "leadership and women in STEM. Includes studies of youth " - "employment and entrepreneurship, child welfare and protection, " - "gender mainstreaming and harnessing the demographic dividend." - ), - "keywords": [ - "gender equality", "women empowerment", - "gender-based violence", "women in leadership", - "girls' education", "youth empowerment", "youth employment", - "child welfare", "child protection", - "female genital mutilation", "child marriage", - "women in STEM", "youth entrepreneurship", - "demographic dividend", "gender mainstreaming", - ], - }, - "aspiration_7_global_player": { - "name": "Africa as a Strong Global Player", - "goals": [19, 20], - "description": ( - "Research on Africa's place in the global economy and global " - "governance, including South-South cooperation and " - "international partnerships. Covers development finance, " - "domestic resource mobilisation, illicit financial flows, debt " - "sustainability and tax reform. Includes studies of foreign " - "direct investment, capital markets, remittances and " - "development assistance." - ), - "keywords": [ - "global governance", "South-South cooperation", - "development finance", "domestic resource mobilisation", - "illicit financial flows", "capital markets", - "foreign direct investment", "debt sustainability", - "international partnerships", "tax reform", "remittances", - "development assistance", - ], - }, - }, - }, - # ------------------------------------------------------------------ - # Regional blocs - # ------------------------------------------------------------------ - "ecowas": { - "name": "ECOWAS Vision 2050 (West Africa)", - "type": "regional_bloc", - "year": 2022, - "color": "#0369a1", # sky blue - "pillars": { - "trade_free_movement": { - "name": "Trade and Free Movement", - "description": ( - "Research on trade integration and free movement in West " - "Africa, including the ECOWAS Trade Liberalisation Scheme, " - "common external tariff, rules of origin and trade " - "facilitation. Covers intra-regional and informal cross-border " - "trade and customs administration. Includes studies of monetary " - "integration, the West African Monetary Zone and the proposed " - "ECO single currency." - ), - "keywords": [ - "ECOWAS Trade Liberalisation Scheme", "intra-regional trade", - "free movement of persons", "common external tariff", - "rules of origin", "informal cross-border trade", - "ECO currency", "monetary integration", - "West African Monetary Zone", "customs", "trade facilitation", - ], - }, - "agriculture_ecowap": { - "name": "Agriculture and Food Security (ECOWAP)", - "description": ( - "Studies of agriculture and food security in West Africa under " - "ECOWAP and CAADP regional agricultural policies. Covers " - "smallholder farmers, agricultural transformation, food " - "sovereignty, agro-pastoralism, irrigation and fertiliser " - "policy. Includes research on key value chains such as rice and " - "cocoa." - ), - "keywords": [ - "ECOWAP", "regional agricultural policy", - "food security West Africa", "CAADP", "rice value chain", - "cocoa", "smallholder farmers", "agricultural transformation", - "food sovereignty", "agro-pastoralism", "irrigation", - "fertiliser policy", - ], - }, - "energy_wapp": { - "name": "Energy and the West African Power Pool", - "description": ( - "Research on energy access and power systems in West Africa, " - "including the West African Power Pool, regional electricity " - "markets and cross-border power interconnection. Covers rural " - "electrification, energy poverty, off-grid solar and renewable " - "energy deployment. Includes studies of the West African Gas " - "Pipeline and regional energy infrastructure." - ), - "keywords": [ - "West African Power Pool", "regional electricity market", - "energy access", "rural electrification", - "renewable energy West Africa", "solar power", - "West African Gas Pipeline", "power interconnection", - "energy poverty", "off-grid solar", - ], - }, - "peace_security_governance": { - "name": "Peace, Security and Governance", - "description": ( - "Research on peace, security and democratic governance in West " - "Africa and the Sahel, including jihadist insurgency, " - "counter-terrorism and recent coups. Covers ECOWAS conflict " - "prevention, ECOMOG interventions, early warning systems and " - "the regional security architecture. Includes studies of " - "democratic governance and political instability in the " - "subregion." - ), - "keywords": [ - "ECOWAS conflict prevention", "ECOMOG", "Sahel security", - "coup West Africa", "jihadist insurgency", - "regional security architecture", "early warning systems", - "democratic governance West Africa", "counter-terrorism", - ], - }, - "health_waho": { - "name": "Regional Health (WAHO)", - "description": ( - "Research on regional health systems and epidemic preparedness " - "in West Africa, coordinated through the West African Health " - "Organisation. Covers disease surveillance and pandemic " - "response, including outbreaks of Ebola and Lassa fever. " - "Includes studies of pharmaceutical regulation and One Health " - "approaches linking human, animal and environmental health." - ), - "keywords": [ - "West African Health Organisation", "epidemic preparedness", - "Ebola", "Lassa fever", "regional health systems", - "disease surveillance", "pandemic response West Africa", - "pharmaceutical regulation", "One Health", - ], - }, - }, - }, - "sadc": { - "name": ( - "SADC Regional Indicative Strategic Development Plan 2020-2030 " - "(Southern Africa)" - ), - "type": "regional_bloc", - "year": 2020, - "color": "#4338ca", # indigo violet - "pillars": { - "industrial_development_market_integration": { - "name": "Industrial Development and Market Integration", - "description": ( - "Research on industrialisation and market integration in " - "Southern Africa, including regional value chains, " - "agro-processing, mineral beneficiation and pharmaceutical " - "manufacturing. Covers the SADC free trade area, macroeconomic " - "convergence, financial integration and manufacturing " - "competitiveness. Includes studies of the green economy, blue " - "economy and tourism development." - ), - "keywords": [ - "SADC industrialisation", "regional value chains", - "agro-processing", "mineral beneficiation", - "pharmaceutical manufacturing", "SADC free trade area", - "macroeconomic convergence", "financial integration", - "green economy", "blue economy", - "manufacturing competitiveness", "tourism development", - ], - }, - "infrastructure": { - "name": "Regional Infrastructure", - "description": ( - "Studies of regional infrastructure in Southern Africa, " - "including the Southern African Power Pool, transport " - "corridors, railways and port infrastructure. Covers energy " - "security, renewable energy, transboundary water resources and " - "ICT connectivity. Includes research on broadband access and " - "infrastructure financing for regional development." - ), - "keywords": [ - "Southern African Power Pool", "transport corridors", - "regional infrastructure", "ICT connectivity", - "transboundary water", "energy security", - "renewable energy Southern Africa", "railway development", - "port infrastructure", "broadband access", - ], - }, - "social_human_capital": { - "name": "Social and Human Capital Development", - "description": ( - "Research on health, education and human capital in Southern " - "Africa, including health systems strengthening, HIV/AIDS and " - "tuberculosis. Covers skills development, qualifications " - "frameworks, education quality, decent work and labour " - "migration. Includes studies of food and nutrition security and " - "social protection programmes." - ), - "keywords": [ - "health systems strengthening", "HIV/AIDS Southern Africa", - "tuberculosis", "skills development", - "qualifications framework", "food and nutrition security", - "decent work", "labour migration", "education quality", - "social protection", - ], - }, - "peace_security_governance": { - "name": "Peace, Security and Governance", - "description": ( - "Research on political stability, peace and governance in " - "Southern Africa, including SADC mediation and electoral " - "observation missions. Covers regional peacekeeping, conflict " - "prevention and the rule of law. Includes studies of democratic " - "governance and political dynamics in SADC member states." - ), - "keywords": [ - "SADC mediation", "electoral observation SADC", - "political stability", "regional peacekeeping", - "governance Southern Africa", "conflict prevention", - "rule of law", - ], - }, - "climate_gender_youth": { - "name": "Climate, Gender and Youth (Cross-cutting)", - "description": ( - "Research on cross-cutting development themes in Southern " - "Africa, including climate change, drought, El Nino effects, " - "cyclones and disaster risk management. Covers environmental " - "management and biodiversity conservation in the region. " - "Includes studies of gender mainstreaming and youth empowerment " - "in regional development." - ), - "keywords": [ - "climate change Southern Africa", "drought", "El Nino", - "disaster risk management", "cyclone", "gender mainstreaming", - "youth empowerment", "environmental management", - "biodiversity", - ], - }, - }, - }, - "eac": { - "name": "EAC Vision 2050 (East Africa)", - "type": "regional_bloc", - "year": 2015, - "color": "#dc2626", # red - "pillars": { - "common_market_integration": { - "name": "Common Market and Integration", - "description": ( - "Research on East African economic and political integration, " - "including the EAC common market, customs union, single customs " - "territory and proposed monetary union and political " - "federation. Covers free movement of labour, non-tariff " - "barriers, harmonisation of standards and cross-border " - "investment. Includes studies of the progress and challenges of " - "regional integration in East Africa." - ), - "keywords": [ - "EAC common market", "customs union", - "East African monetary union", "free movement of labour", - "non-tariff barriers", "single customs territory", - "regional integration East Africa", "political federation", - "harmonisation of standards", "cross-border investment", - ], - }, - "infrastructure": { - "name": "Regional Infrastructure", - "description": ( - "Studies of transport, energy and ICT infrastructure in East " - "Africa, including the Northern and Central Corridors and the " - "standard gauge railway. Covers regional transport networks, " - "energy interconnection, one-stop border posts and ICT " - "infrastructure. Includes research on infrastructure financing " - "and the economics of regional connectivity." - ), - "keywords": [ - "Northern Corridor", "Central Corridor", - "standard gauge railway", "regional transport", - "ICT infrastructure", "energy interconnection", - "infrastructure financing", "one-stop border posts", - ], - }, - "agriculture_food_security": { - "name": "Agriculture and Food Security", - "description": ( - "Research on agriculture, the rural economy and food security " - "in East Africa. Covers smallholder productivity, agricultural " - "trade and key value chains including dairy, coffee, tea and " - "horticulture. Includes studies of food safety challenges such " - "as aflatoxin contamination." - ), - "keywords": [ - "agriculture East Africa", "food security", "rural economy", - "dairy value chain", "coffee", "tea", "horticulture", - "smallholder productivity", "agricultural trade", "aflatoxin", - ], - }, - "industrialisation_trade_services": { - "name": "Industrialisation, Trade and Services", - "description": ( - "Studies of industrialisation, manufacturing and services in " - "East Africa, including SME development, value addition and the " - "textile, leather and pharmaceutical industries. Covers " - "services trade and tourism in the region. Includes research on " - "industrial policy and competitiveness of East African firms." - ), - "keywords": [ - "industrialisation East Africa", "manufacturing", - "SME development", "value addition", "tourism East Africa", - "services trade", "textile industry", - "pharmaceutical production", "leather value chain", - ], - }, - "natural_resources_environment": { - "name": "Natural Resources and Environment", - "description": ( - "Research on natural resources and environmental management in " - "East Africa, including the Lake Victoria basin and " - "transboundary water management. Covers climate change, " - "forestry, wildlife conservation and biodiversity in the " - "region. Includes studies of the blue economy and sustainable " - "use of shared ecosystems." - ), - "keywords": [ - "Lake Victoria basin", "transboundary water management", - "climate change East Africa", "forestry", - "wildlife conservation", "biodiversity", - "environmental management", "blue economy East Africa", - ], - }, - }, - }, -} - -FRAMEWORK_GROUPS = { - "AU Charters": [ - "banjul", "public_service", "cultural_renaissance", - "malabo", "youth", "acdeg", - ], - "Agenda 2063": ["agenda2063"], - "Regional Blocs": ["ecowas", "sadc", "eac"], -} - - -def get_framework(key: str) -> dict | None: - return FRAMEWORKS.get(key) - - -def all_framework_keys() -> list: - return list(FRAMEWORKS) +"""Alignment scoring taxonomy for continental and regional policy frameworks. + +This module defines the canonical taxonomy used by the alignment engine to score +African research output against three families of policy frameworks: + + 1. AU Charters (Banjul, Public Service, Cultural Renaissance, Malabo, Youth, ACDEG) + 2. AU Agenda 2063 (the seven aspirations, with their numbered goals) + 3. Regional bloc strategies (ECOWAS Vision 2050, SADC RISDP 2020-2030, EAC Vision 2050) + +Each framework is decomposed into thematic pillars. A pillar carries two layers: + + - ``description``: 2-3 sentences of prose written in the vocabulary of African + academic paper abstracts. These descriptions are embedded by a Model2Vec + sentence-embedding model and compared with paper abstracts via cosine + similarity to produce the semantic component of the alignment score. + - ``keywords``: a curated term list that forms the auditable evidence layer. + Keyword hits in titles/abstracts are surfaced to users as transparent + justification for why a paper was mapped to a pillar. + +Plain dicts only — this module must stay import-light and serialisable. +""" + +ALIGNMENT_VERSION = 1 + +# Minimum pillar coverage (percent) below which a framework pillar is flagged +# as a research gap for an institution or country. +GAP_THRESHOLD = 25 + +FRAMEWORKS = { + # ------------------------------------------------------------------ + # AU Charters + # ------------------------------------------------------------------ + "banjul": { + "name": "African Charter on Human and Peoples' Rights (Banjul Charter)", + "type": "au_charter", + "year": 1981, + "color": "#1d4ed8", # blue + "pillars": { + "civil_political_rights": { + "name": "Civil and Political Rights", + "description": ( + "Research addressing civil liberties, political participation, due " + "process, freedom of expression and assembly in African states. " + "Includes studies of fair trial rights, arbitrary detention, press " + "freedom, torture prevention and the rule of law. Examines how " + "constitutions, courts and security agencies protect or violate " + "fundamental freedoms." + ), + "keywords": [ + "human rights", "civil liberties", "freedom of expression", + "press freedom", "fair trial", "rule of law", + "arbitrary detention", "torture", "freedom of association", + "political rights", "right to life", "due process", + "fundamental freedoms", + ], + }, + "socioeconomic_cultural_rights": { + "name": "Socio-economic and Cultural Rights", + "description": ( + "Studies of the right to health, education, work, housing and " + "social protection in African countries. Covers access to " + "healthcare and schooling, labour rights, decent working " + "conditions and social justice for poor and marginalised " + "populations. Includes analyses of state obligations to " + "progressively realise socio-economic and cultural rights." + ), + "keywords": [ + "right to health", "right to education", "socio-economic rights", + "social protection", "labour rights", "right to work", + "housing rights", "cultural rights", "social justice", + "access to healthcare", "educational access", + ], + }, + "peoples_collective_rights": { + "name": "Peoples' and Collective Rights", + "description": ( + "Research on collective and peoples' rights, including " + "self-determination, the right to development and sovereignty " + "over natural resources. Covers environmental rights and " + "environmental justice, land rights, community rights and the " + "claims of indigenous peoples. Includes studies of resource " + "governance, extractive industries and communities affected by " + "mining, oil and land dispossession." + ), + "keywords": [ + "self-determination", "right to development", "collective rights", + "environmental rights", "indigenous peoples", + "natural resource sovereignty", "peoples' rights", + "right to peace", "land rights", "community rights", + "environmental justice", "resource governance", + ], + }, + "vulnerable_groups_protection": { + "name": "Protection of Vulnerable Groups", + "description": ( + "Research on the rights and protection of women, children, " + "persons with disabilities, older persons, refugees and " + "internally displaced persons in Africa. Covers gender " + "discrimination, gender-based violence, child labour, human " + "trafficking and minority rights. Includes studies of social " + "inclusion and legal frameworks safeguarding vulnerable and " + "marginalised populations." + ), + "keywords": [ + "women's rights", "child protection", "disability rights", + "rights of older persons", "gender discrimination", + "child labour", "human trafficking", "refugee rights", + "internally displaced persons", "minority rights", + "gender-based violence", "social inclusion", + ], + }, + "african_human_rights_system": { + "name": "African Human Rights System", + "description": ( + "Scholarship on the institutions and procedures of the African " + "human rights system, including the African Commission and the " + "African Court on Human and Peoples' Rights. Examines human " + "rights litigation, jurisprudence, state reporting, treaty " + "compliance and domestication of regional instruments. Includes " + "studies of transitional justice and accountability mechanisms " + "for past violations." + ), + "keywords": [ + "African Commission on Human and Peoples' Rights", + "African Court", "human rights litigation", "treaty compliance", + "state reporting", "regional human rights mechanisms", + "human rights jurisprudence", "domestication of treaties", + "transitional justice", "accountability", + ], + }, + }, + }, + "public_service": { + "name": ( + "African Charter on Values and Principles of Public Service " + "and Administration" + ), + "type": "au_charter", + "year": 2011, + "color": "#0f766e", # teal + "pillars": { + "service_delivery": { + "name": "Public Service Delivery", + "description": ( + "Research on the quality, efficiency and equity of public " + "service delivery in African countries. Covers citizen " + "satisfaction, local government services, public utilities and " + "public sector performance management. Includes studies of " + "service delivery innovation and administrative reforms aimed " + "at improving outcomes for citizens." + ), + "keywords": [ + "public service delivery", "service quality", + "citizen satisfaction", "local government services", + "public sector performance", "service delivery innovation", + "public utilities", "administrative efficiency", + "performance management", + ], + }, + "ethics_anticorruption": { + "name": "Ethics and Anti-corruption", + "description": ( + "Studies of corruption, bribery and integrity in the African " + "public sector. Covers anti-corruption agencies and reforms, " + "procurement fraud, illicit enrichment, conflict of interest, " + "asset declaration and whistleblowing. Includes research on " + "public sector ethics, codes of conduct, accountability and " + "citizens' trust in government." + ), + "keywords": [ + "corruption", "anti-corruption", "public sector ethics", + "accountability", "integrity", "bribery", + "conflict of interest", "asset declaration", "whistleblowing", + "procurement fraud", "illicit enrichment", "code of conduct", + "public trust", + ], + }, + "transparency_access_to_information": { + "name": "Transparency and Access to Information", + "description": ( + "Research on access to information, freedom of information laws " + "and open government in Africa. Covers transparency in public " + "budgets and fiscal management, open data initiatives and " + "disclosure of public records. Includes studies of the right to " + "information as a tool for accountability and citizen oversight." + ), + "keywords": [ + "access to information", "freedom of information", + "open government", "transparency", "open data", + "fiscal transparency", "budget transparency", "public records", + "right to information", "information disclosure", + ], + }, + "egovernment_modernisation": { + "name": "E-government and Administrative Modernisation", + "description": ( + "Research on e-government, digital government and the digital " + "transformation of public administration in Africa. Covers " + "digital public services, digital identity systems, civil " + "service and bureaucratic reform, and administrative " + "modernisation programmes. Includes studies of e-governance " + "adoption, implementation challenges and impacts on service " + "delivery." + ), + "keywords": [ + "e-government", "digital government", "digital public services", + "civil service reform", "public administration reform", + "digital identity", "administrative modernisation", + "bureaucratic reform", "digital transformation of government", + "e-governance", + ], + }, + "civil_service_hrm": { + "name": "Civil Service and Human Resource Management", + "description": ( + "Studies of civil service systems, public sector employment and " + "human resource management in African states. Covers " + "meritocracy, recruitment, training and capacity building of " + "public officials, motivation and working conditions of public " + "servants. Includes research on decentralisation and the " + "staffing and management of local governance structures." + ), + "keywords": [ + "civil service", "public servants", + "human resource management", "public sector employment", + "meritocracy", "capacity building", + "training of public officials", "public sector motivation", + "working conditions", "decentralisation", "local governance", + ], + }, + }, + }, + "cultural_renaissance": { + "name": "Charter for African Cultural Renaissance", + "type": "au_charter", + "year": 2006, + "color": "#b45309", # amber + "pillars": { + "heritage_preservation_restitution": { + "name": "Heritage Preservation and Restitution", + "description": ( + "Research on the preservation, conservation and digitisation of " + "African cultural heritage, including museums, archives, " + "monuments and world heritage sites. Covers archaeology and " + "heritage management as well as the restitution and " + "repatriation of cultural property and looted artefacts. " + "Includes debates on ownership, custodianship and access to " + "African cultural objects." + ), + "keywords": [ + "cultural heritage", "heritage preservation", "museums", + "archives", "restitution of cultural property", + "repatriation of artefacts", "world heritage sites", + "monuments", "archaeology", "heritage conservation", + "digitisation of heritage", "cultural property", + ], + }, + "african_languages": { + "name": "African Languages", + "description": ( + "Linguistic research on African and indigenous languages, " + "including documentation of endangered languages, orthography " + "development, sociolinguistics and translation. Covers language " + "policy, mother tongue education and multilingualism in African " + "classrooms and societies. Includes studies of major languages " + "such as Yoruba, Swahili, Hausa and Igbo." + ), + "keywords": [ + "African languages", "indigenous languages", "language policy", + "mother tongue education", "multilingualism", "linguistics", + "language documentation", "endangered languages", "Yoruba", + "Swahili", "Hausa", "Igbo", "orthography", "sociolinguistics", + "translation", + ], + }, + "creative_cultural_industries": { + "name": "Creative and Cultural Industries", + "description": ( + "Research on Africa's creative economy and cultural industries, " + "including film, music, literature, publishing, performing and " + "visual arts. Covers Nollywood and African cinema, festivals, " + "cultural tourism and cultural entrepreneurship. Includes " + "studies of the economic contribution, value chains and policy " + "environment of the creative sector." + ), + "keywords": [ + "creative industries", "creative economy", "Nollywood", + "African cinema", "African music", "performing arts", + "visual arts", "cultural entrepreneurship", "film industry", + "literature", "publishing", "festivals", "cultural tourism", + ], + }, + "identity_diversity": { + "name": "Cultural Identity and Diversity", + "description": ( + "Scholarship on African and cultural identity, pan-Africanism, " + "afrocentricity and cultural diversity. Covers oral tradition, " + "folklore, ethnicity, religion and cultural values, and " + "intercultural dialogue within and between African societies. " + "Includes studies of decolonisation of culture, thought and " + "representation." + ), + "keywords": [ + "cultural identity", "African identity", "pan-Africanism", + "cultural diversity", "intercultural dialogue", + "decolonisation", "afrocentricity", "oral tradition", + "folklore", "ethnicity", "religion and culture", + "cultural values", + ], + }, + "indigenous_knowledge_systems": { + "name": "Indigenous Knowledge Systems", + "description": ( + "Research on indigenous, traditional and endogenous knowledge " + "systems in Africa, including traditional medicine, " + "ethnobotany, ethnomedicine and traditional ecological " + "knowledge. Covers the documentation, validation and protection " + "of traditional knowledge and cultural expressions. Includes " + "debates on knowledge sovereignty, biopiracy and benefit " + "sharing." + ), + "keywords": [ + "indigenous knowledge", "traditional knowledge", + "endogenous knowledge", "traditional medicine", "ethnobotany", + "ethnomedicine", "indigenous knowledge systems", + "traditional ecological knowledge", "knowledge sovereignty", + "biopiracy", "traditional cultural expressions", + ], + }, + "diaspora_historical_memory": { + "name": "Diaspora and Historical Memory", + "description": ( + "Historical research on the African diaspora, the slave trade, " + "colonialism and their post-colonial legacies. Covers African " + "history, oral history, memory studies and the politics of " + "historical memory and reparations. Includes scholarship on the " + "African renaissance and connections between the continent and " + "its diaspora." + ), + "keywords": [ + "African diaspora", "slave trade", "historical memory", + "colonialism", "post-colonial", "African history", + "oral history", "memory studies", "reparations", + "African renaissance", + ], + }, + }, + }, + "malabo": { + "name": ( + "AU Convention on Cyber Security and Personal Data Protection " + "(Malabo Convention)" + ), + "type": "au_charter", + "year": 2014, + "color": "#6d28d9", # violet + "pillars": { + "electronic_transactions": { + "name": "Electronic Transactions and Digital Economy", + "description": ( + "Research on e-commerce, electronic transactions and the " + "digital economy in Africa. Covers digital payments, mobile " + "money, fintech, digital financial services and online " + "marketplaces. Includes studies of electronic contracts, " + "electronic signatures, payment systems and the regulation of " + "digital trade." + ), + "keywords": [ + "e-commerce", "electronic transactions", "digital payments", + "mobile money", "electronic contracts", "digital trade", + "fintech", "online marketplaces", "electronic signatures", + "digital economy", "payment systems", + "digital financial services", + ], + }, + "data_protection_privacy": { + "name": "Data Protection and Privacy", + "description": ( + "Studies of personal data protection, privacy law and data " + "governance in African countries. Covers data protection " + "authorities, consent, biometric data, cross-border data flows, " + "data sovereignty and data localisation. Includes analyses of " + "compliance with privacy frameworks and protection of " + "information privacy in digital systems." + ), + "keywords": [ + "data protection", "personal data", "privacy", "data privacy", + "data protection authority", "consent", "data governance", + "cross-border data flows", "data sovereignty", + "data localisation", "privacy law", "information privacy", + "biometric data", + ], + }, + "cybersecurity": { + "name": "Cybersecurity", + "description": ( + "Research on cybersecurity, network and information security in " + "African contexts. Covers protection of critical " + "infrastructure, cyber resilience, cyber threats, malware, " + "intrusion detection, encryption and vulnerability assessment. " + "Includes studies of national cybersecurity strategies, " + "capacity and incident response." + ), + "keywords": [ + "cybersecurity", "cyber security", "network security", + "information security", "critical infrastructure protection", + "cyber resilience", "cyber threats", "malware", + "intrusion detection", "encryption", + "vulnerability assessment", + ], + }, + "cybercrime": { + "name": "Cybercrime", + "description": ( + "Research on cybercrime and online criminality in Africa, " + "including computer fraud, phishing, identity theft, hacking " + "and ransomware. Covers cybercrime legislation, digital " + "forensics, electronic evidence and law enforcement responses " + "to internet fraud. Includes criminological studies of online " + "fraud networks and victimisation." + ), + "keywords": [ + "cybercrime", "computer fraud", "phishing", "online fraud", + "identity theft", "ransomware", "digital forensics", + "cybercrime legislation", "electronic evidence", + "internet fraud", "hacking", + ], + }, + "digital_rights_cooperation": { + "name": "Digital Rights and Internet Governance", + "description": ( + "Scholarship on digital rights, internet governance and online " + "freedom of expression in Africa. Covers surveillance, internet " + "shutdowns, platform regulation, misinformation and digital " + "inclusion. Includes studies of internet policy and the balance " + "between security, rights and access in digital spaces." + ), + "keywords": [ + "digital rights", "internet governance", + "online freedom of expression", "surveillance", + "internet shutdowns", "digital inclusion", "internet policy", + "platform regulation", "misinformation", + ], + }, + }, + }, + "youth": { + "name": "African Youth Charter", + "type": "au_charter", + "year": 2006, + "color": "#15803d", # green + "pillars": { + "education_skills": { + "name": "Youth Education and Skills", + "description": ( + "Research on education and skills development for African " + "youth, including technical and vocational education and " + "training (TVET), tertiary education and STEM education. Covers " + "literacy, digital skills, curriculum reform, graduate " + "employability skills and access to education. Includes studies " + "of out-of-school youth and barriers to educational attainment." + ), + "keywords": [ + "youth education", "technical and vocational education", "TVET", + "skills development", "tertiary education", + "out-of-school youth", "digital skills", "literacy", + "STEM education", "curriculum reform", "graduate skills", + "education access", + ], + }, + "employment_entrepreneurship": { + "name": "Youth Employment and Entrepreneurship", + "description": ( + "Studies of youth employment, unemployment and livelihoods in " + "African labour markets. Covers entrepreneurship and youth-led " + "start-ups, self-employment, the informal sector, the gig " + "economy and small and medium enterprises. Includes research on " + "job creation, decent work and policies to absorb young people " + "into productive employment." + ), + "keywords": [ + "youth employment", "youth unemployment", "entrepreneurship", + "youth entrepreneurship", "informal sector", "job creation", + "start-ups", "self-employment", "labour market", "decent work", + "gig economy", "small and medium enterprises", "livelihoods", + ], + }, + "youth_health_wellbeing": { + "name": "Youth Health and Wellbeing", + "description": ( + "Research on adolescent and youth health in Africa, including " + "sexual and reproductive health, HIV/AIDS, teenage pregnancy " + "and adolescent nutrition. Covers substance and drug abuse, " + "mental health and access to youth-friendly health services. " + "Includes studies of risk behaviours and interventions " + "promoting young people's wellbeing." + ), + "keywords": [ + "adolescent health", "youth health", "HIV/AIDS", + "sexual and reproductive health", "teenage pregnancy", + "substance abuse", "drug abuse", "mental health", + "adolescent nutrition", "youth-friendly services", + ], + }, + "participation_civic_engagement": { + "name": "Youth Participation and Civic Engagement", + "description": ( + "Scholarship on youth participation in politics, governance and " + "civic life in Africa. Covers youth policy, youth leadership " + "and representation, student activism, social movements, " + "volunteerism and digital activism. Includes studies of how " + "young people engage with, contest and shape political " + "processes." + ), + "keywords": [ + "youth participation", "civic engagement", "youth in politics", + "youth policy", "student activism", "youth leadership", + "youth representation", "social movements", "volunteerism", + "digital activism", "political participation", + ], + }, + "youth_peace_security": { + "name": "Youth, Peace and Security", + "description": ( + "Research on young people in conflict and peacebuilding in " + "Africa, including youth radicalisation, violent extremism, " + "gangs, cultism and youth violence. Covers the roles of youth " + "in post-conflict recovery and the youth, peace and security " + "agenda. Includes studies of drivers of youth recruitment into " + "armed groups and programmes for reintegration." + ), + "keywords": [ + "youth and conflict", "youth radicalisation", + "violent extremism", "youth in peacebuilding", "gangs", + "cultism", "youth violence", "post-conflict youth", + "youth peace and security", + ], + }, + }, + }, + "acdeg": { + "name": "African Charter on Democracy, Elections and Governance (ACDEG)", + "type": "au_charter", + "year": 2007, + "color": "#b91c1c", # red + "pillars": { + "democracy_rule_of_law": { + "name": "Democracy and Rule of Law", + "description": ( + "Research on democracy, democratisation and constitutionalism " + "in African states. Covers the rule of law, separation of " + "powers, judicial independence, constitutional reform, " + "political parties and civil society. Includes studies of " + "democratic consolidation, democratic backsliding, " + "authoritarianism and presidential term limits." + ), + "keywords": [ + "democracy", "democratisation", "constitutionalism", + "rule of law", "separation of powers", "judicial independence", + "constitutional reform", "civil society", "political parties", + "democratic consolidation", "authoritarianism", "term limits", + "democratic backsliding", + ], + }, + "elections": { + "name": "Elections and Electoral Integrity", + "description": ( + "Studies of elections and electoral processes in Africa, " + "including electoral commissions, voter registration, biometric " + "voter technologies and election observation. Covers electoral " + "violence, electoral reform, election petitions, campaign " + "finance and voter turnout. Includes research on the conduct " + "and integrity of free and fair elections." + ), + "keywords": [ + "elections", "electoral commission", "election observation", + "voter registration", "electoral violence", + "free and fair elections", "electoral reform", "voter turnout", + "election petition", "biometric voter", "electoral integrity", + "campaign finance", + ], + }, + "unconstitutional_change": { + "name": "Unconstitutional Changes of Government", + "description": ( + "Research on coups d'etat, military rule and unconstitutional " + "changes of government in Africa. Covers civil-military " + "relations, juntas, political transitions, power transfers and " + "third-term bids that subvert constitutional order. Includes " + "studies of regional and continental responses to military " + "takeovers." + ), + "keywords": [ + "coup d'etat", "military coup", + "unconstitutional change of government", "military rule", + "civil-military relations", "junta", "political transition", + "power transfer", "third-term bids", + ], + }, + "governance_institutions": { + "name": "Governance and Public Institutions", + "description": ( + "Scholarship on good governance and the performance of public " + "institutions in Africa. Covers anti-corruption, " + "accountability, transparency, the African Peer Review " + "Mechanism, decentralisation and local government. Includes " + "studies of state capacity, institutional reform and political " + "leadership." + ), + "keywords": [ + "good governance", "governance", "anti-corruption", + "accountability", "African Peer Review Mechanism", + "public institutions", "decentralisation", "local government", + "state capacity", "institutional reform", "transparency", + "leadership", + ], + }, + "participation_gender_equity": { + "name": "Participation, Gender and Equity", + "description": ( + "Research on political participation and inclusive governance " + "in Africa, with emphasis on women in politics, gender quotas " + "and women's political leadership. Covers citizen " + "participation, participatory democracy, civic education and " + "the representation of marginalised groups. Includes studies of " + "barriers to and strategies for equitable political inclusion." + ), + "keywords": [ + "political participation", "women in politics", "gender quota", + "citizen participation", "inclusive governance", + "marginalised groups", "representation", + "women's political leadership", "participatory democracy", + "civic education", + ], + }, + }, + }, + # ------------------------------------------------------------------ + # Agenda 2063 + # ------------------------------------------------------------------ + "agenda2063": { + "name": "AU Agenda 2063: The Africa We Want", + "type": "agenda2063", + "year": 2013, + "color": "#047857", # emerald green + "pillars": { + "aspiration_1_prosperity": { + "name": ( + "A Prosperous Africa (Inclusive Growth & Sustainable " + "Development)" + ), + "goals": [1, 2, 3, 4, 5, 6, 7], + "description": ( + "Research on poverty reduction, inclusive growth and " + "sustainable development in Africa. Covers food security and " + "agricultural productivity, public health including maternal " + "health and infectious diseases such as malaria, education " + "quality, nutrition and social protection. Includes studies of " + "renewable energy, climate change adaptation, water security, " + "the blue economy, industrialisation and economic " + "diversification." + ), + "keywords": [ + "poverty reduction", "inclusive growth", + "sustainable development", "food security", + "agricultural productivity", "climate-smart agriculture", + "public health", "maternal health", "infectious diseases", + "malaria", "education quality", "STEM education", + "renewable energy", "climate change adaptation", + "water security", "blue economy", "industrialisation", + "economic diversification", "social protection", "nutrition", + "universal health coverage", + ], + }, + "aspiration_2_integration": { + "name": ( + "An Integrated Continent (Pan-Africanism & African " + "Renaissance)" + ), + "goals": [8, 9, 10], + "description": ( + "Studies of continental and regional integration in Africa, " + "including the African Continental Free Trade Area (AfCFTA), " + "intra-African trade, customs unions and regional value " + "chains. Covers free movement of persons, the African passport, " + "monetary union and cross-border trade. Includes research on " + "infrastructure development, transport corridors and regional " + "connectivity in the spirit of pan-Africanism." + ), + "keywords": [ + "regional integration", "African Continental Free Trade Area", + "AfCFTA", "intra-African trade", "free movement of persons", + "African passport", "monetary union", + "infrastructure development", "transport corridors", + "regional connectivity", "cross-border trade", + "pan-Africanism", "customs union", "regional value chains", + ], + }, + "aspiration_3_governance": { + "name": "Good Governance, Democracy & Human Rights", + "description": ( + "Research on good governance, democracy, justice and human " + "rights across African states. Covers the rule of law, " + "anti-corruption, accountability, transparency and elections. " + "Includes studies of judicial reform, institutional capacity, " + "decentralisation and civic participation in democratic " + "governance." + ), + "goals": [11, 12], + "keywords": [ + "good governance", "democracy", "human rights", "rule of law", + "justice", "anti-corruption", "accountability", "elections", + "judicial reform", "institutional capacity", "transparency", + "decentralisation", "civic participation", + ], + }, + "aspiration_4_peace_security": { + "name": "A Peaceful and Secure Africa", + "goals": [13, 14, 15], + "description": ( + "Research on peace, security and conflict in Africa, including " + "armed conflict, terrorism, violent extremism, insurgency and " + "communal and farmer-herder conflicts. Covers peacebuilding, " + "conflict resolution, mediation, peacekeeping and post-conflict " + "reconstruction. Includes studies of security sector reform, " + "proliferation of small arms and human security." + ), + "keywords": [ + "peacebuilding", "conflict resolution", "armed conflict", + "terrorism", "violent extremism", "insurgency", "peacekeeping", + "security sector reform", "mediation", + "post-conflict reconstruction", "small arms", + "communal conflict", "farmer-herder conflict", + "human security", + ], + }, + "aspiration_5_culture": { + "name": "Strong Cultural Identity & Shared Values", + "goals": [16], + "description": ( + "Scholarship on African cultural identity, heritage and shared " + "values, including African languages, oral tradition and " + "indigenous knowledge. Covers the creative industries, arts and " + "culture, museums, heritage conservation and the cultural " + "economy. Includes studies of the African renaissance, " + "decolonising knowledge and African philosophy." + ), + "keywords": [ + "cultural identity", "cultural heritage", "African languages", + "creative industries", "African renaissance", + "indigenous knowledge", "oral tradition", "African values", + "arts and culture", "heritage conservation", + "cultural economy", "museums", "decolonising knowledge", + "African philosophy", + ], + }, + "aspiration_6_people_driven": { + "name": "People-Driven Development (Women & Youth)", + "goals": [17, 18], + "description": ( + "Research on gender equality, women's empowerment and youth " + "development in Africa. Covers gender-based violence, female " + "genital mutilation, child marriage, girls' education, women in " + "leadership and women in STEM. Includes studies of youth " + "employment and entrepreneurship, child welfare and protection, " + "gender mainstreaming and harnessing the demographic dividend." + ), + "keywords": [ + "gender equality", "women empowerment", + "gender-based violence", "women in leadership", + "girls' education", "youth empowerment", "youth employment", + "child welfare", "child protection", + "female genital mutilation", "child marriage", + "women in STEM", "youth entrepreneurship", + "demographic dividend", "gender mainstreaming", + ], + }, + "aspiration_7_global_player": { + "name": "Africa as a Strong Global Player", + "goals": [19, 20], + "description": ( + "Research on Africa's place in the global economy and global " + "governance, including South-South cooperation and " + "international partnerships. Covers development finance, " + "domestic resource mobilisation, illicit financial flows, debt " + "sustainability and tax reform. Includes studies of foreign " + "direct investment, capital markets, remittances and " + "development assistance." + ), + "keywords": [ + "global governance", "South-South cooperation", + "development finance", "domestic resource mobilisation", + "illicit financial flows", "capital markets", + "foreign direct investment", "debt sustainability", + "international partnerships", "tax reform", "remittances", + "development assistance", + ], + }, + }, + }, + # ------------------------------------------------------------------ + # Regional blocs + # ------------------------------------------------------------------ + "ecowas": { + "name": "ECOWAS Vision 2050 (West Africa)", + "type": "regional_bloc", + "year": 2022, + "color": "#0369a1", # sky blue + "pillars": { + "trade_free_movement": { + "name": "Trade and Free Movement", + "description": ( + "Research on trade integration and free movement in West " + "Africa, including the ECOWAS Trade Liberalisation Scheme, " + "common external tariff, rules of origin and trade " + "facilitation. Covers intra-regional and informal cross-border " + "trade and customs administration. Includes studies of monetary " + "integration, the West African Monetary Zone and the proposed " + "ECO single currency." + ), + "keywords": [ + "ECOWAS Trade Liberalisation Scheme", "intra-regional trade", + "free movement of persons", "common external tariff", + "rules of origin", "informal cross-border trade", + "ECO currency", "monetary integration", + "West African Monetary Zone", "customs", "trade facilitation", + ], + }, + "agriculture_ecowap": { + "name": "Agriculture and Food Security (ECOWAP)", + "description": ( + "Studies of agriculture and food security in West Africa under " + "ECOWAP and CAADP regional agricultural policies. Covers " + "smallholder farmers, agricultural transformation, food " + "sovereignty, agro-pastoralism, irrigation and fertiliser " + "policy. Includes research on key value chains such as rice and " + "cocoa." + ), + "keywords": [ + "ECOWAP", "regional agricultural policy", + "food security West Africa", "CAADP", "rice value chain", + "cocoa", "smallholder farmers", "agricultural transformation", + "food sovereignty", "agro-pastoralism", "irrigation", + "fertiliser policy", + ], + }, + "energy_wapp": { + "name": "Energy and the West African Power Pool", + "description": ( + "Research on energy access and power systems in West Africa, " + "including the West African Power Pool, regional electricity " + "markets and cross-border power interconnection. Covers rural " + "electrification, energy poverty, off-grid solar and renewable " + "energy deployment. Includes studies of the West African Gas " + "Pipeline and regional energy infrastructure." + ), + "keywords": [ + "West African Power Pool", "regional electricity market", + "energy access", "rural electrification", + "renewable energy West Africa", "solar power", + "West African Gas Pipeline", "power interconnection", + "energy poverty", "off-grid solar", + ], + }, + "peace_security_governance": { + "name": "Peace, Security and Governance", + "description": ( + "Research on peace, security and democratic governance in West " + "Africa and the Sahel, including jihadist insurgency, " + "counter-terrorism and recent coups. Covers ECOWAS conflict " + "prevention, ECOMOG interventions, early warning systems and " + "the regional security architecture. Includes studies of " + "democratic governance and political instability in the " + "subregion." + ), + "keywords": [ + "ECOWAS conflict prevention", "ECOMOG", "Sahel security", + "coup West Africa", "jihadist insurgency", + "regional security architecture", "early warning systems", + "democratic governance West Africa", "counter-terrorism", + ], + }, + "health_waho": { + "name": "Regional Health (WAHO)", + "description": ( + "Research on regional health systems and epidemic preparedness " + "in West Africa, coordinated through the West African Health " + "Organisation. Covers disease surveillance and pandemic " + "response, including outbreaks of Ebola and Lassa fever. " + "Includes studies of pharmaceutical regulation and One Health " + "approaches linking human, animal and environmental health." + ), + "keywords": [ + "West African Health Organisation", "epidemic preparedness", + "Ebola", "Lassa fever", "regional health systems", + "disease surveillance", "pandemic response West Africa", + "pharmaceutical regulation", "One Health", + ], + }, + }, + }, + "sadc": { + "name": ( + "SADC Regional Indicative Strategic Development Plan 2020-2030 " + "(Southern Africa)" + ), + "type": "regional_bloc", + "year": 2020, + "color": "#4338ca", # indigo violet + "pillars": { + "industrial_development_market_integration": { + "name": "Industrial Development and Market Integration", + "description": ( + "Research on industrialisation and market integration in " + "Southern Africa, including regional value chains, " + "agro-processing, mineral beneficiation and pharmaceutical " + "manufacturing. Covers the SADC free trade area, macroeconomic " + "convergence, financial integration and manufacturing " + "competitiveness. Includes studies of the green economy, blue " + "economy and tourism development." + ), + "keywords": [ + "SADC industrialisation", "regional value chains", + "agro-processing", "mineral beneficiation", + "pharmaceutical manufacturing", "SADC free trade area", + "macroeconomic convergence", "financial integration", + "green economy", "blue economy", + "manufacturing competitiveness", "tourism development", + ], + }, + "infrastructure": { + "name": "Regional Infrastructure", + "description": ( + "Studies of regional infrastructure in Southern Africa, " + "including the Southern African Power Pool, transport " + "corridors, railways and port infrastructure. Covers energy " + "security, renewable energy, transboundary water resources and " + "ICT connectivity. Includes research on broadband access and " + "infrastructure financing for regional development." + ), + "keywords": [ + "Southern African Power Pool", "transport corridors", + "regional infrastructure", "ICT connectivity", + "transboundary water", "energy security", + "renewable energy Southern Africa", "railway development", + "port infrastructure", "broadband access", + ], + }, + "social_human_capital": { + "name": "Social and Human Capital Development", + "description": ( + "Research on health, education and human capital in Southern " + "Africa, including health systems strengthening, HIV/AIDS and " + "tuberculosis. Covers skills development, qualifications " + "frameworks, education quality, decent work and labour " + "migration. Includes studies of food and nutrition security and " + "social protection programmes." + ), + "keywords": [ + "health systems strengthening", "HIV/AIDS Southern Africa", + "tuberculosis", "skills development", + "qualifications framework", "food and nutrition security", + "decent work", "labour migration", "education quality", + "social protection", + ], + }, + "peace_security_governance": { + "name": "Peace, Security and Governance", + "description": ( + "Research on political stability, peace and governance in " + "Southern Africa, including SADC mediation and electoral " + "observation missions. Covers regional peacekeeping, conflict " + "prevention and the rule of law. Includes studies of democratic " + "governance and political dynamics in SADC member states." + ), + "keywords": [ + "SADC mediation", "electoral observation SADC", + "political stability", "regional peacekeeping", + "governance Southern Africa", "conflict prevention", + "rule of law", + ], + }, + "climate_gender_youth": { + "name": "Climate, Gender and Youth (Cross-cutting)", + "description": ( + "Research on cross-cutting development themes in Southern " + "Africa, including climate change, drought, El Nino effects, " + "cyclones and disaster risk management. Covers environmental " + "management and biodiversity conservation in the region. " + "Includes studies of gender mainstreaming and youth empowerment " + "in regional development." + ), + "keywords": [ + "climate change Southern Africa", "drought", "El Nino", + "disaster risk management", "cyclone", "gender mainstreaming", + "youth empowerment", "environmental management", + "biodiversity", + ], + }, + }, + }, + "eac": { + "name": "EAC Vision 2050 (East Africa)", + "type": "regional_bloc", + "year": 2015, + "color": "#dc2626", # red + "pillars": { + "common_market_integration": { + "name": "Common Market and Integration", + "description": ( + "Research on East African economic and political integration, " + "including the EAC common market, customs union, single customs " + "territory and proposed monetary union and political " + "federation. Covers free movement of labour, non-tariff " + "barriers, harmonisation of standards and cross-border " + "investment. Includes studies of the progress and challenges of " + "regional integration in East Africa." + ), + "keywords": [ + "EAC common market", "customs union", + "East African monetary union", "free movement of labour", + "non-tariff barriers", "single customs territory", + "regional integration East Africa", "political federation", + "harmonisation of standards", "cross-border investment", + ], + }, + "infrastructure": { + "name": "Regional Infrastructure", + "description": ( + "Studies of transport, energy and ICT infrastructure in East " + "Africa, including the Northern and Central Corridors and the " + "standard gauge railway. Covers regional transport networks, " + "energy interconnection, one-stop border posts and ICT " + "infrastructure. Includes research on infrastructure financing " + "and the economics of regional connectivity." + ), + "keywords": [ + "Northern Corridor", "Central Corridor", + "standard gauge railway", "regional transport", + "ICT infrastructure", "energy interconnection", + "infrastructure financing", "one-stop border posts", + ], + }, + "agriculture_food_security": { + "name": "Agriculture and Food Security", + "description": ( + "Research on agriculture, the rural economy and food security " + "in East Africa. Covers smallholder productivity, agricultural " + "trade and key value chains including dairy, coffee, tea and " + "horticulture. Includes studies of food safety challenges such " + "as aflatoxin contamination." + ), + "keywords": [ + "agriculture East Africa", "food security", "rural economy", + "dairy value chain", "coffee", "tea", "horticulture", + "smallholder productivity", "agricultural trade", "aflatoxin", + ], + }, + "industrialisation_trade_services": { + "name": "Industrialisation, Trade and Services", + "description": ( + "Studies of industrialisation, manufacturing and services in " + "East Africa, including SME development, value addition and the " + "textile, leather and pharmaceutical industries. Covers " + "services trade and tourism in the region. Includes research on " + "industrial policy and competitiveness of East African firms." + ), + "keywords": [ + "industrialisation East Africa", "manufacturing", + "SME development", "value addition", "tourism East Africa", + "services trade", "textile industry", + "pharmaceutical production", "leather value chain", + ], + }, + "natural_resources_environment": { + "name": "Natural Resources and Environment", + "description": ( + "Research on natural resources and environmental management in " + "East Africa, including the Lake Victoria basin and " + "transboundary water management. Covers climate change, " + "forestry, wildlife conservation and biodiversity in the " + "region. Includes studies of the blue economy and sustainable " + "use of shared ecosystems." + ), + "keywords": [ + "Lake Victoria basin", "transboundary water management", + "climate change East Africa", "forestry", + "wildlife conservation", "biodiversity", + "environmental management", "blue economy East Africa", + ], + }, + }, + }, +} + +FRAMEWORK_GROUPS = { + "AU Charters": [ + "banjul", "public_service", "cultural_renaissance", + "malabo", "youth", "acdeg", + ], + "Agenda 2063": ["agenda2063"], + "Regional Blocs": ["ecowas", "sadc", "eac"], +} + + +def get_framework(key: str) -> dict | None: + return FRAMEWORKS.get(key) + + +def all_framework_keys() -> list: + return list(FRAMEWORKS) diff --git a/uraas/config/institutions.py b/uraas/config/institutions.py index eb481afc9531806e7138eaa9e986561a81fee899..3ecc319171e8fc8ed78251241a6ad697d74dbe07 100644 --- a/uraas/config/institutions.py +++ b/uraas/config/institutions.py @@ -1,299 +1,299 @@ -""" -URAAS Institution Configuration System -Manages multi-institution support with ROR identifiers -Supports rich staff format: [{name, orcid, department, faculty}] -""" - -import json -import os -from pathlib import Path -from typing import Any, Dict, List, Optional - - -class InstitutionConfig: - """Configuration for a single institution""" - - def __init__( - self, - ror, - name, - short_name, - country, - staff_file, - affiliation_patterns, - faculties=None, - crawler_settings=None, - sub_region="Unknown", - oai_endpoint=None, - ): - self.ror = ror - self.name = name - self.short_name = short_name - self.country = country - self.staff_file = staff_file - self.affiliation_patterns = affiliation_patterns - self.faculties = faculties or [] - self.crawler_settings = crawler_settings or {} - self.sub_region = sub_region - # Optional public OAI-PMH base URL for the institution's repository - # (read-only harvest). None means no OAI harvest is configured. - self.oai_endpoint = oai_endpoint or None - # Optional OAI-PMH set spec for filtering at source (e.g. DSpace community - # handle "com_1234_5"). When set, the OAI spider passes &set=... so only - # records from that collection are returned, reducing network overhead. - self.oai_set = (crawler_settings or {}).get("oai_set") or None - self._raw_staff: List[Any] = self._load_staff_raw() - - def _resolve_staff_file(self) -> str: - if os.path.isabs(self.staff_file): - return self.staff_file - base_dir = Path(__file__).parent.parent.parent - candidate = base_dir / self.staff_file - if candidate.exists(): - return str(candidate) - if os.path.exists(self.staff_file): - return self.staff_file - return str(candidate) - - def _load_staff_raw(self) -> List[Any]: - path = self._resolve_staff_file() - if not os.path.exists(path): - return [] - try: - with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - if isinstance(data, list): - return data - if isinstance(data, dict): - if "staff" in data: - return data["staff"] - if "names" in data: - return data["names"] - names = [] - for value in data.values(): - if isinstance(value, list): - names.extend(value) - return names - return [] - except Exception as e: - print(f"Warning: Error loading staff file {path}: {e}") - return [] - - @property - def staff_names(self) -> List[str]: - """Flat list of staff names (backwards compatible).""" - names = [] - for entry in self._raw_staff: - if isinstance(entry, str): - names.append(entry) - elif isinstance(entry, dict): - n = entry.get("name") or entry.get("display_name", "") - if n: - names.append(n) - return names - - @property - def staff_records(self) -> List[Dict]: - """Rich staff records: [{name, orcid, department, faculty}].""" - records = [] - for entry in self._raw_staff: - if isinstance(entry, str): - records.append( - {"name": entry, "orcid": None, "department": None, "faculty": None} - ) - elif isinstance(entry, dict): - records.append( - { - "name": entry.get("name") or entry.get("display_name", ""), - "orcid": entry.get("orcid"), - "department": entry.get("department"), - "faculty": entry.get("faculty"), - "openalex_id": entry.get("openalex_id"), - "paper_count": entry.get("paper_count", 0), - } - ) - return [r for r in records if r["name"]] - - @property - def staff_with_orcid(self) -> List[Dict]: - return [r for r in self.staff_records if r.get("orcid")] - - @property - def departments(self) -> List[str]: - depts = set() - for r in self.staff_records: - if r.get("department"): - depts.add(r["department"]) - return sorted(depts) - - def matches_affiliation(self, affiliation_text: str) -> bool: - if not affiliation_text: - return False - affiliation_lower = affiliation_text.lower() - return any(p.lower() in affiliation_lower for p in self.affiliation_patterns) - - def verify_ror_in_authorships(self, authorships: List[Dict]) -> bool: - """ - Verify at least one author has this institution's ROR. - Critical gate for 98% precision crawling. - """ - if not authorships: - return False - target_short = self.ror.split("/")[-1] - for authorship in authorships: - for inst in authorship.get("institutions", []): - inst_ror = inst.get("ror", "") or inst.get("id", "") or "" - if target_short in inst_ror or self.ror == inst_ror: - return True - return False - - def to_dict(self) -> Dict: - return { - "ror": self.ror, - "name": self.name, - "short_name": self.short_name, - "country": self.country, - "staff_file": self.staff_file, - "affiliation_patterns": self.affiliation_patterns, - "faculties": self.faculties, - "crawler_settings": self.crawler_settings, - "sub_region": self.sub_region, - "oai_endpoint": self.oai_endpoint, - "staff_count": len(self.staff_names), - "staff_with_orcid_count": len(self.staff_with_orcid), - } - - @classmethod - def from_dict(cls, data: Dict) -> "InstitutionConfig": - sub_region = data.get("sub_region") - if not sub_region: - country = data.get("country", "") - if country in ("Nigeria", "Ghana"): - sub_region = "West Africa" - elif country in ( - "South Africa", - "Zimbabwe", - "Zambia", - "Namibia", - "Botswana", - "Lesotho", - "Eswatini", - "Malawi", - "Mozambique", - ): - sub_region = "Southern Africa" - elif country in ("Kenya", "Uganda", "Ethiopia", "Rwanda", "Tanzania"): - sub_region = "East Africa" - elif country in ( - "Egypt", - "Morocco", - "Tunisia", - "Algeria", - "Libya", - "Sudan", - ): - sub_region = "North Africa" - elif country in ( - "Cameroon", - "DR Congo", - "Angola", - "Gabon", - "Republic of the Congo", - ): - sub_region = "Central Africa" - else: - sub_region = "Unknown" - return cls( - ror=data["ror"], - name=data["name"], - short_name=data["short_name"], - country=data["country"], - staff_file=data["staff_file"], - affiliation_patterns=data["affiliation_patterns"], - faculties=data.get("faculties", []), - crawler_settings=data.get("crawler_settings", {}), - sub_region=sub_region, - oai_endpoint=data.get("oai_endpoint"), - ) - - @classmethod - def from_json_file(cls, file_path: str) -> "InstitutionConfig": - with open(file_path, "r", encoding="utf-8") as f: - data = json.load(f) - return cls.from_dict(data) - - -class InstitutionRegistry: - """Registry of all configured institutions""" - - def __init__(self, config_dir: str = None): - if config_dir is None: - base_dir = Path(__file__).parent.parent.parent - config_dir = base_dir / "config" / "institutions" - self.config_dir = Path(config_dir) - self.institutions: Dict[str, InstitutionConfig] = {} - self._load_all_institutions() - - def _load_all_institutions(self): - if not self.config_dir.exists(): - print(f"Warning: Institution config directory not found: {self.config_dir}") - return - for json_file in sorted(self.config_dir.glob("*.json")): - try: - config = InstitutionConfig.from_json_file(str(json_file)) - self.institutions[config.short_name.lower()] = config - except Exception as e: - print(f"Error loading {json_file}: {e}") - - def get(self, identifier: str) -> Optional[InstitutionConfig]: - identifier_lower = identifier.lower() - if identifier_lower in self.institutions: - return self.institutions[identifier_lower] - for config in self.institutions.values(): - if ( - config.ror == identifier - or identifier_lower == config.ror.split("/")[-1] - ): - return config - return None - - def get_by_ror(self, ror: str) -> Optional[InstitutionConfig]: - for config in self.institutions.values(): - if config.ror == ror: - return config - return None - - def list_all(self) -> List[InstitutionConfig]: - return list(self.institutions.values()) - - def list_by_country(self, country: str) -> List[InstitutionConfig]: - return [ - c - for c in self.institutions.values() - if c.country.lower() == country.lower() - ] - - def add_institution(self, config: InstitutionConfig): - self.institutions[config.short_name.lower()] = config - - def save_institution(self, config: InstitutionConfig): - self.config_dir.mkdir(parents=True, exist_ok=True) - file_path = self.config_dir / f"{config.short_name.lower()}.json" - with open(file_path, "w", encoding="utf-8") as f: - json.dump(config.to_dict(), f, indent=2, ensure_ascii=False) - self.add_institution(config) - - -_registry = None - - -def get_registry() -> InstitutionRegistry: - global _registry - if _registry is None: - _registry = InstitutionRegistry() - return _registry - - -def reset_registry(): - global _registry - _registry = None +""" +URAAS Institution Configuration System +Manages multi-institution support with ROR identifiers +Supports rich staff format: [{name, orcid, department, faculty}] +""" + +import json +import os +from pathlib import Path +from typing import Any, Dict, List, Optional + + +class InstitutionConfig: + """Configuration for a single institution""" + + def __init__( + self, + ror, + name, + short_name, + country, + staff_file, + affiliation_patterns, + faculties=None, + crawler_settings=None, + sub_region="Unknown", + oai_endpoint=None, + ): + self.ror = ror + self.name = name + self.short_name = short_name + self.country = country + self.staff_file = staff_file + self.affiliation_patterns = affiliation_patterns + self.faculties = faculties or [] + self.crawler_settings = crawler_settings or {} + self.sub_region = sub_region + # Optional public OAI-PMH base URL for the institution's repository + # (read-only harvest). None means no OAI harvest is configured. + self.oai_endpoint = oai_endpoint or None + # Optional OAI-PMH set spec for filtering at source (e.g. DSpace community + # handle "com_1234_5"). When set, the OAI spider passes &set=... so only + # records from that collection are returned, reducing network overhead. + self.oai_set = (crawler_settings or {}).get("oai_set") or None + self._raw_staff: List[Any] = self._load_staff_raw() + + def _resolve_staff_file(self) -> str: + if os.path.isabs(self.staff_file): + return self.staff_file + base_dir = Path(__file__).parent.parent.parent + candidate = base_dir / self.staff_file + if candidate.exists(): + return str(candidate) + if os.path.exists(self.staff_file): + return self.staff_file + return str(candidate) + + def _load_staff_raw(self) -> List[Any]: + path = self._resolve_staff_file() + if not os.path.exists(path): + return [] + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, list): + return data + if isinstance(data, dict): + if "staff" in data: + return data["staff"] + if "names" in data: + return data["names"] + names = [] + for value in data.values(): + if isinstance(value, list): + names.extend(value) + return names + return [] + except Exception as e: + print(f"Warning: Error loading staff file {path}: {e}") + return [] + + @property + def staff_names(self) -> List[str]: + """Flat list of staff names (backwards compatible).""" + names = [] + for entry in self._raw_staff: + if isinstance(entry, str): + names.append(entry) + elif isinstance(entry, dict): + n = entry.get("name") or entry.get("display_name", "") + if n: + names.append(n) + return names + + @property + def staff_records(self) -> List[Dict]: + """Rich staff records: [{name, orcid, department, faculty}].""" + records = [] + for entry in self._raw_staff: + if isinstance(entry, str): + records.append( + {"name": entry, "orcid": None, "department": None, "faculty": None} + ) + elif isinstance(entry, dict): + records.append( + { + "name": entry.get("name") or entry.get("display_name", ""), + "orcid": entry.get("orcid"), + "department": entry.get("department"), + "faculty": entry.get("faculty"), + "openalex_id": entry.get("openalex_id"), + "paper_count": entry.get("paper_count", 0), + } + ) + return [r for r in records if r["name"]] + + @property + def staff_with_orcid(self) -> List[Dict]: + return [r for r in self.staff_records if r.get("orcid")] + + @property + def departments(self) -> List[str]: + depts = set() + for r in self.staff_records: + if r.get("department"): + depts.add(r["department"]) + return sorted(depts) + + def matches_affiliation(self, affiliation_text: str) -> bool: + if not affiliation_text: + return False + affiliation_lower = affiliation_text.lower() + return any(p.lower() in affiliation_lower for p in self.affiliation_patterns) + + def verify_ror_in_authorships(self, authorships: List[Dict]) -> bool: + """ + Verify at least one author has this institution's ROR. + Critical gate for 98% precision crawling. + """ + if not authorships: + return False + target_short = self.ror.split("/")[-1] + for authorship in authorships: + for inst in authorship.get("institutions", []): + inst_ror = inst.get("ror", "") or inst.get("id", "") or "" + if target_short in inst_ror or self.ror == inst_ror: + return True + return False + + def to_dict(self) -> Dict: + return { + "ror": self.ror, + "name": self.name, + "short_name": self.short_name, + "country": self.country, + "staff_file": self.staff_file, + "affiliation_patterns": self.affiliation_patterns, + "faculties": self.faculties, + "crawler_settings": self.crawler_settings, + "sub_region": self.sub_region, + "oai_endpoint": self.oai_endpoint, + "staff_count": len(self.staff_names), + "staff_with_orcid_count": len(self.staff_with_orcid), + } + + @classmethod + def from_dict(cls, data: Dict) -> "InstitutionConfig": + sub_region = data.get("sub_region") + if not sub_region: + country = data.get("country", "") + if country in ("Nigeria", "Ghana"): + sub_region = "West Africa" + elif country in ( + "South Africa", + "Zimbabwe", + "Zambia", + "Namibia", + "Botswana", + "Lesotho", + "Eswatini", + "Malawi", + "Mozambique", + ): + sub_region = "Southern Africa" + elif country in ("Kenya", "Uganda", "Ethiopia", "Rwanda", "Tanzania"): + sub_region = "East Africa" + elif country in ( + "Egypt", + "Morocco", + "Tunisia", + "Algeria", + "Libya", + "Sudan", + ): + sub_region = "North Africa" + elif country in ( + "Cameroon", + "DR Congo", + "Angola", + "Gabon", + "Republic of the Congo", + ): + sub_region = "Central Africa" + else: + sub_region = "Unknown" + return cls( + ror=data["ror"], + name=data["name"], + short_name=data["short_name"], + country=data["country"], + staff_file=data["staff_file"], + affiliation_patterns=data["affiliation_patterns"], + faculties=data.get("faculties", []), + crawler_settings=data.get("crawler_settings", {}), + sub_region=sub_region, + oai_endpoint=data.get("oai_endpoint"), + ) + + @classmethod + def from_json_file(cls, file_path: str) -> "InstitutionConfig": + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + return cls.from_dict(data) + + +class InstitutionRegistry: + """Registry of all configured institutions""" + + def __init__(self, config_dir: str = None): + if config_dir is None: + base_dir = Path(__file__).parent.parent.parent + config_dir = base_dir / "config" / "institutions" + self.config_dir = Path(config_dir) + self.institutions: Dict[str, InstitutionConfig] = {} + self._load_all_institutions() + + def _load_all_institutions(self): + if not self.config_dir.exists(): + print(f"Warning: Institution config directory not found: {self.config_dir}") + return + for json_file in sorted(self.config_dir.glob("*.json")): + try: + config = InstitutionConfig.from_json_file(str(json_file)) + self.institutions[config.short_name.lower()] = config + except Exception as e: + print(f"Error loading {json_file}: {e}") + + def get(self, identifier: str) -> Optional[InstitutionConfig]: + identifier_lower = identifier.lower() + if identifier_lower in self.institutions: + return self.institutions[identifier_lower] + for config in self.institutions.values(): + if ( + config.ror == identifier + or identifier_lower == config.ror.split("/")[-1] + ): + return config + return None + + def get_by_ror(self, ror: str) -> Optional[InstitutionConfig]: + for config in self.institutions.values(): + if config.ror == ror: + return config + return None + + def list_all(self) -> List[InstitutionConfig]: + return list(self.institutions.values()) + + def list_by_country(self, country: str) -> List[InstitutionConfig]: + return [ + c + for c in self.institutions.values() + if c.country.lower() == country.lower() + ] + + def add_institution(self, config: InstitutionConfig): + self.institutions[config.short_name.lower()] = config + + def save_institution(self, config: InstitutionConfig): + self.config_dir.mkdir(parents=True, exist_ok=True) + file_path = self.config_dir / f"{config.short_name.lower()}.json" + with open(file_path, "w", encoding="utf-8") as f: + json.dump(config.to_dict(), f, indent=2, ensure_ascii=False) + self.add_institution(config) + + +_registry = None + + +def get_registry() -> InstitutionRegistry: + global _registry + if _registry is None: + _registry = InstitutionRegistry() + return _registry + + +def reset_registry(): + global _registry + _registry = None diff --git a/uraas/config/language_research.py b/uraas/config/language_research.py index e5a3d801bfe3966c47862bc173586c103482fa6d..195b43da2071685212a336bea5021f56d5a117a3 100644 --- a/uraas/config/language_research.py +++ b/uraas/config/language_research.py @@ -1,66 +1,66 @@ -""" -Language & Culture research detector — shared regex blueprints (Phase 7 cleanup). - -Extracted from uraas/dashboard/app.py language_research() route to: - 1. Allow unit-testing the classifier in isolation. - 2. Eliminate the per-request re-compile of large regex patterns. - 3. Serve as the single source of truth for keyword lists. -""" -import re - -# Tier-1: strong African-language / humanities signals (each match = 2 pts) -LANG_TIER1 = re.compile( - r"\b(yoruba|igbo|hausa|pidgin|efik|tiv|fulani|ibibio|ijaw|kanuri" - r"|sociolinguistics|lexicography|phonology|phonetics|morphosyntax" - r"|oral tradition|oral literature|oral poetry|oral narrative|proverbs" - r"|folklore|folktale|griot|african literature|nigerian literature" - r"|postcolonial literature|literary criticism|literary theory|narratology" - r"|language policy|multilingualism|bilingualism|code.switching" - r"|indigenous language|vernacular|dialect continuum|pragmatics" - r"|discourse analysis|stylistics|nollywood|yoruba drama|african theatre)\b", - re.IGNORECASE, -) - -# Tier-2: broad humanities signals (each match = 1 pt) -LANG_TIER2 = re.compile( - r"\b(morphology|syntax|semantics|translation|literary|language|linguistic" - r"|dialect|narrative|discourse|rhetoric|poetry|prose|fiction|novel|drama" - r"|theatre|culture|cultural identity|cultural heritage|african studies|humanities)\b", - re.IGNORECASE, -) - -# Exclusion: STEM / clinical topics that accidentally hit tier-2 keywords -LANG_EXCLUDE = re.compile( - r"\b(machine learning|deep learning|neural network|artificial intelligence" - r"|clinical trial|randomized|patient|hospital|surgery|cancer|tumor" - r"|cardiovascular|hypertension|diabetes|preeclampsia|concrete|cement" - r"|compressive strength|tensile|alloy|composite|carbon emission" - r"|ecological footprint|gdp|economic growth|galaxy|astrophysic|ionosphere" - r"|plasma|quantum|semiconductor|mpox|covid|sars|influenza|malaria|hiv" - r"|antibiotic|cybersecurity|blockchain|iot|cloud computing|petroleum" - r"|crude oil|refinery|corrosion|mentoring|capacity building|faculty development)\b", - re.IGNORECASE, -) - -# Minimum relevance threshold: 2 combined tier-1/2 points, with ≥1 tier-1 hit -# or 3+ tier-2 hits. -LANG_MIN_SCORE = 2 - - -def score_item(title: str, abstract: str) -> tuple[int, list[str]]: - """Return (score, matched_terms) for a title+abstract pair. - - score == 0 means the item should be excluded. - """ - text = (f"{title} {abstract}").lower() - if LANG_EXCLUDE.search(text): - return 0, [] - t1 = LANG_TIER1.findall(text) - t2 = LANG_TIER2.findall(text) - score = len(t1) * 2 + len(t2) - if score < LANG_MIN_SCORE: - return 0, [] - if not t1 and len(t2) < 3: - return 0, [] - matched = list(dict.fromkeys(t1 + t2)) # deduplicate, preserve order - return score, matched +""" +Language & Culture research detector — shared regex blueprints (Phase 7 cleanup). + +Extracted from uraas/dashboard/app.py language_research() route to: + 1. Allow unit-testing the classifier in isolation. + 2. Eliminate the per-request re-compile of large regex patterns. + 3. Serve as the single source of truth for keyword lists. +""" +import re + +# Tier-1: strong African-language / humanities signals (each match = 2 pts) +LANG_TIER1 = re.compile( + r"\b(yoruba|igbo|hausa|pidgin|efik|tiv|fulani|ibibio|ijaw|kanuri" + r"|sociolinguistics|lexicography|phonology|phonetics|morphosyntax" + r"|oral tradition|oral literature|oral poetry|oral narrative|proverbs" + r"|folklore|folktale|griot|african literature|nigerian literature" + r"|postcolonial literature|literary criticism|literary theory|narratology" + r"|language policy|multilingualism|bilingualism|code.switching" + r"|indigenous language|vernacular|dialect continuum|pragmatics" + r"|discourse analysis|stylistics|nollywood|yoruba drama|african theatre)\b", + re.IGNORECASE, +) + +# Tier-2: broad humanities signals (each match = 1 pt) +LANG_TIER2 = re.compile( + r"\b(morphology|syntax|semantics|translation|literary|language|linguistic" + r"|dialect|narrative|discourse|rhetoric|poetry|prose|fiction|novel|drama" + r"|theatre|culture|cultural identity|cultural heritage|african studies|humanities)\b", + re.IGNORECASE, +) + +# Exclusion: STEM / clinical topics that accidentally hit tier-2 keywords +LANG_EXCLUDE = re.compile( + r"\b(machine learning|deep learning|neural network|artificial intelligence" + r"|clinical trial|randomized|patient|hospital|surgery|cancer|tumor" + r"|cardiovascular|hypertension|diabetes|preeclampsia|concrete|cement" + r"|compressive strength|tensile|alloy|composite|carbon emission" + r"|ecological footprint|gdp|economic growth|galaxy|astrophysic|ionosphere" + r"|plasma|quantum|semiconductor|mpox|covid|sars|influenza|malaria|hiv" + r"|antibiotic|cybersecurity|blockchain|iot|cloud computing|petroleum" + r"|crude oil|refinery|corrosion|mentoring|capacity building|faculty development)\b", + re.IGNORECASE, +) + +# Minimum relevance threshold: 2 combined tier-1/2 points, with ≥1 tier-1 hit +# or 3+ tier-2 hits. +LANG_MIN_SCORE = 2 + + +def score_item(title: str, abstract: str) -> tuple[int, list[str]]: + """Return (score, matched_terms) for a title+abstract pair. + + score == 0 means the item should be excluded. + """ + text = (f"{title} {abstract}").lower() + if LANG_EXCLUDE.search(text): + return 0, [] + t1 = LANG_TIER1.findall(text) + t2 = LANG_TIER2.findall(text) + score = len(t1) * 2 + len(t2) + if score < LANG_MIN_SCORE: + return 0, [] + if not t1 and len(t2) < 3: + return 0, [] + matched = list(dict.fromkeys(t1 + t2)) # deduplicate, preserve order + return score, matched diff --git a/uraas/config/methodology.py b/uraas/config/methodology.py index 2a5d6bd6e851b9e4a4f1b3b8cb3883c99d722f77..63674a0aadda45349ff89c5be18c9bea262404ca 100644 --- a/uraas/config/methodology.py +++ b/uraas/config/methodology.py @@ -1,188 +1,188 @@ -""" -Per-metric methodology definitions — the "how is this computed" layer. - -Served whole via GET /api/methodology (frontend caches it for the ⓘ tooltips) -and attached per-endpoint through responses.api_ok(methodology_key=...). - -Every number on the dashboard must be auditable: formula, data source, -counting method, known limitations. This is the product's differentiator -versus closed bibliometric platforms (Barcelona Declaration on Open Research -Information, 2024; Leiden Ranking Open Edition). -""" - -METHODOLOGY = { - "alignment_score": { - "title": "Framework Alignment Score", - "formula": ( - "0.6 × semantic similarity (Model2Vec potion-base-8M cosine between the " - "paper's title+abstract and the pillar description) + 0.4 × keyword " - "evidence (matched pillar keywords, saturating at 4 hits). Scaled 0–100." - ), - "scale": "0–100 per pillar; institutional profile = mean over papers scoring ≥5.", - "source": "Open metadata (OpenAlex, CC0); URAAS alignment engine v1.", - "caveats": ( - "Falls back to keyword-only scoring when the embedding model is " - "unavailable (marked in the API response). Matched keywords are shown " - "as evidence so every score can be audited." - ), - }, - "alignment_gap": { - "title": "Research Gap Flag", - "formula": ( - "A framework pillar is flagged as a gap when its institutional average " - "alignment score falls below the gap threshold (default 25/100)." - ), - "scale": "Binary flag per pillar, ranked by ascending score.", - "source": "Derived from Framework Alignment Scores.", - "caveats": "A gap signals low measured output, which may also reflect metadata coverage.", - }, - "intra_african_collaboration": { - "title": "Intra-African Collaboration Index", - "formula": ( - "% of works whose author affiliations span ≥2 distinct African countries " - "(full counting at the work level — the Scimago/Leiden international-" - "collaboration indicator restricted to AU member states)." - ), - "scale": "0–100%.", - "source": "OpenAlex authorship affiliations (institutions[].country_code).", - "benchmark": { - "value": 8.4, - "label": "Continental average (Research Policy, 2022)", - "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC9746314/", - }, - "caveats": ( - "Works lacking affiliation metadata are excluded from the denominator; " - "coverage share is reported alongside the index." - ), - }, - "country_pair_matrix": { - "title": "Country-Pair Collaboration Matrix", - "formula": ( - "For each work, every unordered pair of distinct African co-author " - "countries increments that pair's count by 1 (full counting)." - ), - "scale": "Raw co-publication counts per country pair.", - "source": "OpenAlex authorship affiliations.", - "caveats": "Counts works, not authors; a work with 3 countries contributes 3 pairs.", - }, - "citation_velocity": { - "title": "Citation Velocity", - "formula": ( - "Citations accrued per calendar year from OpenAlex counts_by_year; " - "early velocity = mean citations in the first 2 years after publication." - ), - "scale": "Citations/year.", - "source": "OpenAlex counts_by_year (last 10 years, inline field).", - "caveats": "OpenAlex citation coverage lags recent months; counts are full, not field-normalized.", - }, - "pan_african_citation_share": { - "title": "Pan-African Citation Share", - "formula": ( - "% of a work's citing works that have ≥1 author affiliated with an " - "African institution (OpenAlex group_by country of citing works)." - ), - "scale": "0–100% per work; institutional figure is citation-weighted mean.", - "source": "OpenAlex cites: filter grouped by authorships.institutions.country_code.", - "caveats": "Computed for the most-cited works first; citing works without affiliations are excluded.", - }, - "tk_vitality": { - "title": "TK Vitality Score", - "formula": ( - "Weighted share of indigenous-knowledge content types: " - "indigenous_knowledge=3.0, cultural_heritage=2.5, oral_tradition=2.5, " - "grey_literature=1.5, thesis/dataset=1.2, patent=1.0, research_paper=0.5; " - "normalised by total items × 3.0, scaled to 100." - ), - "scale": "0–100.", - "source": "URAAS content-type classification.", - "caveats": "Depends on content_type assignment quality at ingest.", - }, - "linguistic_diversity": { - "title": "Linguistic Diversity Index", - "formula": "% of repository output published in African languages (23 ISO 639-1 codes tracked).", - "scale": "0–100%.", - "source": "Item language metadata (dc_language / language_code).", - "caveats": "Language metadata is sparse in upstream sources; treat as a lower bound.", - }, - "sc_score": { - "title": "Special Collections Score", - "formula": ( - "Multi-gate keyword decision engine: strong-category matches × 3 + " - "support matches × 1, with ambiguous-token guards and STEM exclusion. " - "Score > 0 = genuine Special Collections item." - ), - "scale": "0 = not SC; higher = stronger signal.", - "source": "URAAS SC decision engine (uraas/services/sc_engine.py — open source).", - "caveats": "All dashboard analytics are gated to SC items (score > 0) by design.", - }, - "ark_identifier": { - "title": "ARK Persistent Identifier", - "formula": ( - "ark:// minted deterministically from the " - "item's DocID hash, betanumeric alphabet, NCDA check character." - ), - "scale": "—", - "source": "ARK Alliance specification (arks.org); Africa PID Alliance × ARK Alliance partnership (2025).", - "caveats": "Test NAAN (99999) until the production NAAN registration completes; ARKs are free to mint.", - }, - "sc_thematic_composition": { - "title": "Thematic Composition", - "formula": ( - "Count of Special Collections items tagged with each of the 8 SC " - "themes (Indigenous Knowledge, African Literature, Cultural Heritage, " - "Ethnic Languages & Groups, Postcolonial Studies, Pan-African Studies, " - "African Philosophy, Ethnomusicology). Items are multi-themed, so " - "shares sum to more than 100%." - ), - "scale": "Item counts per theme.", - "source": "URAAS SC decision engine categories (special_collection_categories).", - "caveats": "An item tagged with N themes contributes to all N counts.", - }, - "sc_cooccurrence": { - "title": "Theme Co-occurrence", - "formula": ( - "For each SC item, every unordered pair of its distinct themes " - "increments that pair's link weight by 1 (full counting); single-theme " - "items contribute to the diagonal. Visualised as a chord diagram so " - "interdisciplinary overlaps (e.g. Indigenous Knowledge ↔ Cultural " - "Heritage) are legible — a view commercial bibliometric tools omit." - ), - "scale": "Co-occurrence counts per theme pair.", - "source": "Derived from special_collection_categories.", - "caveats": "Reflects classifier theme assignment; sparse themes show few links.", - }, - "sc_knowledge_sovereignty": { - "title": "Knowledge Sovereignty", - "formula": ( - "Distribution of contributing African countries across SC works " - "(from co-author affiliations), plus the share of SC works spanning " - "≥2 African countries. A custodianship lens that centres African " - "authorship rather than North-export citation prestige." - ), - "scale": "Country item counts; intra-African share 0–100%.", - "source": "OpenAlex authorship affiliations (coauthor_countries, is_intra_african).", - "caveats": "Works without affiliation metadata are excluded from country/intra-African figures.", - }, - "sc_sdg_alignment": { - "title": "SDG & Development Alignment", - "formula": ( - "Count of SC works tagged with each UN Sustainable Development Goal " - "(works may carry multiple SDG tags), connecting cultural and " - "indigenous-knowledge scholarship to development relevance." - ), - "scale": "Item counts per SDG (1–17).", - "source": "URAAS SDG tagging (sdg_tags).", - "caveats": "Multi-tagged works count toward each of their SDGs; tagging coverage is partial.", - }, - "sc_influential_works": { - "title": "Most Influential Works", - "formula": ( - "SC works ranked by total citations received (OpenAlex cited_by_count). " - "Used as a reach signal alongside the sovereignty and SDG lenses, not " - "as the sole measure of value." - ), - "scale": "Total citations per work.", - "source": "OpenAlex cited_by_count.", - "caveats": "Citation coverage lags recent works; counts are full, not field-normalized.", - }, -} +""" +Per-metric methodology definitions — the "how is this computed" layer. + +Served whole via GET /api/methodology (frontend caches it for the ⓘ tooltips) +and attached per-endpoint through responses.api_ok(methodology_key=...). + +Every number on the dashboard must be auditable: formula, data source, +counting method, known limitations. This is the product's differentiator +versus closed bibliometric platforms (Barcelona Declaration on Open Research +Information, 2024; Leiden Ranking Open Edition). +""" + +METHODOLOGY = { + "alignment_score": { + "title": "Framework Alignment Score", + "formula": ( + "0.6 × semantic similarity (Model2Vec potion-base-8M cosine between the " + "paper's title+abstract and the pillar description) + 0.4 × keyword " + "evidence (matched pillar keywords, saturating at 4 hits). Scaled 0–100." + ), + "scale": "0–100 per pillar; institutional profile = mean over papers scoring ≥5.", + "source": "Open metadata (OpenAlex, CC0); URAAS alignment engine v1.", + "caveats": ( + "Falls back to keyword-only scoring when the embedding model is " + "unavailable (marked in the API response). Matched keywords are shown " + "as evidence so every score can be audited." + ), + }, + "alignment_gap": { + "title": "Research Gap Flag", + "formula": ( + "A framework pillar is flagged as a gap when its institutional average " + "alignment score falls below the gap threshold (default 25/100)." + ), + "scale": "Binary flag per pillar, ranked by ascending score.", + "source": "Derived from Framework Alignment Scores.", + "caveats": "A gap signals low measured output, which may also reflect metadata coverage.", + }, + "intra_african_collaboration": { + "title": "Intra-African Collaboration Index", + "formula": ( + "% of works whose author affiliations span ≥2 distinct African countries " + "(full counting at the work level — the Scimago/Leiden international-" + "collaboration indicator restricted to AU member states)." + ), + "scale": "0–100%.", + "source": "OpenAlex authorship affiliations (institutions[].country_code).", + "benchmark": { + "value": 8.4, + "label": "Continental average (Research Policy, 2022)", + "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC9746314/", + }, + "caveats": ( + "Works lacking affiliation metadata are excluded from the denominator; " + "coverage share is reported alongside the index." + ), + }, + "country_pair_matrix": { + "title": "Country-Pair Collaboration Matrix", + "formula": ( + "For each work, every unordered pair of distinct African co-author " + "countries increments that pair's count by 1 (full counting)." + ), + "scale": "Raw co-publication counts per country pair.", + "source": "OpenAlex authorship affiliations.", + "caveats": "Counts works, not authors; a work with 3 countries contributes 3 pairs.", + }, + "citation_velocity": { + "title": "Citation Velocity", + "formula": ( + "Citations accrued per calendar year from OpenAlex counts_by_year; " + "early velocity = mean citations in the first 2 years after publication." + ), + "scale": "Citations/year.", + "source": "OpenAlex counts_by_year (last 10 years, inline field).", + "caveats": "OpenAlex citation coverage lags recent months; counts are full, not field-normalized.", + }, + "pan_african_citation_share": { + "title": "Pan-African Citation Share", + "formula": ( + "% of a work's citing works that have ≥1 author affiliated with an " + "African institution (OpenAlex group_by country of citing works)." + ), + "scale": "0–100% per work; institutional figure is citation-weighted mean.", + "source": "OpenAlex cites: filter grouped by authorships.institutions.country_code.", + "caveats": "Computed for the most-cited works first; citing works without affiliations are excluded.", + }, + "tk_vitality": { + "title": "TK Vitality Score", + "formula": ( + "Weighted share of indigenous-knowledge content types: " + "indigenous_knowledge=3.0, cultural_heritage=2.5, oral_tradition=2.5, " + "grey_literature=1.5, thesis/dataset=1.2, patent=1.0, research_paper=0.5; " + "normalised by total items × 3.0, scaled to 100." + ), + "scale": "0–100.", + "source": "URAAS content-type classification.", + "caveats": "Depends on content_type assignment quality at ingest.", + }, + "linguistic_diversity": { + "title": "Linguistic Diversity Index", + "formula": "% of repository output published in African languages (23 ISO 639-1 codes tracked).", + "scale": "0–100%.", + "source": "Item language metadata (dc_language / language_code).", + "caveats": "Language metadata is sparse in upstream sources; treat as a lower bound.", + }, + "sc_score": { + "title": "Special Collections Score", + "formula": ( + "Multi-gate keyword decision engine: strong-category matches × 3 + " + "support matches × 1, with ambiguous-token guards and STEM exclusion. " + "Score > 0 = genuine Special Collections item." + ), + "scale": "0 = not SC; higher = stronger signal.", + "source": "URAAS SC decision engine (uraas/services/sc_engine.py — open source).", + "caveats": "All dashboard analytics are gated to SC items (score > 0) by design.", + }, + "ark_identifier": { + "title": "ARK Persistent Identifier", + "formula": ( + "ark:// minted deterministically from the " + "item's DocID hash, betanumeric alphabet, NCDA check character." + ), + "scale": "—", + "source": "ARK Alliance specification (arks.org); Africa PID Alliance × ARK Alliance partnership (2025).", + "caveats": "Test NAAN (99999) until the production NAAN registration completes; ARKs are free to mint.", + }, + "sc_thematic_composition": { + "title": "Thematic Composition", + "formula": ( + "Count of Special Collections items tagged with each of the 8 SC " + "themes (Indigenous Knowledge, African Literature, Cultural Heritage, " + "Ethnic Languages & Groups, Postcolonial Studies, Pan-African Studies, " + "African Philosophy, Ethnomusicology). Items are multi-themed, so " + "shares sum to more than 100%." + ), + "scale": "Item counts per theme.", + "source": "URAAS SC decision engine categories (special_collection_categories).", + "caveats": "An item tagged with N themes contributes to all N counts.", + }, + "sc_cooccurrence": { + "title": "Theme Co-occurrence", + "formula": ( + "For each SC item, every unordered pair of its distinct themes " + "increments that pair's link weight by 1 (full counting); single-theme " + "items contribute to the diagonal. Visualised as a chord diagram so " + "interdisciplinary overlaps (e.g. Indigenous Knowledge ↔ Cultural " + "Heritage) are legible — a view commercial bibliometric tools omit." + ), + "scale": "Co-occurrence counts per theme pair.", + "source": "Derived from special_collection_categories.", + "caveats": "Reflects classifier theme assignment; sparse themes show few links.", + }, + "sc_knowledge_sovereignty": { + "title": "Knowledge Sovereignty", + "formula": ( + "Distribution of contributing African countries across SC works " + "(from co-author affiliations), plus the share of SC works spanning " + "≥2 African countries. A custodianship lens that centres African " + "authorship rather than North-export citation prestige." + ), + "scale": "Country item counts; intra-African share 0–100%.", + "source": "OpenAlex authorship affiliations (coauthor_countries, is_intra_african).", + "caveats": "Works without affiliation metadata are excluded from country/intra-African figures.", + }, + "sc_sdg_alignment": { + "title": "SDG & Development Alignment", + "formula": ( + "Count of SC works tagged with each UN Sustainable Development Goal " + "(works may carry multiple SDG tags), connecting cultural and " + "indigenous-knowledge scholarship to development relevance." + ), + "scale": "Item counts per SDG (1–17).", + "source": "URAAS SDG tagging (sdg_tags).", + "caveats": "Multi-tagged works count toward each of their SDGs; tagging coverage is partial.", + }, + "sc_influential_works": { + "title": "Most Influential Works", + "formula": ( + "SC works ranked by total citations received (OpenAlex cited_by_count). " + "Used as a reach signal alongside the sovereignty and SDG lenses, not " + "as the sole measure of value." + ), + "scale": "Total citations per work.", + "source": "OpenAlex cited_by_count.", + "caveats": "Citation coverage lags recent works; counts are full, not field-normalized.", + }, +} diff --git a/uraas/config/special_collections.py b/uraas/config/special_collections.py index d134b8cb462a077b5355c49cb2a34e39358684ec..ebe2195458a97242be9962d23b036c05397c0228 100644 --- a/uraas/config/special_collections.py +++ b/uraas/config/special_collections.py @@ -1,67 +1,67 @@ -""" -Centralized Special Collections seed keywords used by spiders + pipeline. - -The exhaustive 133-term taxonomy lives in `uraas.utils.ai_classifier.SPECIAL_COLLECTIONS` -(that's the ground truth used for classification/scoring). The constants here are the -*crawler seed terms*: the short, high-signal phrases we inject into upstream search APIs -(OpenAlex / Crossref / arXiv) to oversample SC-relevant papers. Keeping them small keeps -URLs short and avoids hammering the APIs with 100+ OR clauses. -""" - -from typing import List - -from uraas.utils.ai_classifier import SPECIAL_COLLECTIONS - -# Hand-picked seeds per category: the 2-3 most distinctive, low-ambiguity phrases. -# These get used directly in API search params, so they must be unambiguous enough -# that a vanilla full-text search returns relevant hits. -SC_SEED_KEYWORDS: List[str] = [ - # Indigenous Knowledge - "indigenous knowledge", - "traditional knowledge", - "ethnobotany", - "traditional ecological knowledge", - # African Literature - "postcolonial literature", - "african literature", - "oral literature", - # Cultural Heritage - "cultural heritage", - "intangible heritage", - "oral history", - # Postcolonial Studies - "postcolonialism", - "decolonization", - "decolonial", - # Pan-African Studies - "pan-africanism", - "african renaissance", - # African Philosophy - "ubuntu philosophy", - "african philosophy", - # Ethnomusicology - "ethnomusicology", - "african music", - "traditional music", -] - -# OpenAlex concept search terms — these match against OpenAlex's concept taxonomy -# (broader than free-text). Used in concepts.display_name.search filter. -SC_OPENALEX_CONCEPTS: List[str] = [ - "indigenous", - "postcolonial", - "ethnography", - "cultural heritage", - "decolonization", - "ubuntu", - "ethnomusicology", - "oral tradition", -] - - -def all_classifier_keywords() -> List[str]: - """Full 133-term list used by the in-pipeline classifier for scoring.""" - out = [] - for kws in SPECIAL_COLLECTIONS.values(): - out.extend(kws) - return out +""" +Centralized Special Collections seed keywords used by spiders + pipeline. + +The exhaustive 133-term taxonomy lives in `uraas.utils.ai_classifier.SPECIAL_COLLECTIONS` +(that's the ground truth used for classification/scoring). The constants here are the +*crawler seed terms*: the short, high-signal phrases we inject into upstream search APIs +(OpenAlex / Crossref / arXiv) to oversample SC-relevant papers. Keeping them small keeps +URLs short and avoids hammering the APIs with 100+ OR clauses. +""" + +from typing import List + +from uraas.utils.ai_classifier import SPECIAL_COLLECTIONS + +# Hand-picked seeds per category: the 2-3 most distinctive, low-ambiguity phrases. +# These get used directly in API search params, so they must be unambiguous enough +# that a vanilla full-text search returns relevant hits. +SC_SEED_KEYWORDS: List[str] = [ + # Indigenous Knowledge + "indigenous knowledge", + "traditional knowledge", + "ethnobotany", + "traditional ecological knowledge", + # African Literature + "postcolonial literature", + "african literature", + "oral literature", + # Cultural Heritage + "cultural heritage", + "intangible heritage", + "oral history", + # Postcolonial Studies + "postcolonialism", + "decolonization", + "decolonial", + # Pan-African Studies + "pan-africanism", + "african renaissance", + # African Philosophy + "ubuntu philosophy", + "african philosophy", + # Ethnomusicology + "ethnomusicology", + "african music", + "traditional music", +] + +# OpenAlex concept search terms — these match against OpenAlex's concept taxonomy +# (broader than free-text). Used in concepts.display_name.search filter. +SC_OPENALEX_CONCEPTS: List[str] = [ + "indigenous", + "postcolonial", + "ethnography", + "cultural heritage", + "decolonization", + "ubuntu", + "ethnomusicology", + "oral tradition", +] + + +def all_classifier_keywords() -> List[str]: + """Full 133-term list used by the in-pipeline classifier for scoring.""" + out = [] + for kws in SPECIAL_COLLECTIONS.values(): + out.extend(kws) + return out diff --git a/uraas/dashboard/app.py b/uraas/dashboard/app.py index 2d6f989d3fe2c24b46a9404cce538a888b38d8f6..cc5faad7e20a54d745d4cdff0f9fa3ef56b1f29e 100644 --- a/uraas/dashboard/app.py +++ b/uraas/dashboard/app.py @@ -1,3035 +1,3035 @@ -import csv -import io -import json -import logging -import os -import re -import subprocess -import threading - -from flask import ( - Flask, - Response, - jsonify, - redirect, - render_template, - request, - send_file, - session, - url_for, -) -from flask_socketio import SocketIO -from sqlalchemy import desc, extract, func, or_ -from sqlalchemy.orm import selectinload - -from uraas.analytics.engine import analytics -from uraas.config import config -from uraas.database import ( - Author, - Collection, - Community, - File, - Item, - SessionLocal, - db_year, - db_year_month, -) -from uraas.dashboard.auth import ( - ADMIN, - check_credentials, - clamped_int, - current_role, -) -from flask_limiter import Limiter -from flask_limiter.util import get_remote_address -from uraas.dashboard.responses import api_error, api_ok, csv_response -from uraas.production_config import ProductionConfig -from uraas.utils.analytics_cache import analytics_cache - -# Fail fast on insecure production config before the app even binds. -config.validate() - -app = Flask(__name__) -app.config["SECRET_KEY"] = config.DASHBOARD_SECRET_KEY -# Session-cookie hardening applies in every environment (SECURE only where TLS -# is present, i.e. production) so it is not Render-specific. -app.config.update( - SESSION_COOKIE_HTTPONLY=True, - SESSION_COOKIE_SAMESITE="Lax", - SESSION_COOKIE_SECURE=config.is_production(), -) -# Apply production hardening at import time so it also runs under gunicorn -# (the __main__ block below never executes in a WSGI deployment). -ProductionConfig.apply_config(app) - -# SocketIO: restrict the handshake to an explicit origin allowlist (no wildcard). -socketio = SocketIO( - app, - cors_allowed_origins=config.cors_origins(), - async_mode="threading", -) -logger = logging.getLogger(__name__) -crawler_process = None -crawler_lock = threading.Lock() - -# Rate limiter — in-memory storage (no Redis dep). Limits login to 10 attempts -# per minute to prevent brute-force attacks on admin/viewer credentials. -limiter = Limiter( - key_func=get_remote_address, - app=app, - default_limits=[], # No global limit; only login is rate-limited. - storage_uri="memory://", -) - -# ── Access control (fail-closed) ─────────────────────────────────────────── -# Everything is gated by default. Endpoints are authorised by *endpoint name* -# (function name) so path params don't matter and any NEW route is protected -# until explicitly listed here. -# -# PUBLIC_ENDPOINTS — reachable without a session (login page, health, static). -# ADMIN_ENDPOINTS — require role == admin (crawler, mutations, bulk exports, -# staff directory PII). Everything else needs any login. -PUBLIC_ENDPOINTS = {"login", "logout", "health_check", "api_version", "static"} -ADMIN_ENDPOINTS = { - "start_crawler", - "stop_crawler", - "crawler_status", - "flush_analytics_cache", - "prune_non_sc", - "test_smtp", - "recompute_alignment", - "update_citations", - "bulk_update_citations", - "export_csv", - "export_bibtex", - "export_special_collections_csv", - "alignment_export_csv", - "collaboration_export_csv", - "citations_velocity_csv", - "staff_directory", -} - - -@app.before_request -def _enforce_authentication(): - endpoint = request.endpoint - # Unknown endpoint (404s) and explicit public routes pass through. - if endpoint is None or endpoint in PUBLIC_ENDPOINTS: - return None - role = current_role() - if not role: - if request.path.startswith("/api/"): - return jsonify({"status": "error", "message": "Authentication required"}), 401 - return redirect(url_for("login", next=request.path)) - if endpoint in ADMIN_ENDPOINTS and role != ADMIN: - return jsonify({"status": "error", "message": "Administrator access required"}), 403 - return None - - -def crawler_monitor(process): - global crawler_process - try: - for line in iter(process.stdout.readline, b""): - with crawler_lock: - if crawler_process is None or crawler_process != process: - break - line_decoded = line.decode("utf-8", errors="replace").strip() - if not line_decoded: - continue - if line_decoded.startswith("[INIT]"): - socketio.emit( - "crawl_status", {"status": "initializing", "message": line_decoded} - ) - socketio.emit("terminal_output", {"line": line_decoded}) - elif "URAAS_DOWNLOAD:" in line_decoded: - socketio.emit("crawl_status", {"status": "running"}) - try: - title = line_decoded.split("URAAS_DOWNLOAD:", 1)[-1].strip() - socketio.emit("crawl_progress", {"title": title}) - except Exception: - pass - socketio.emit("terminal_output", {"line": line_decoded}) - else: - socketio.emit("terminal_output", {"line": line_decoded}) - except Exception: - pass - finally: - try: - process.stdout.close() - except Exception: - pass - process.wait() - with crawler_lock: - if crawler_process == process: - crawler_process = None - analytics_cache.invalidate_all() # Flush stale analytics after crawl - socketio.emit("crawl_status", {"status": "stopped"}) - - -@app.context_processor -def inject_asset_version(): - """Cache-bust static assets by their file mtime so browsers always load - the latest JS/CSS after a deploy or edit (no more stale cached app.js).""" - - def asset_version(filename): - try: - path = os.path.join(app.static_folder, filename) - return str(int(os.path.getmtime(path))) - except OSError: - return "1" - - return {"asset_version": asset_version} - - -@app.route("/") -def index(): - return render_template("index.html") - - -@app.route("/login", methods=["GET", "POST"]) -@limiter.limit("10 per minute", methods=["POST"], error_message="Too many login attempts — wait 60 seconds.") -def login(): - if request.method == "POST": - username = (request.form.get("username") or "").strip() - password = request.form.get("password") or "" - role = check_credentials(username, password) - if role: - session.clear() - session["role"] = role - session["user"] = username - session.permanent = True - dest = request.args.get("next") or url_for("index") - # Prevent open redirect: reject any URL with a scheme or netloc - # (this blocks both http://evil.com AND //evil.com style redirects). - from urllib.parse import urlparse as _urlparse - _parsed = _urlparse(dest) - if _parsed.scheme or _parsed.netloc or not dest.startswith("/"): - dest = url_for("index") - return redirect(dest) - # Generic message — no user enumeration. - return render_template("login.html", error="Invalid username or password"), 401 - if current_role(): - return redirect(url_for("index")) - return render_template("login.html", error=None) - - -@app.route("/logout") -def logout(): - session.clear() - return redirect(url_for("login")) - - -@app.after_request -def set_security_headers(response): - """Defense-in-depth headers. CSP allows only the CDNs the dashboard uses.""" - response.headers["X-Content-Type-Options"] = "nosniff" - response.headers["X-Frame-Options"] = "DENY" - response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" - response.headers["Content-Security-Policy"] = ( - "default-src 'self'; " - "script-src 'self' 'unsafe-inline' 'unsafe-eval' " - "https://cdn.jsdelivr.net https://cdnjs.cloudflare.com " - "https://unpkg.com https://cdn.tailwindcss.com; " - "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net " - "https://cdnjs.cloudflare.com https://fonts.googleapis.com https://unpkg.com; " - "font-src 'self' https://fonts.gstatic.com https://cdnjs.cloudflare.com data:; " - "img-src 'self' data: blob: https:; " - "connect-src 'self' https://api.openalex.org https://api.crossref.org " - "https://ror.org https://*.basemaps.cartocdn.com; " - "worker-src 'self' blob:; " - "frame-ancestors 'none'" - ) - if config.is_production(): - response.headers["Strict-Transport-Security"] = ( - "max-age=31536000; includeSubDomains" - ) - return response - - -@socketio.on("connect") -def _socket_auth(): - """Reject WebSocket connections that are not from a logged-in session, - so the crawler event stream can't be driven anonymously.""" - if not session.get("role"): - return False - return True - - -@app.route("/ark://") -def resolve_ark(naan, name): - """Local ARK resolver (ARK spec: resolver base + '/' + ark). - - Redirects to the dashboard with the paper modal auto-opened; the ARK - 'inflection' suffix '?' / '?info' returns the metadata record as JSON.""" - from flask import redirect - - ark = f"ark:/{naan}/{name}" - session = SessionLocal() - try: - item = session.query(Item).filter_by(ark=ark).first() - if not item: - return jsonify({"error": "ARK not found", "ark": ark}), 404 - # ARK inflection: '?info' / '?json' returns the metadata record. (A bare - # '?' is the spec's brief-metadata inflection, but Flask's full_path - # always appends '?', so it can't be told apart from a plain resolve — - # we require the explicit suffix.) - wants_info = "info" in request.args or "json" in request.args - if wants_info: - return jsonify( - { - "ark": ark, - "docid": item.docid or "", - "doi": item.doi or "", - "title": item.title or "", - "institution": item.institution or "", - "publication_date": ( - item.publication_date.isoformat() - if item.publication_date - else None - ), - "authors": [a.name for a in item.authors], - "resolver": "URAAS / Africa PID Alliance", - } - ) - return redirect(f"/?paper={item.id}") - except Exception as e: - logger.error("resolve_ark %s: %s", ark, e) - return jsonify({"error": "Internal server error"}), 500 - finally: - session.close() - - -@app.route("/api/methodology") -def get_methodology(): - """Full per-metric methodology dictionary — feeds the ⓘ tooltips. - - Every metric on the dashboard documents its formula, data source and - caveats here so each number can be audited (open-methodology principle).""" - from uraas.config.methodology import METHODOLOGY - - return jsonify({"status": "success", "data": METHODOLOGY}) - - -@app.route("/api/stats") -def get_stats(): - try: - return jsonify( - { - "status": "success", - "top_authors": analytics.get_top_authors(limit=5), - "network_edges": analytics.get_department_collaboration_network(), - } - ) - except Exception as e: - logger.error("get_stats: %s", e) - return jsonify({"status": "error", "top_authors": [], "network_edges": []}), 500 - - -@app.route("/api/papers/tree") -def papers_tree(): - try: - institution = request.args.get("institution", None) - return jsonify( - { - "status": "success", - "data": analytics.get_papers_by_faculty_and_department( - institution=institution - ), - } - ) - except Exception as e: - logger.error("papers_tree: %s", e) - return jsonify({"status": "error", "data": []}), 500 - - -@app.route("/api/papers/") -def get_paper(item_id): - session = SessionLocal() - try: - item = session.query(Item).filter_by(id=item_id).first() - if not item: - return jsonify({"error": "Paper not found"}), 404 - file_record = session.query(File).filter_by(item_id=item_id).first() - collections = [ - { - "id": c.id, - "name": c.name, - "faculty": c.community.name if c.community else "Unknown", - } - for c in item.collections - ] - return jsonify( - { - "id": item.id, - "docid": item.docid or "", - "ark": item.ark or "", - "ark_url": f"/{item.ark}" if item.ark else "", - "cited_by_count": item.cited_by_count or 0, - "african_citation_share": item.african_citation_share, - "counts_by_year": ( - json.loads(item.counts_by_year) if item.counts_by_year else [] - ), - "coauthor_countries": item.coauthor_countries or "", - "is_intra_african": bool(item.is_intra_african), - "title": item.title or "Untitled", - "abstract": item.abstract or "", - "doi": item.doi or "", - "url": item.url or "", - "pdf_url": item.pdf_url or "", - "publication_date": ( - item.publication_date.isoformat() if item.publication_date else None - ), - "source_repository": item.source_repository or "", - "authors": [{"name": a.name} for a in item.authors], - "collections": collections, - "dc": { - "title": item.dc_title or "", - "date_issued": item.dc_date_issued or "", - "identifier_uri": item.dc_identifier_uri or "", - "identifier_doi": item.dc_identifier_doi or "", - "description_provenance": item.dc_description_provenance or "", - "rights": item.dc_rights or "", - }, - "file": ( - { - "has_local_pdf": file_record is not None, - "access_policy": ( - file_record.access_policy if file_record else None - ), - "download_url": ( - f"/api/papers/{item_id}/download" if file_record else None - ), - "sha256": file_record.sha256_hash if file_record else None, - } - if file_record - else {"has_local_pdf": False} - ), - "created_at": item.created_at.isoformat() if item.created_at else None, - } - ) - except Exception as e: - logger.error("get_paper %s: %s", item_id, e) - return jsonify({"error": "Internal server error"}), 500 - finally: - session.close() - - -def _is_open_access(item, file_record) -> bool: - """Open-access when the item's rights say so OR the file is policy-Public.""" - rights = (item.dc_rights or "").lower() if item else "" - if "openaccess" in rights.replace("/", "").replace("-", ""): - return True - if file_record and (file_record.access_policy or "").lower() == "public": - return True - return False - - -@app.route("/api/papers//download") -def download_paper(item_id): - session = SessionLocal() - try: - file_record = session.query(File).filter_by(item_id=item_id).first() - if not file_record: - return jsonify({"error": "PDF not found"}), 404 - item = session.query(Item).filter_by(id=item_id).first() - - # Copyright gate: viewers may only download verified open-access files; - # restricted items are admin-only (see PRIVACY_NOTICE / readiness doc). - if not _is_open_access(item, file_record) and current_role() != ADMIN: - return ( - jsonify( - { - "error": "This item is not open access.", - "message": ( - "Full text is restricted by the publisher's licence. " - "Use the publisher or open-access link on the record." - ), - } - ), - 403, - ) - - # Resolve and contain the path under the project/storage root so a stored - # path can never escape via traversal into arbitrary files. - project_root = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ) - file_path = file_record.file_path - if not os.path.isabs(file_path): - file_path = os.path.join(project_root, file_path) - real_path = os.path.realpath(file_path) - allowed_roots = [ - os.path.realpath(project_root), - os.path.realpath(config.STORAGE_PATH), - ] - if not any( - os.path.commonpath([real_path, root]) == root for root in allowed_roots - ): - logger.warning("download_paper %s: path escape blocked: %s", item_id, real_path) - return jsonify({"error": "Access denied"}), 403 - if not os.path.exists(real_path): - return jsonify({"error": "PDF file missing from storage"}), 404 - - filename = ( - f"{item.title[:50]}.pdf" if item and item.title else f"paper_{item_id}.pdf" - ) - filename = "".join( - c for c in filename if c.isalnum() or c in (" ", "-", "_", ".") - ).strip() - return send_file( - real_path, - mimetype="application/pdf", - as_attachment=True, - download_name=filename, - ) - except Exception as e: - logger.error("download_paper %s: %s", item_id, e) - return jsonify({"error": "Failed to download PDF"}), 500 - finally: - session.close() - - -@app.route("/api/papers//bibtex") -def export_single_bibtex(item_id): - session = SessionLocal() - try: - item = session.query(Item).filter_by(id=item_id).first() - if not item: - return jsonify({"error": "Paper not found"}), 404 - authors = [a.name for a in item.authors] - first_last = authors[0].split()[-1] if authors else "Unknown" - year = str(item.publication_date.year) if item.publication_date else "nd" - key = re.sub(r"[^a-zA-Z0-9]", "", f"{first_last}{year}") - author_str = " and ".join(authors) if authors else "Unknown" - title = (item.title or "Untitled").replace("{", "").replace("}", "") - doi_line = f" doi = {{{item.doi}}},\n" if item.doi else "" - url_line = f" url = {{{item.url}}},\n" if item.url else "" - # Persistent identifiers — ARK is resolvable even without a DOI. - note_bits = [] - if item.ark: - note_bits.append(f"ARK: {item.ark}") - if item.docid: - note_bits.append(f"DocID: {item.docid}") - note_line = f" note = {{{'; '.join(note_bits)}}},\n" if note_bits else "" - institution = (item.institution or "").strip() or "Unknown" - bibtex = ( - f"@article{{{key},\n title = {{{title}}},\n author = {{{author_str}}},\n year = {{{year}}},\n institution = {{{institution}}},\n" - + doi_line - + url_line - + note_line - + "}" - ) - return Response( - bibtex, - mimetype="text/plain", - headers={ - "Content-Disposition": f"attachment; filename=paper_{item_id}.bib" - }, - ) - except Exception as e: - logger.error("bibtex %s: %s", item_id, e) - return jsonify({"error": "Export failed"}), 500 - finally: - session.close() - - -@app.route("/api/analytics/overview") -def analytics_overview(): - session = SessionLocal() - institution = request.args.get("institution") - try: - q_item = session.query(Item) - if institution and institution.lower() != "all": - q_item = q_item.filter(Item.institution == institution) - - total = q_item.count() - if total == 0: - return jsonify( - { - "total_papers": 0, - "total_authors": 0, - "total_faculties": 0, - "open_access_papers": 0, - "papers_with_local_pdf": 0, - "oa_percentage": 0, - } - ) - - item_ids_query = q_item.with_entities(Item.id) - - q_author = session.query(Author).join(Author.items).filter(Item.id.in_(item_ids_query)) - q_comm = session.query(Community).filter( - Community.collections.any(Collection.items.any(Item.id.in_(item_ids_query))) - ) - q_file = session.query(File).filter(File.item_id.in_(item_ids_query)) - - authors = q_author.distinct().count() - faculties = q_comm.distinct().count() - oa = q_item.filter(Item.dc_rights.like("%openAccess%")).count() - pdfs = q_file.count() - - return jsonify( - { - "total_papers": total, - "total_authors": authors, - "total_faculties": faculties, - "open_access_papers": oa, - "papers_with_local_pdf": pdfs, - "oa_percentage": round((oa / total * 100) if total else 0, 1), - } - ) - except Exception as e: - logger.error("analytics_overview: %s", e) - return ( - jsonify( - { - "total_papers": 0, - "total_authors": 0, - "total_faculties": 0, - "open_access_papers": 0, - "papers_with_local_pdf": 0, - "oa_percentage": 0, - } - ), - 500, - ) - finally: - session.close() - - -@app.route("/api/analytics/publications-by-year") -def publications_by_year(): - institution = request.args.get("institution") - return jsonify(analytics.get_publications_by_year(institution=institution)) - - -@app.route("/api/analytics/papers-by-faculty") -def papers_by_faculty(): - institution = request.args.get("institution") - return jsonify(analytics.get_papers_by_faculty(institution=institution)) - - -@app.route("/api/analytics/top-authors") -def top_authors_analytics(): - limit = clamped_int("limit", 15, 1, 200) - institution = request.args.get("institution") - return jsonify( - analytics.get_authors_by_papers(limit=limit, institution=institution) - ) - - -@app.route("/api/analytics/open-access-breakdown") -def oa_breakdown(): - institution = request.args.get("institution") - return jsonify(analytics.get_open_access_breakdown(institution=institution)) - - -@app.route("/api/analytics/recent-papers") -def recent_papers(): - limit = clamped_int("limit", 10, 1, 50) - session = SessionLocal() - try: - items = session.query(Item).order_by(desc(Item.created_at)).limit(limit).all() - return jsonify( - [ - { - "id": i.id, - "title": i.title, - "doi": i.doi, - "authors": [a.name for a in i.authors[:3]], - "created_at": i.created_at.isoformat() if i.created_at else None, - "is_oa": "openAccess" in (i.dc_rights or ""), - } - for i in items - ] - ) - finally: - session.close() - - -@app.route("/api/analytics/growth-rate") -def growth_rate(): - institution = request.args.get("institution") - # Backward compatibility with JS which expects 'session' instead of 'month' - data = analytics.get_institutional_growth(institution=institution) - return jsonify([{"session": d["month"], "count": d["count"]} for d in data]) - - -@app.route("/api/analytics/timeline") -def timeline(): - institution = request.args.get("institution") - return jsonify(analytics.get_timeline_data(institution=institution)) - - -@app.route("/api/analytics/papers-by-year-faculty") -def papers_by_year_faculty(): - session = SessionLocal() - try: - rows = ( - session.query( - db_year(Item.publication_date).label("year"), - Community.name.label("faculty"), - func.count(Item.id).label("count"), - ) - .join(Item.collections) - .join(Collection.community) - .filter(Item.publication_date.isnot(None)) - .group_by("year", Community.name) - .order_by("year") - .all() - ) - return jsonify( - [ - {"year": int(r.year), "faculty": r.faculty, "count": r.count} - for r in rows - if r.year - ] - ) - finally: - session.close() - - -@app.route("/api/analytics/faculty-oa-breakdown") -def faculty_oa_breakdown(): - """Phase 7 fix: group-by aggregate, no per-faculty item loads.""" - institution = request.args.get("institution") - session = SessionLocal() - try: - # Two aggregate queries instead of N item fetches - base_q = ( - session.query( - Community.name, - func.count(Item.id).label("total"), - ) - .join(Item.collections) - .join(Collection.community) - ) - oa_q = ( - session.query( - Community.name, - func.count(Item.id).label("oa"), - ) - .join(Item.collections) - .join(Collection.community) - .filter(Item.dc_rights.like("%openAccess%")) - ) - if institution: - inst_name = analytics._resolve_institution_name(institution) - if inst_name: - base_q = base_q.filter(Item.institution.ilike(f"%{inst_name}%")) - oa_q = oa_q.filter(Item.institution.ilike(f"%{inst_name}%")) - - totals = {row.name: row.total for row in base_q.group_by(Community.name).all()} - oas = {row.name: row.oa for row in oa_q.group_by(Community.name).all()} - - return jsonify( - [ - { - "faculty": name, - "oa": oas.get(name, 0), - "restricted": totals[name] - oas.get(name, 0), - } - for name in totals - ] - ) - except Exception as e: - logger.error("faculty_oa_breakdown: %s", e) - return jsonify([]) - finally: - session.close() - - -@app.route("/api/analytics/impact-metrics") -def impact_metrics(): - session = SessionLocal() - institution = request.args.get("institution") - try: - inst_name = ( - analytics._resolve_institution_name(institution) if institution else None - ) - - q_item = session.query(Item) - if inst_name: - q_item = q_item.filter(Item.institution.ilike(f"%{inst_name}%")) - - total = q_item.count() - oa = q_item.filter(Item.dc_rights.like("%openAccess%")).count() - with_doi = q_item.filter(Item.doi.isnot(None)).count() - - q_file = session.query(File) - if inst_name: - q_file = q_file.join(File.item).filter( - Item.institution.ilike(f"%{inst_name}%") - ) - with_pdf = q_file.count() - - q_years = session.query(db_year(Item.publication_date)).filter( - Item.publication_date.isnot(None) - ) - if inst_name: - q_years = q_years.filter(Item.institution.ilike(f"%{inst_name}%")) - years = q_years.distinct().count() - - return jsonify( - { - "total_papers": total, - "open_access_papers": oa, - "oa_rate": round(oa / total * 100, 1) if total else 0, - "papers_with_doi": with_doi, - "doi_rate": round(with_doi / total * 100, 1) if total else 0, - "papers_with_local_pdf": with_pdf, - "pdf_rate": round(with_pdf / total * 100, 1) if total else 0, - "years_covered": years, - } - ) - except Exception as e: - logger.error("impact_metrics: %s", e) - return jsonify({"error": str(e)}), 500 - finally: - session.close() - - -@app.route("/api/analytics/search") -def analytics_search(): - q = request.args.get("q", "").strip() - faculty = request.args.get("faculty", "").strip() - year_from = request.args.get("year_from", type=int) - year_to = request.args.get("year_to", type=int) - oa_only = request.args.get("oa_only", "").lower() == "true" - limit = clamped_int("limit", 50, 1, 200) - session = SessionLocal() - try: - q_obj = session.query(Item) - if q: - author_subq = ( - session.query(Item.id) - .join(Item.authors) - .filter(Author.name.ilike(f"%{q}%")) - .subquery() - ) - q_obj = q_obj.filter( - or_( - Item.title.ilike(f"%{q}%"), - Item.abstract.ilike(f"%{q}%"), - Item.doi.ilike(f"%{q}%"), - Item.id.in_(author_subq), - ) - ) - if faculty: - q_obj = ( - q_obj.join(Item.collections) - .join(Collection.community) - .filter(Community.name.ilike(f"%{faculty}%")) - ) - if year_from: - q_obj = q_obj.filter(db_year(Item.publication_date) >= str(year_from)) - if year_to: - q_obj = q_obj.filter(db_year(Item.publication_date) <= str(year_to)) - if oa_only: - q_obj = q_obj.filter(Item.dc_rights.like("%openAccess%")) - items = q_obj.order_by(desc(Item.created_at)).limit(limit).all() - return jsonify( - [ - { - "id": i.id, - "title": i.title, - "doi": i.doi, - "abstract_snippet": ( - (i.abstract or "")[:200] - if q and i.abstract and q.lower() in (i.abstract or "").lower() - else None - ), - "authors": [a.name for a in i.authors[:4]], - "year": i.publication_date.year if i.publication_date else None, - "is_oa": "openAccess" in (i.dc_rights or ""), - "faculty": ( - i.collections[0].community.name if i.collections else None - ), - } - for i in items - ] - ) - finally: - session.close() - - -@app.route("/api/analytics/faculties") -def list_faculties(): - session = SessionLocal() - institution = request.args.get("institution") - try: - inst_name = ( - analytics._resolve_institution_name(institution) if institution else None - ) - q = session.query(Community.name).order_by(Community.name) - if inst_name: - q = q.filter( - Community.institution.ilike(f"%{inst_name}%") - | Community.name.ilike(f"%{inst_name}%") - ) - rows = q.all() - return jsonify([r[0] for r in rows]) - finally: - session.close() - - -@app.route("/api/institutions") -def list_institutions(): - """List all configured institutions with their staff counts and OAI support flag.""" - from uraas.config.institutions import get_registry - - registry = get_registry() - results = [] - for inst in registry.list_all(): - results.append( - { - "name": inst.name, - "short_name": inst.short_name, - "ror": inst.ror, - "sub_region": inst.sub_region, - "staff_count": len(inst.staff_names), - "has_oai": bool(inst.oai_endpoint), - "oai_endpoint": inst.oai_endpoint or None, - } - ) - return jsonify(results) - - -@app.route("/api/analytics/authors-search") -def authors_search(): - q = request.args.get("q", "") - limit = clamped_int("limit", 10, 1, 100) - institution = request.args.get("institution") - session = SessionLocal() - try: - inst_name = ( - analytics._resolve_institution_name(institution) if institution else None - ) - query = ( - session.query(Author.name, func.count(Item.id)) - .join(Author.items) - .filter(Author.name.ilike(f"%{q}%")) - ) - if inst_name: - query = query.filter(Item.institution.ilike(f"%{inst_name}%")) - authors = ( - query.group_by(Author.name) - .order_by(desc(func.count(Item.id))) - .limit(limit) - .all() - ) - return jsonify([{"name": a[0], "papers": a[1]} for a in authors]) - finally: - session.close() - - -@app.route("/api/analytics/author-network") -def author_network(): - author_name = request.args.get("author", "").strip() - return jsonify(analytics.get_author_network(author_name=author_name or None)) - - -@app.route("/api/analytics/keyword-cloud") -def keyword_cloud(): - institution = request.args.get("institution", None) - top_n = clamped_int("top_n", 60, 1, 150) - return jsonify(analytics.get_keyword_cloud(top_n=top_n, institution=institution)) - - -@app.route("/api/analytics/institution-leaderboard") -def institution_leaderboard(): - return jsonify(analytics.get_institution_leaderboard()) - - -@app.route("/api/analytics/cache-flush", methods=["POST"]) -def flush_analytics_cache(): - """Manual cache flush endpoint (admin use).""" - analytics_cache.invalidate_all() - return jsonify({"status": "success", "message": "Analytics cache flushed"}) - - -@app.route("/api/admin/prune-non-sc", methods=["POST"]) -def prune_non_sc(): - """Re-classify all items and delete non-SC papers (admin only). - - POST body: {"apply": true} — actually prune - POST body: {"apply": false} — dry run (default), just return counts - """ - from uraas.database import Author, Collection, Community, SessionLocal as _SL, item_authors - from uraas.services.sc_engine import is_special_collection - from sqlalchemy import text as _text - - data = request.get_json() or {} - apply = bool(data.get("apply", False)) - - session = _SL() - try: - items = session.query(Item).all() - total = len(items) - keep_ids, drop_ids = [], [] - samples = [] - - for it in items: - is_sc, score, cats = is_special_collection( - it.title or "", it.abstract or "", it.dc_subject or "" - ) - if apply: - it.special_collection_score = float(score) - it.special_collection_categories = ",".join(cats) if is_sc else "" - if is_sc: - keep_ids.append(it.id) - else: - drop_ids.append(it.id) - if len(samples) < 20: - samples.append({ - "id": it.id, - "title": (it.title or "")[:80], - "institution": it.institution or "", - }) - - if apply: - session.commit() - # Delete non-SC items in chunks - session.execute(_text("PRAGMA foreign_keys=ON")) - deleted = 0 - for i in range(0, len(drop_ids), 500): - chunk = drop_ids[i:i + 500] - for it in session.query(Item).filter(Item.id.in_(chunk)).all(): - session.delete(it) - deleted += 1 - session.commit() - - # Sweep stray association rows - session.execute(_text( - "DELETE FROM item_authors WHERE item_id NOT IN (SELECT id FROM items) " - "OR author_id NOT IN (SELECT id FROM authors)" - )) - session.execute(_text( - "DELETE FROM item_collections WHERE item_id NOT IN (SELECT id FROM items) " - "OR collection_id NOT IN (SELECT id FROM collections)" - )) - session.commit() - - # Orphan authors - orphan_authors = ( - session.query(Author) - .filter(~Author.id.in_(session.query(item_authors.c.author_id))) - .all() - ) - for a in orphan_authors: - session.delete(a) - empty_colls = [c for c in session.query(Collection).all() if not c.items] - for c in empty_colls: - session.delete(c) - session.commit() - empty_comms = [c for c in session.query(Community).all() if not c.collections] - for c in empty_comms: - session.delete(c) - session.commit() - - analytics_cache.invalidate_all() - - return jsonify({ - "status": "success", - "applied": apply, - "total_before": total, - "kept_sc": len(keep_ids), - "pruned_non_sc": len(drop_ids), - "sample_dropped": samples, - }) - except Exception as e: - logger.error("prune_non_sc: %s", e) - session.rollback() - return jsonify({"status": "error", "message": str(e)}), 500 - finally: - session.close() - - -@app.route("/api/admin/test-smtp", methods=["POST"]) -def test_smtp(): - """Send a test email to verify SMTP configuration (admin only).""" - from uraas.services.email_service import _is_smtp_configured - from uraas.config import config - import smtplib - from email.mime.text import MIMEText - from email.utils import parseaddr - - if not _is_smtp_configured(): - return jsonify({ - "status": "error", - "message": "SMTP not configured — set SMTP_HOST, SMTP_USER, SMTP_PASSWORD in environment/secrets.", - "smtp_host": config.SMTP_HOST or "(not set)", - "smtp_user": config.SMTP_USER or "(not set)", - }), 400 - - to_email = (request.get_json(silent=True) or {}).get("to", config.SMTP_USER) - try: - msg = MIMEText("URAAS SMTP test — configuration is working correctly.", "plain", "utf-8") - msg["Subject"] = "[URAAS] SMTP Test" - msg["From"] = config.SMTP_FROM - msg["To"] = to_email - if config.SMTP_USE_TLS: - server = smtplib.SMTP(config.SMTP_HOST, config.SMTP_PORT, timeout=15) - server.ehlo(); server.starttls(); server.ehlo() - else: - server = smtplib.SMTP_SSL(config.SMTP_HOST, config.SMTP_PORT, timeout=15) - server.login(config.SMTP_USER, config.SMTP_PASSWORD) - envelope_from = parseaddr(config.SMTP_FROM)[1] or config.SMTP_USER - server.sendmail(envelope_from, [to_email], msg.as_bytes()) - server.quit() - logger.info("SMTP test email sent to %s", to_email) - return jsonify({"status": "success", "message": f"Test email sent to {to_email}"}) - except Exception as exc: - logger.error("SMTP test failed: %s", exc) - return jsonify({"status": "error", "message": str(exc)}), 500 - - -@app.route("/api/admin/recompute-alignment", methods=["POST"]) -def recompute_alignment(): - """Score all SC items against AU framework pillars and rebuild AlignmentAggregate (admin only). - - This is the web-UI equivalent of running scripts/backfill_alignment.py. - Runs synchronously; expect 5-30 s for 100-200 papers.""" - from uraas.services.alignment_engine import recompute_aggregates, score_item_alignment - - session = SessionLocal() - try: - q = session.query(Item).filter(Item.special_collection_score > 0) - items = q.all() - scored = 0 - for it in items: - try: - al_json, al_ver = score_item_alignment( - it.title or "", it.abstract or "", it.dc_subject or "" - ) - it.alignment_scores = al_json - it.alignment_version = al_ver - scored += 1 - except Exception as e: - logger.warning("alignment scoring failed for item %s: %s", it.id, e) - session.commit() - - rows = recompute_aggregates(session) - analytics_cache.invalidate_all() - logger.info("recompute_alignment: scored=%d, aggregate_rows=%d", scored, rows) - return jsonify({"status": "success", "scored": scored, "aggregate_rows": rows}) - except Exception as e: - logger.error("recompute_alignment: %s", e) - session.rollback() - return jsonify({"status": "error", "message": str(e)}), 500 - finally: - session.close() - - -@app.route("/api/analytics/faculty-comparison") -def faculty_comparison(): - faculties = request.args.getlist("faculty") - institution = request.args.get("institution", None) - session = SessionLocal() - try: - result = {} - # If no specific faculties given, return all - if not faculties: - q = session.query(Community) - if institution: - q = q.filter(Community.institution.ilike(f"%{institution}%")) - comms = q.all() - else: - comms = [] - for fac_name in faculties: - comm = ( - session.query(Community) - .filter(Community.name.ilike(f"%{fac_name}%")) - .first() - ) - if comm: - comms.append(comm) - - for comm in comms: - # Skip vague community names that are actually institution names - if comm.name and len(comm.name) < 5: - continue - items = ( - session.query(Item) - .join(Item.collections) - .join(Collection.community) - .filter(Community.id == comm.id) - .options(selectinload(Item.authors)) - .all() - ) - if not items: - continue - oa_count = sum(1 for i in items if "openAccess" in (i.dc_rights or "")) - years = [i.publication_date.year for i in items if i.publication_date] - authors_set = set(a.name for i in items for a in i.authors) - # Top 3 keywords across all papers - from collections import Counter - - kw_counter = Counter() - for i in items: - for kw in (i.ai_keywords or "").split(","): - kw = kw.strip() - if len(kw) > 3: - kw_counter[kw] += 1 - top_keywords = [k for k, _ in kw_counter.most_common(5)] - result[comm.name] = { - "total_papers": len(items), - "open_access": oa_count, - "restricted": len(items) - oa_count, - "oa_rate": round(oa_count / len(items) * 100, 1) if items else 0, - "unique_authors": len(authors_set), - "year_range": [min(years), max(years)] if years else [], - "peak_year": max(set(years), key=years.count) if years else None, - "departments": len(comm.collections), - "top_keywords": top_keywords, - "institution": comm.institution or "Unknown", - } - return jsonify(result) - finally: - session.close() - - -@app.route("/api/analytics/department-comparison") -def department_comparison(): - faculty = request.args.get("faculty", "") - session = SessionLocal() - try: - comm = ( - session.query(Community) - .filter(Community.name.ilike(f"%{faculty}%")) - .first() - ) - if not comm: - return jsonify({}) - result = {} - for coll in comm.collections: - items = ( - session.query(Item) - .join(Item.collections) - .filter(Collection.id == coll.id) - .options(selectinload(Item.authors)) - .all() - ) - if not items: - continue - oa = sum(1 for i in items if "openAccess" in (i.dc_rights or "")) - years = [i.publication_date.year for i in items if i.publication_date] - result[coll.name] = { - "total": len(items), - "open_access": oa, - "restricted": len(items) - oa, - "oa_rate": round(oa / len(items) * 100, 1) if items else 0, - "unique_authors": len(set(a.name for i in items for a in i.authors)), - "years": sorted(set(years)), - } - return jsonify(result) - finally: - session.close() - - -@app.route("/api/analytics/lecturer-profile") -def lecturer_profile(): - name = request.args.get("name", "").strip() - session = SessionLocal() - try: - author = session.query(Author).filter(Author.name.ilike(f"%{name}%")).first() - if not author: - return jsonify({"error": "Author not found"}), 404 - # Eager-load the relationships walked below (collections→community, authors) - # so the profile renders in a handful of queries instead of one-per-paper. - items = ( - session.query(Item) - .filter(Item.authors.any(Author.id == author.id)) - .options( - selectinload(Item.authors), - selectinload(Item.collections).selectinload(Collection.community), - ) - .all() - ) - oa = sum(1 for i in items if "openAccess" in (i.dc_rights or "")) - years = sorted( - set(i.publication_date.year for i in items if i.publication_date) - ) - faculties = list( - set(c.community.name for i in items for c in i.collections if c.community) - ) - depts = list(set(c.name for i in items for c in i.collections)) - co_authors = {} - for item in items: - for a in item.authors: - if a.name != author.name: - co_authors[a.name] = co_authors.get(a.name, 0) + 1 - top_co = sorted(co_authors.items(), key=lambda x: -x[1])[:10] - return jsonify( - { - "name": author.name, - "total_papers": len(items), - "open_access": oa, - "oa_rate": round(oa / len(items) * 100, 1) if items else 0, - "active_years": years, - "faculties": faculties, - "departments": depts, - "top_collaborators": [{"name": n, "papers": c} for n, c in top_co], - "papers": [ - { - "id": i.id, - "title": i.title, - "doi": i.doi, - "year": i.publication_date.year if i.publication_date else None, - "is_oa": "openAccess" in (i.dc_rights or ""), - } - for i in sorted( - items, - key=lambda x: x.publication_date - or __import__("datetime").datetime.min, - reverse=True, - )[:20] - ], - } - ) - finally: - session.close() - - -@app.route("/api/analytics/language-research") -def language_research(): - """Language & Culture research — returns SC papers matched by language keywords.""" - from uraas.config.language_research import score_item - from uraas.services.sc_engine import SC_FILTER - - session = SessionLocal() - try: - institution = request.args.get("institution", "").strip() - q = session.query(Item).filter(SC_FILTER) - if institution: - q = q.filter(Item.institution.ilike(f"%{institution}%")) - items = q.all() - - matches = [] - keyword_counts: dict = {} - for item in items: - try: - score, matched = score_item(item.title or "", item.abstract or "") - if not score: - continue - # Count only the actual matched language terms (not all words) - for term in matched: - t = term.lower() - keyword_counts[t] = keyword_counts.get(t, 0) + 1 - matches.append( - { - "id": item.id, - "title": item.title, - "year": ( - item.publication_date.year if item.publication_date else None - ), - "authors": [a.name for a in item.authors[:4]], - "is_oa": "openAccess" in (item.dc_rights or ""), - "score": score, - "matched_terms": matched[:6], - } - ) - except Exception as e: - logger.error("language_research item %s: %s", item.id, e) - continue - - matches.sort(key=lambda x: (-x["score"], -(x.get("year") or 0))) - top_keywords = sorted(keyword_counts.items(), key=lambda x: -x[1])[:20] - return jsonify( - { - "total_language_papers": len(matches), - "top_keywords": [{"keyword": k, "count": v} for k, v in top_keywords], - "papers": matches[:50], - } - ) - except Exception as e: - logger.error("language_research: %s", e) - return ( - jsonify( - { - "error": "Internal server error", - "total_language_papers": 0, - "top_keywords": [], - "papers": [], - } - ), - 500, - ) - finally: - session.close() - - - - -# Multi-Institution Comparator (APA Core Feature) - - -@app.route("/api/comparator/compare", methods=["POST"]) -def compare_institutions(): - """ - Compare multiple institutions across all metrics. - Request body: {"ror_ids": ["ror1", "ror2", "ror3"]} - """ - try: - from uraas.services.comparator_engine import ComparatorEngine - - data = request.get_json() - ror_ids = data.get("ror_ids", []) - - if not ror_ids or len(ror_ids) < 2: - return jsonify({"error": "Provide at least 2 ROR IDs"}), 400 - - if len(ror_ids) > 15: - return jsonify({"error": "Maximum 15 institutions"}), 400 - - comparison = ComparatorEngine.compare_institutions(ror_ids) - return jsonify(comparison) - - except Exception as e: - logger.error(f"compare_institutions: {e}") - return jsonify({"error": str(e)}), 500 - - -@app.route("/api/comparator/collaboration-mesh", methods=["POST"]) -def collaboration_mesh(): - """ - Get collaboration network data for geographic visualization. - Request body: {"ror_ids": ["ror1", "ror2", "ror3"]} - """ - try: - from uraas.services.comparator_engine import ComparatorEngine - - data = request.get_json() - ror_ids = data.get("ror_ids", []) - - if not ror_ids: - return jsonify({"error": "Provide ROR IDs"}), 400 - - mesh = ComparatorEngine.get_collaboration_matrix(ror_ids) - return jsonify(mesh) - - except Exception as e: - logger.error(f"collaboration_mesh: {e}") - return jsonify({"error": str(e)}), 500 - - -@app.route("/api/comparator/senate-report", methods=["POST"]) -def generate_senate_report(): - """ - Generate comprehensive senate report. - Request body: {"ror_ids": ["ror1", "ror2"], "format": "json"} - """ - try: - from uraas.services.comparator_engine import ComparatorEngine - - data = request.get_json() - ror_ids = data.get("ror_ids", []) - format_type = data.get("format", "json") - - if not ror_ids: - return jsonify({"error": "Provide ROR IDs"}), 400 - - report = ComparatorEngine.generate_senate_report(ror_ids, format_type) - - if format_type == "json": - return jsonify(report) - elif format_type == "csv": - output = io.StringIO() - writer = csv.writer(output) - writer.writerow( - [ - "Institution", - "ROR", - "SC Papers", - "SC Authors", - "OA Rate %", - "Indigenous Knowledge", - "African Literature", - "Papers/Author", - ] - ) - for inst in report["detailed_comparison"]["institutions"]: - m = inst["metrics"] - writer.writerow( - [ - inst["name"], - inst["ror_id"], - m.get("total_papers", 0), - m.get("total_authors", 0), - m.get("oa_rate", 0), - m.get("indigenous_knowledge", 0), - m.get("african_literature", 0), - m.get("papers_per_author", 0), - ] - ) - output.seek(0) - return Response( - output.getvalue(), - mimetype="text/csv", - headers={ - "Content-Disposition": "attachment; filename=senate_report.csv" - }, - ) - elif format_type == "pdf": - # Plain-text report (PDF rendering would need reportlab; keep deps minimal) - lines = [ - report["title"], - "=" * len(report["title"]), - f"Generated: {report['generated_at']}", - f"Institutions: {report['institutions_analyzed']}", - "", - "Executive Summary:", - ] - for k, v in report["executive_summary"].items(): - lines.append(f" {k}: {v}") - lines += ["", "Recommendations:"] - for r in report.get("recommendations", []): - lines.append(f" - {r}") - return Response( - "\n".join(lines), - mimetype="text/plain", - headers={ - "Content-Disposition": "attachment; filename=senate_report.txt" - }, - ) - else: - return jsonify({"error": "Invalid format"}), 400 - - except Exception as e: - logger.error(f"generate_senate_report: {e}") - return jsonify({"error": str(e)}), 500 - - -# Citation Tracking & Bibliometrics - - -@app.route("/api/citations/") -def get_citations(item_id): - """Get citation data for a paper — enriched with ARK + Pan-African share (Phase 5).""" - session = SessionLocal() - try: - item = session.query(Item).filter_by(id=item_id).first() - # Base citation count from DB (fast, no API call) - base = { - "item_id": item_id, - "citation_count": item.cited_by_count or 0 if item else 0, - "ark": item.ark or "" if item else "", - "docid": item.docid or "" if item else "", - "african_citation_share": item.african_citation_share if item else None, - "openalex_id": item.openalex_id or "" if item else "", - } - except Exception: - base = {"item_id": item_id, "citation_count": 0} - finally: - session.close() - - # Supplement with live CitationMetrics record if available - try: - from uraas.services.citation_tracker import get_paper_citations - live = get_paper_citations(item_id) - base.update(live) - except Exception as e: - logger.debug(f"get_citations live lookup {item_id}: {e}") - return jsonify(base) - - -@app.route("/api/citations/velocity/export.csv") -def citations_velocity_csv(): - """Phase 4 — Streaming CSV export of the citation velocity time-series. - - Columns: year, citations_received, pan_african_share_pct, items_covered - Query param: ?institution= - """ - try: - institution = request.args.get("institution", "").strip().lower() or None - data = analytics.get_citation_velocity(institution) - series = data.get("by_year", []) - share = data.get("pan_african_share_pct") - covered = data.get("pan_african_share_items", 0) - - def gen(): - yield [ - "Year", - "Citations Received", - "Pan-African Share %", - "Items Covered (pan-African)", - ] - for row in series: - yield [ - row["year"], - row["citations"], - share if share is not None else "", - covered, - ] - - return csv_response(gen(), "citation_velocity.csv") - except Exception as e: - logger.error(f"citations_velocity_csv: {e}") - return api_error(str(e)) - - -@app.route("/api/citations/update/", methods=["POST"]) -def update_citations(item_id): - """Manually trigger citation update for a paper.""" - try: - from uraas.services.citation_tracker import CitationTracker - - success = CitationTracker.update_paper_citations(item_id) - return jsonify({"success": success, "item_id": item_id}) - except Exception as e: - logger.error(f"update_citations {item_id}: {e}") - return jsonify({"error": str(e)}), 500 - - -@app.route("/api/author//metrics") -def get_author_metrics(author_id): - """Get bibliometric indicators for an author (h-index, citations, etc.).""" - try: - from uraas.services.citation_tracker import get_author_bibliometrics - - return jsonify(get_author_bibliometrics(author_id)) - except Exception as e: - logger.error(f"get_author_metrics {author_id}: {e}") - return jsonify({"error": str(e)}), 500 - - -@app.route("/api/citations/bulk-update", methods=["POST"]) -def bulk_update_citations(): - """Bulk update citations for papers (admin endpoint).""" - try: - from uraas.services.citation_tracker import CitationTracker - - limit = clamped_int("limit", 50, 1, 200) - force = request.args.get("force", "false").lower() == "true" - stats = CitationTracker.bulk_update_citations(limit=limit, force=force) - return jsonify(stats) - except Exception as e: - logger.error(f"bulk_update_citations: {e}") - return jsonify({"error": str(e)}), 500 - - -# Advanced Search - - -@app.route("/api/search/advanced") -def advanced_search(): - """ - Advanced search with Boolean operators and field-specific queries. - - Query examples: - ?q="machine learning" AND author:smith - ?q=(covid OR pandemic) AND year:2020 - ?q=title:cancer NOT lung - ?q=author:okonkwo AND faculty:science - - Supported fields: - title, abstract, author, year, doi, faculty, department, keyword, language, oa - - Operators: - AND, OR, NOT, parentheses for grouping, "quotes" for phrases - """ - try: - from uraas.services.advanced_search import SearchQuery - - query = request.args.get("q", "").strip() - limit = clamped_int("limit", 50, 1, 200) - offset = clamped_int("offset", 0, 0, 100000) - sort_by = request.args.get( - "sort", "relevance" - ) # relevance, date, citations, title - - filters = { - "year_from": request.args.get("year_from", type=int), - "year_to": request.args.get("year_to", type=int), - "oa_only": request.args.get("oa_only", "false").lower() == "true", - "faculty": request.args.get("faculty"), - "has_pdf": request.args.get("has_pdf", "false").lower() == "true", - } - - results = SearchQuery.execute_search( - query=query, limit=limit, offset=offset, sort_by=sort_by, filters=filters - ) - - return jsonify(results) - - except Exception as e: - logger.error(f"advanced_search: {e}") - return jsonify({"error": str(e), "total": 0, "results": []}), 500 - - -@app.route("/api/search/suggest") -def search_suggestions(): - """Get autocomplete suggestions for search queries.""" - try: - from uraas.services.advanced_search import SearchQuery - - partial = request.args.get("q", "").strip() - field = request.args.get("field", "all") - - suggestions = SearchQuery.get_search_suggestions(partial, field) - return jsonify({"suggestions": suggestions}) - - except Exception as e: - logger.error(f"search_suggestions: {e}") - return jsonify({"suggestions": []}), 500 - - -# APA Novel Metrics - - -@app.route("/api/analytics/tk-vitality-score") -def tk_vitality_score(): - institution = request.args.get("institution", None) - return jsonify(analytics.get_tk_vitality_score(institution=institution)) - - -@app.route("/api/analytics/linguistic-diversity-index") -def linguistic_diversity_index(): - return jsonify(analytics.get_linguistic_diversity_index()) - - -@app.route("/api/analytics/pid-coverage") -def pid_coverage(): - institution = request.args.get("institution", None) - return jsonify(analytics.get_pid_coverage(institution=institution)) - - -@app.route("/api/analytics/knowledge-repatriation") -def knowledge_repatriation(): - institution = request.args.get("institution", None) - return jsonify(analytics.get_knowledge_repatriation(institution=institution)) - - -@app.route("/api/analytics/research-diversity") -def research_diversity(): - institution = request.args.get("institution", None) - return jsonify(analytics.get_research_diversity(institution=institution)) - - -@app.route("/api/analytics/open-science-health") -def open_science_health(): - institution = request.args.get("institution", None) - return jsonify(analytics.get_open_science_health(institution=institution)) - - -@app.route("/api/analytics/special-collections") -def special_collections(): - institution = request.args.get("institution", None) - return jsonify(analytics.get_special_collections_metrics(institution=institution)) - - -@app.route("/api/analytics/special-collections/overview") -def special_collections_overview(): - institution = request.args.get("institution", None) - return jsonify(analytics.get_special_collections_overview(institution=institution)) - - -@app.route("/api/analytics/special-collections/export.csv") -def export_special_collections_csv(): - """Download special collections data as CSV.""" - try: - rows = analytics.get_special_collections_csv_data() - output = io.StringIO() - writer = csv.writer(output) - for row in rows: - writer.writerow(row) - output.seek(0) - return Response( - output.getvalue(), - mimetype="text/csv", - headers={ - "Content-Disposition": "attachment; filename=uraas_special_collections.csv" - }, - ) - except Exception as e: - logger.error(f"export_special_collections_csv: {e}") - return jsonify({"error": str(e)}), 500 - - -# ── Framework Alignment endpoints (AU charters / Agenda 2063 / blocs) ──────── -# Scores are precomputed at ingest / by scripts/backfill_alignment.py and read -# from alignment_aggregates — these endpoints never score at request time. - - -def _resolve_inst_name(): - institution = request.args.get("institution", "").strip().lower() - return analytics._resolve_institution_name(institution) if institution else None - - -def _alignment_top_papers(session, top_item_ids, framework, pillar): - """Evidence chips: resolve top item ids to titles + matched keywords.""" - from uraas.services.alignment_engine import get_alignment - - ids = [int(i) for i in (top_item_ids or "").split(",") if i] - papers = [] - for item in session.query(Item).filter(Item.id.in_(ids)).all(): - pdata = ( - get_alignment(item).get(framework, {}).get("pillars", {}).get(pillar, {}) - ) - papers.append( - { - "id": item.id, - "title": item.title or "Untitled", - "score": pdata.get("score", 0), - "matched_keywords": pdata.get("matched_keywords", [])[:4], - } - ) - papers.sort(key=lambda p: -p["score"]) - return papers - - -@app.route("/api/alignment/frameworks") -def alignment_frameworks(): - """All frameworks + pillar metadata for the selector UI.""" - from uraas.config.alignment_frameworks import FRAMEWORK_GROUPS, FRAMEWORKS - from uraas.services.alignment_engine import scoring_mode - - frameworks = [ - { - "key": fkey, - "name": f["name"], - "type": f["type"], - "year": f.get("year"), - "color": f.get("color", "#3b82f6"), - "pillars": [ - {"key": pk, "name": p["name"], "description": p["description"]} - for pk, p in f["pillars"].items() - ], - } - for fkey, f in FRAMEWORKS.items() - ] - return api_ok( - {"frameworks": frameworks, "groups": FRAMEWORK_GROUPS, "mode": scoring_mode()} - ) - - -@app.route("/api/alignment/profile") -def alignment_profile(): - """Radar-chart profile for one framework: per-pillar avg score + evidence.""" - from uraas.config.alignment_frameworks import FRAMEWORKS, GAP_THRESHOLD - from uraas.database import AlignmentAggregate - from uraas.services.narratives import narrate - - framework = request.args.get("framework", "agenda2063") - if framework not in FRAMEWORKS: - return api_error(f"Unknown framework: {framework}", 400) - inst_name = _resolve_inst_name() - - cache_key = f"align_profile_{framework}_{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached: - return jsonify(cached) - - session = SessionLocal() - try: - fdef = FRAMEWORKS[framework] - rows = { - r.pillar: r - for r in session.query(AlignmentAggregate) - .filter_by(institution=inst_name or "", framework=framework) - .all() - } - pillars = [] - for pkey, pdef in fdef["pillars"].items(): - r = rows.get(pkey) - avg = r.avg_score if r else 0.0 - pillars.append( - { - "key": pkey, - "name": pdef["name"], - "avg_score": avg, - "paper_count": r.paper_count if r else 0, - "is_gap": avg < GAP_THRESHOLD, - "top_papers": ( - _alignment_top_papers(session, r.top_item_ids, framework, pkey) - if r - else [] - ), - } - ) - - scores = [p["avg_score"] for p in pillars] - top = max(pillars, key=lambda p: p["avg_score"]) if pillars else None - data = { - "framework": framework, - "framework_name": fdef["name"], - "color": fdef.get("color", "#3b82f6"), - "institution": inst_name or "All Institutions", - "pillars": pillars, - "overall_score": round(sum(scores) / len(scores), 1) if scores else 0, - "gap_threshold": GAP_THRESHOLD, - } - narrative = ( - narrate( - "alignment_profile", - institution=inst_name or "The repository", - top_pillar=top["name"], - top_score=top["avg_score"], - top_count=top["paper_count"], - gap_count=sum(1 for p in pillars if p["is_gap"]), - pillar_count=len(pillars), - threshold=GAP_THRESHOLD, - ) - if top - else "" - ) - resp = api_ok(data, narrative=narrative, methodology_key="alignment_score") - analytics_cache.set(cache_key, resp.get_json(), ttl=1800) - return resp - except Exception as e: - logger.error(f"alignment_profile: {e}") - return api_error(str(e)) - finally: - session.close() - - -@app.route("/api/alignment/matrix") -def alignment_matrix(): - """Heatmap cells: every (framework, pillar) avg score for one institution.""" - from uraas.config.alignment_frameworks import FRAMEWORKS - from uraas.database import AlignmentAggregate - - inst_name = _resolve_inst_name() - cache_key = f"align_matrix_{inst_name or 'all'}" - cached = analytics_cache.get(cache_key) - if cached: - return jsonify(cached) - - session = SessionLocal() - try: - rows = ( - session.query(AlignmentAggregate) - .filter_by(institution=inst_name or "") - .all() - ) - scored = {(r.framework, r.pillar): r for r in rows} - cells = [] - for fkey, f in FRAMEWORKS.items(): - for pkey, pdef in f["pillars"].items(): - r = scored.get((fkey, pkey)) - cells.append( - { - "framework": fkey, - "framework_name": f["name"], - "pillar": pkey, - "pillar_name": pdef["name"], - "avg_score": r.avg_score if r else 0.0, - "paper_count": r.paper_count if r else 0, - } - ) - data = { - "institution": inst_name or "All Institutions", - "frameworks": [ - {"key": k, "name": f["name"], "color": f.get("color")} - for k, f in FRAMEWORKS.items() - ], - "cells": cells, - } - resp = api_ok(data, methodology_key="alignment_score") - analytics_cache.set(cache_key, resp.get_json(), ttl=1800) - return resp - except Exception as e: - logger.error(f"alignment_matrix: {e}") - return api_error(str(e)) - finally: - session.close() - - -@app.route("/api/alignment/gaps") -def alignment_gaps(): - """Research Gap Radar: pillars below threshold, ascending by score.""" - from uraas.config.alignment_frameworks import FRAMEWORKS, GAP_THRESHOLD - from uraas.database import AlignmentAggregate - from uraas.services.narratives import narrate - - inst_name = _resolve_inst_name() - try: - threshold = float(request.args.get("threshold", GAP_THRESHOLD)) - except ValueError: - threshold = GAP_THRESHOLD - - session = SessionLocal() - try: - rows = { - (r.framework, r.pillar): r - for r in session.query(AlignmentAggregate) - .filter_by(institution=inst_name or "") - .all() - } - gaps = [] - for fkey, f in FRAMEWORKS.items(): - for pkey, pdef in f["pillars"].items(): - r = rows.get((fkey, pkey)) - avg = r.avg_score if r else 0.0 - if avg < threshold: - gaps.append( - { - "framework": fkey, - "framework_name": f["name"], - "pillar": pkey, - "pillar_name": pdef["name"], - "avg_score": avg, - "paper_count": r.paper_count if r else 0, - } - ) - gaps.sort(key=lambda g: g["avg_score"]) - worst = gaps[0] if gaps else None - narrative = ( - narrate( - "alignment_gaps", - gap_count=len(gaps), - framework_count=len({g["framework"] for g in gaps}), - worst_pillar=worst["pillar_name"], - worst_framework=worst["framework_name"], - worst_score=worst["avg_score"], - ) - if worst - else "" - ) - return api_ok( - { - "gaps": gaps, - "threshold": threshold, - "institution": inst_name or "All Institutions", - }, - narrative=narrative, - methodology_key="alignment_gap", - ) - except Exception as e: - logger.error(f"alignment_gaps: {e}") - return api_error(str(e)) - finally: - session.close() - - -@app.route("/api/alignment/export.csv") -def alignment_export_csv(): - """CSV of the full alignment matrix for the selected institution.""" - from uraas.config.alignment_frameworks import FRAMEWORKS - from uraas.database import AlignmentAggregate - - inst_name = _resolve_inst_name() - session = SessionLocal() - try: - rows = { - (r.framework, r.pillar): r - for r in session.query(AlignmentAggregate) - .filter_by(institution=inst_name or "") - .all() - } - - def generate(): - yield ["Framework", "Pillar", "Avg Score (0-100)", "Aligned Papers"] - for fkey, f in FRAMEWORKS.items(): - for pkey, pdef in f["pillars"].items(): - r = rows.get((fkey, pkey)) - yield [ - f["name"], - pdef["name"], - r.avg_score if r else 0.0, - r.paper_count if r else 0, - ] - - return csv_response(generate(), "alignment_matrix.csv") - except Exception as e: - logger.error(f"alignment_export_csv: {e}") - return api_error(str(e)) - finally: - session.close() - - -# ── Intra-African collaboration endpoints ───────────────────────────────────── - - -@app.route("/api/collaboration/overview") -def collaboration_overview(): - """Intra-African Collaboration Index + benchmark context.""" - from uraas.services.narratives import narrate - - try: - institution = request.args.get("institution", "").strip().lower() or None - data = analytics.get_collaboration_overview(institution) - partners = data.get("top_partner_countries", []) - narrative = narrate( - "intra_african" if partners else "intra_african_no_partner", - pct=data.get("intra_african_pct", 0), - ratio=data.get("ratio_vs_baseline", 0), - baseline=data.get("baseline_pct", 8.4), - top_partner=partners[0]["name"] if partners else "", - ) - return api_ok( - data, - narrative=narrative, - methodology_key="intra_african_collaboration", - ) - except Exception as e: - logger.error(f"collaboration_overview: {e}") - return api_error(str(e)) - - -@app.route("/api/collaboration/matrix") -def collaboration_matrix(): - """African country-pair co-publication matrix.""" - from uraas.services.narratives import narrate - - try: - institution = request.args.get("institution", "").strip().lower() or None - data = analytics.get_country_pair_matrix(institution) - top = data["pairs"][0] if data["pairs"] else None - narrative = ( - narrate( - "country_pairs", - pair_count=len(data["pairs"]), - country_a=top["source_name"], - country_b=top["target_name"], - count=top["count"], - ) - if top - else "" - ) - return api_ok(data, narrative=narrative, methodology_key="country_pair_matrix") - except Exception as e: - logger.error(f"collaboration_matrix: {e}") - return api_error(str(e)) - - -@app.route("/api/collaboration/countries") -def collaboration_countries(): - """Per-country paper counts for the Africa choropleth.""" - try: - institution = request.args.get("institution", "").strip().lower() or None - return api_ok( - {"countries": analytics.get_country_aggregates(institution)}, - methodology_key="intra_african_collaboration", - ) - except Exception as e: - logger.error(f"collaboration_countries: {e}") - return api_error(str(e)) - - -@app.route("/api/collaboration/arcs") -def collaboration_arcs(): - """GeoJSON great-circle arcs for the collaboration map (bare GeoJSON).""" - try: - institution = request.args.get("institution", "").strip().lower() or None - return jsonify(analytics.get_collaboration_arcs(institution)) - except Exception as e: - logger.error(f"collaboration_arcs: {e}") - return api_error(str(e)) - - -@app.route("/api/collaboration/network") -def collaboration_network(): - """Author collaboration network with Louvain communities + centrality.""" - try: - author = request.args.get("author", "").strip() or None - limit = clamped_int("limit", 30, 1, 100) - return api_ok(analytics.get_author_network(author, limit)) - except Exception as e: - logger.error(f"collaboration_network: {e}") - return api_error(str(e)) - - -@app.route("/api/citations/velocity") -def citations_velocity(): - """Repository citation velocity + Pan-African citation share.""" - from uraas.services.narratives import narrate - - try: - institution = request.args.get("institution", "").strip().lower() or None - data = analytics.get_citation_velocity(institution) - series = data.get("by_year", []) - recent = series[-1]["citations"] if series else 0 - parts = [] - if series: - parts.append( - narrate( - "citation_velocity", - recent_rate=recent, - avg_first2y=data.get("avg_first2y", 0), - ) - ) - if data.get("pan_african_share_pct") is not None: - parts.append( - narrate( - "pan_african_share", - share=data["pan_african_share_pct"], - covered=data.get("pan_african_share_items", 0), - ) - ) - return api_ok( - data, - narrative=" ".join(p for p in parts if p), - methodology_key="citation_velocity", - ) - except Exception as e: - logger.error(f"citations_velocity: {e}") - return api_error(str(e)) - - -@app.route("/api/collaboration/export.csv") -def collaboration_export_csv(): - """CSV export — ?view=matrix (default) or ?view=countries.""" - try: - institution = request.args.get("institution", "").strip().lower() or None - view = request.args.get("view", "matrix") - if view == "countries": - rows_data = analytics.get_country_aggregates(institution) - - def gen_countries(): - yield ["Country", "ISO2", "Papers", "Intra-African Papers"] - for r in rows_data: - yield [r["name"], r["code"], r["papers"], r["intra_african"]] - - return csv_response(gen_countries(), "collaboration_countries.csv") - - matrix = analytics.get_country_pair_matrix(institution) - - def gen_matrix(): - yield ["Country A", "Country B", "Co-publications"] - for p in matrix["pairs"]: - yield [p["source_name"], p["target_name"], p["count"]] - - return csv_response(gen_matrix(), "collaboration_matrix.csv") - except Exception as e: - logger.error(f"collaboration_export_csv: {e}") - return api_error(str(e)) - - -@app.route("/api/analytics/staff-directory") -def staff_directory(): - """ - Returns real staff records with name, department, faculty, ORCID for each institution. - Query params: ?institution=unilag (optional; returns all if omitted) - """ - from uraas.config.institutions import get_registry - - registry = get_registry() - institution_filter = request.args.get("institution", "").strip().lower() - - result = [] - insts = ( - [registry.get(institution_filter)] - if institution_filter - else registry.list_all() - ) - insts = [i for i in insts if i] # filter None - - for inst in insts: - # Get dynamic authors from database - dynamic_authors = analytics.get_top_authors( - limit=5000, institution=inst.short_name - ) - - # Merge dynamic ORCIDs/RORs with static departments - staff_data = [] - static_lookup = {r["name"].lower(): r for r in inst.staff_records} - - for author in dynamic_authors: - a_name = author.get("author", "") - static_rec = static_lookup.get(a_name.lower(), {}) - - staff_data.append( - { - "name": a_name, - "orcid": author.get("orcid") or static_rec.get("orcid"), - "ror": author.get("ror"), - "department": static_rec.get("department"), - "faculty": static_rec.get("faculty"), - "paper_count": author.get("count", 0), - } - ) - - result.append( - { - "institution": inst.name, - "short_name": inst.short_name, - "country": inst.country, - "staff": staff_data, - "staff_count": len(staff_data), - "staff_with_orcid": sum(1 for s in staff_data if s.get("orcid")), - "departments": inst.departments, - } - ) - - return jsonify(result) - - -# Export - - -@app.route("/api/export/papers.csv") -def export_csv(): - """Streaming CSV of every paper (yield_per avoids loading all rows).""" - from sqlalchemy.orm import selectinload - - def generate(): - session = SessionLocal() - try: - yield [ - "ID", - "Title", - "Authors", - "DOI", - "ARK", - "DocID", - "Year", - "Faculty", - "Open Access", - "Source", - ] - # selectinload (not joinedload) is required with yield_per — joined - # eager loads against collections need row-uniquing, which yield_per - # forbids. - q = ( - session.query(Item) - .options( - selectinload(Item.authors), - selectinload(Item.collections).selectinload(Collection.community), - ) - .order_by(desc(Item.created_at)) - ) - for i in q.yield_per(200): - authors = "; ".join(a.name for a in i.authors) - faculty = i.collections[0].community.name if i.collections else "" - year = i.publication_date.year if i.publication_date else "" - yield [ - i.id, - i.title or "", - authors, - i.doi or "", - i.ark or "", - i.docid or "", - year, - faculty, - "Yes" if "openAccess" in (i.dc_rights or "") else "No", - i.source_repository or "", - ] - finally: - session.close() - - return csv_response(generate(), "uraas_papers.csv") - - -@app.route("/api/export/papers.bibtex") -def export_bibtex(): - session = SessionLocal() - try: - items = session.query(Item).order_by(desc(Item.created_at)).all() - entries = [] - for i in items: - authors = [a.name for a in i.authors] - first_last = authors[0].split()[-1] if authors else "Unknown" - year = str(i.publication_date.year) if i.publication_date else "nd" - key = __import__("re").sub(r"[^a-zA-Z0-9]", "", f"{first_last}{year}") - author_str = " and ".join(authors) if authors else "Unknown" - title = (i.title or "Untitled").replace("{", "").replace("}", "") - doi_line = f" doi = {{{i.doi}}},\n" if i.doi else "" - url_line = f" url = {{{i.url}}},\n" if i.url else "" - institution = (i.institution or "").strip() or "Unknown" - entries.append( - f"@article{{{key},\n title = {{{title}}},\n author = {{{author_str}}},\n year = {{{year}}},\n institution = {{{institution}}},\n" - + doi_line - + url_line - + "}" - ) - return Response( - "\n\n".join(entries), - mimetype="text/plain", - headers={"Content-Disposition": "attachment; filename=uraas_papers.bib"}, - ) - finally: - session.close() - - -# Crawler control - - -@app.route("/api/crawler/start", methods=["POST"]) -def start_crawler(): - global crawler_process - with crawler_lock: - if crawler_process and crawler_process.poll() is None: - return ( - jsonify({"status": "error", "message": "Crawler already running"}), - 400, - ) - data = request.get_json() or {} - target = min(max(int(data.get("target", 20)), 1), 250) - institution = data.get("institution", "unilag") - # Validate institution against registry before passing to subprocess. - if institution != "all": - from uraas.config.institutions import get_registry as _get_reg - _reg = _get_reg() - if not _reg.get(institution): - return jsonify({"status": "error", "message": f"Unknown institution: {institution}"}), 400 - # Default ON — heavy bias toward Special Collections in every crawl. - boost_special = bool(data.get("boost_special", True)) - sc_only = bool(data.get("sc_only", False)) - # Optional spider selection (allowlisted). "oai" = read-only harvest of - # the institution's own repository (theses/grey literature). - spider = data.get("spider", "openalex") - _allowed_spiders = ("openalex", "crossref", "arxiv", "orcid", "oai", - "semantic_scholar", "europepmc", "core", "pubmed", - "openaire", "doaj", "ajol", "all") - if spider not in _allowed_spiders: - return jsonify({"status": "error", "message": f"Unknown spider: {spider}"}), 400 - # OAI date window — accept only a safe YYYY-MM-DD shape; ignore anything else. - _date_re = re.compile(r"^\d{4}-\d{2}-\d{2}$") - from_date = data.get("from_date") - until_date = data.get("until_date") - from_date = from_date if (from_date and _date_re.match(str(from_date))) else None - until_date = until_date if (until_date and _date_re.match(str(until_date))) else None - try: - # Derive project root and script path - project_root = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ) - script_path = os.path.join( - project_root, "scripts", "crawl_multi_institution.py" - ) - - cmd = [__import__("sys").executable, script_path, "--target", str(target)] - if institution != "all": - cmd.extend(["--institutions", institution]) - cmd.extend(["--spider", spider]) - if spider == "oai": - if from_date: - cmd.extend(["--from-date", from_date]) - if until_date: - cmd.extend(["--until-date", until_date]) - else: - if not boost_special: - cmd.append("--no-boost-special") - if sc_only: - cmd.append("--sc-only") - - logger.info(f"Executing crawler command: {' '.join(cmd)}") - - # Pass PYTHONUNBUFFERED so terminal output appears in real-time order - env = dict(os.environ, PYTHONUNBUFFERED="1") - process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - bufsize=1, - env=env, - ) - crawler_process = process - thread = threading.Thread( - target=crawler_monitor, args=(process,), daemon=True - ) - thread.start() - return jsonify( - { - "status": "success", - "message": f"Crawler started target {target} papers", - } - ) - except FileNotFoundError: - return ( - jsonify( - { - "status": "error", - "message": f"Crawler script not found at {script_path}", - } - ), - 500, - ) - except Exception as e: - logger.error("start_crawler: %s", e) - return jsonify({"status": "error", "message": str(e)}), 500 - - -@app.route("/api/crawler/stop", methods=["POST"]) -def stop_crawler(): - global crawler_process - with crawler_lock: - if crawler_process and crawler_process.poll() is None: - crawler_process.terminate() - crawler_process = None - return jsonify({"status": "success", "message": "Crawler stopped"}) - return jsonify({"status": "warning", "message": "No crawler running"}) - - -@app.route("/api/crawler/status") -def crawler_status(): - with crawler_lock: - running = crawler_process is not None and crawler_process.poll() is None - return jsonify({"status": "running" if running else "idle"}) - - -# Health Check Endpoint for Render - - -@app.route("/health") -def health_check(): - """Render readiness probe (Phase 8 enhanced). - - Checks: DB connectivity + latency, disk space, analytics cache, methodology. - Returns 200 if all checks pass, 503 if any critical check fails. - """ - import time - from datetime import datetime - - from sqlalchemy import text - from uraas.config.methodology import METHODOLOGY - - t_start = time.monotonic() - health_status = { - "status": "healthy", - "timestamp": datetime.utcnow().isoformat(), - "version": os.getenv("RENDER_GIT_COMMIT", "dev")[:8], - "checks": {}, - } - - # DB check (with latency) - t0 = time.monotonic() - try: - session = SessionLocal() - session.execute(text("SELECT 1")) - session.close() - health_status["checks"]["database"] = { - "status": "ok", - "latency_ms": round((time.monotonic() - t0) * 1000, 1), - } - except Exception as e: - health_status["checks"]["database"] = {"status": f"error: {str(e)}"} - health_status["status"] = "unhealthy" - logger.error(f"Database health check failed: {str(e)}") - - # Analytics cache probe - try: - analytics_cache.get("__health__") - health_status["checks"]["cache"] = {"status": "ok"} - except Exception as e: - health_status["checks"]["cache"] = {"status": f"error: {str(e)}"} - - # Methodology sanity - health_status["checks"]["methodology"] = { - "status": "ok", - "metric_count": len(METHODOLOGY), - } - - # Disk space check on persistent volume - try: - storage_path = config.STORAGE_PATH - if os.path.exists(storage_path): - import shutil - stat = shutil.disk_usage(storage_path) - free_gb = stat.free / (1024**3) - health_status["checks"]["disk_space_gb"] = round(free_gb, 2) - if free_gb < 1: - health_status["status"] = "unhealthy" - health_status["checks"]["disk_space"] = "critical" - else: - health_status["checks"]["disk_space"] = "storage path not found" - except Exception as e: - health_status["checks"]["disk_space"] = f"error: {str(e)}" - - health_status["response_ms"] = round((time.monotonic() - t_start) * 1000, 1) - status_code = 200 if health_status["status"] == "healthy" else 503 - return jsonify(health_status), status_code - - -@app.route("/api/university-registry", methods=["GET"]) -def get_university_registry(): - """ - Get the comprehensive 52-country African university registry. - """ - try: - import json - - registry_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "data", - "university_registry.json", - ) - if not os.path.exists(registry_path): - registry_path = "data/university_registry.json" - - with open(registry_path, "r", encoding="utf-8") as f: - data = json.load(f) - return jsonify(data) - except Exception as e: - logger.error(f"get_university_registry: {e}") - return jsonify({"error": str(e)}), 500 - - -@app.route("/api/reports/unilag-subregion", methods=["GET"]) -def get_unilag_report(): - """ - Generate draft report of UNILAG contributions to African languages in West Africa. - """ - session = SessionLocal() - try: - import json - from datetime import datetime - - from uraas.utils.ai_classifier import AU_CHARTER_TARGETS, classify_au_targets - - # UNILAG ROR - unilag_ror = "https://ror.org/05rk03822" - - # 1. Fetch UNILAG items - unilag_items = session.query(Item).filter(Item.ror == unilag_ror).all() - total_unilag = len(unilag_items) - - # 2. Fetch West Africa items (excluding UNILAG, e.g. UI and Covenant) - west_africa_rors = [ - "https://ror.org/01js2sh04", - "https://ror.org/0545s4788", - ] # UI and Covenant - wa_items = session.query(Item).filter(Item.ror.in_(west_africa_rors)).all() - total_wa = len(wa_items) - - # 3. Analyze UNILAG papers against AU Charter Target 2 (African Languages) - target2_compliant = 0 - keywords_found = set() - by_year = {} - - for item in unilag_items: - results = classify_au_targets( - item.title or "", item.abstract or "", item.dc_subject or "" - ) - for r in results: - if r["target_number"] == 2: - target2_compliant += 1 - keywords_found.update(r["matched_keywords"]) - year = item.publication_date.year if item.publication_date else None - if year: - by_year[year] = by_year.get(year, 0) + 1 - - # Determine gaps - all_t2_keywords = AU_CHARTER_TARGETS[2]["keywords"] - keywords_gap = [kw for kw in all_t2_keywords if kw not in keywords_found] - - # 4. Compare with West African average - wa_target2_compliant = 0 - for item in wa_items: - results = classify_au_targets( - item.title or "", item.abstract or "", item.dc_subject or "" - ) - for r in results: - if r["target_number"] == 2: - wa_target2_compliant += 1 - - unilag_compliance_rate = ( - round(target2_compliant / total_unilag * 100, 1) if total_unilag else 0.0 - ) - wa_compliance_rate = ( - round(wa_target2_compliant / total_wa * 100, 1) if total_wa else 0.0 - ) - - report_data = { - "title": "Decolonizing Knowledge: UNILAG Contributions to African Languages & Cultural Renaissance in West Africa", - "metadata": { - "institution": "University of Lagos (UNILAG)", - "subregion": "West Africa", - "generated_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), - }, - "introduction": ( - "This report evaluates the academic contributions of the University of Lagos (UNILAG) " - "toward the development of African languages and the decolonization of science in the West African sub-region. " - "Aligned with the African Union Charter for African Cultural Renaissance, specifically Target 2 (Development of African Languages), " - "this analysis highlights the intersection of linguistic preservation, local knowledge systems, and active " - "institutional engagement in decolonial research." - ), - "statistics": { - "total_curated": total_unilag, - "compliant_count": target2_compliant, - "compliance_rate": unilag_compliance_rate, - "gaps_count": total_unilag - target2_compliant, - "keywords_found": list(keywords_found), - "keywords_gap": keywords_gap, - }, - "scores_and_trends": { - "timeline": [ - {"year": y, "count": by_year[y]} for y in sorted(by_year.keys()) - ], - "comparison": { - "unilag_rate": unilag_compliance_rate, - "west_africa_rate": wa_compliance_rate, - "unilag_compliant": target2_compliant, - "west_africa_compliant": wa_target2_compliant, - }, - }, - "conclusion": ( - f"UNILAG shows solid alignment with the African Union Charter targets, with a decolonial compliance rate of {unilag_compliance_rate}%. " - f"Linguistic preservation is robust, particularly with keywords like '{', '.join(list(keywords_found)[:4])}' being highly active. " - f"However, critical gaps remain in the development of scientific literature in local languages. " - f"To bridge this gap, future research should focus on areas like '{', '.join(keywords_gap[:4])}' to ensure a more comprehensive " - "contribution to the African Union's Renaissance targets." - ), - } - return jsonify(report_data) - except Exception as e: - logger.error(f"get_unilag_report: {e}") - return jsonify({"error": str(e)}), 500 - finally: - session.close() - - -# ── Live IR connection + batch deposit ─────────────────────────────────────── -# All write endpoints are admin-only (ADMIN_ENDPOINTS list at the top of this -# file gates them automatically). The approve/reject token endpoints are -# intentionally PUBLIC — the token itself is the credential. - -ADMIN_ENDPOINTS.update({ - "ir_status", - "ir_collections", - "ir_live_stats", - "ir_queue_batch", - "ir_list_batches", - "ir_batch_status", - "ir_test_harvest", -}) - - -@app.route("/api/ir/test-harvest", methods=["POST"]) -def ir_test_harvest(): - """Dry-run OAI harvest: collect SC papers, send preview email, save JSON. - - Body JSON: - institution – short name (default: "unilag") - count – max SC papers to collect (default: 50, max: 100) - email – confirmation address (required) - from_date – OAI from date YYYY-MM-DD (optional) - - DOES NOT save to DB. DOES NOT deposit to IR. - Runs in a background thread; returns immediately with a job ID. - """ - data = request.get_json(silent=True) or {} - institution = (data.get("institution") or "unilag").strip().lower() - count = min(max(int(data.get("count") or 50), 1), 100) - email = (data.get("email") or "").strip() - from_date = (data.get("from_date") or "").strip() or None - - if not email or "@" not in email: - return jsonify({"status": "error", "message": "A valid email address is required"}), 400 - - from uraas.config.institutions import get_registry as _get_reg - _reg = _get_reg() - inst_cfg = _reg.get(institution) - if not inst_cfg: - return jsonify({"status": "error", "message": f"Unknown institution: {institution}"}), 400 - if not inst_cfg.oai_endpoint: - return jsonify({"status": "error", "message": f"'{institution}' has no OAI endpoint configured"}), 400 - - import subprocess, sys as _sys - script_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "scripts", "test_harvest_50.py", - ) - cmd = [_sys.executable, script_path, - "--institution", institution, - "--count", str(count), - "--email", email] - if from_date: - cmd.extend(["--from-date", from_date]) - - try: - env = dict(os.environ, PYTHONUNBUFFERED="1") - process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - bufsize=1, - env=env, - ) - # Stream output via existing SocketIO terminal - def _monitor(): - for line in iter(process.stdout.readline, b""): - line_text = line.decode("utf-8", errors="replace").strip() - if line_text: - socketio.emit("terminal_output", {"line": line_text}) - process.wait() - socketio.emit("terminal_output", {"line": "[TEST HARVEST] Complete."}) - threading.Thread(target=_monitor, daemon=True).start() - - return jsonify({ - "status": "started", - "message": f"Dry-run harvest started for {inst_cfg.name}. Preview will be emailed to {email}.", - "institution": inst_cfg.name, - "count": count, - "email": email, - }), 202 - except Exception as exc: - logger.error("ir_test_harvest: %s", exc) - return jsonify({"status": "error", "message": str(exc)}), 500 - - -@app.route("/api/ir/status") -def ir_status(): - """Check connectivity to the live DSpace IR (no credentials needed for read).""" - from uraas.services.ir_client import DSpaceClient - client = DSpaceClient() - result = client.probe() - return jsonify(result), 200 if result.get("ok") else 503 - - -@app.route("/api/ir/collections") -def ir_collections(): - """List DSpace collections the configured account can submit to.""" - from uraas.services.ir_client import DSpaceClient - try: - client = DSpaceClient() - client.login() - cols = client.get_submittable_collections() - return jsonify({"status": "success", "collections": cols}) - except Exception as exc: - logger.error("ir_collections: %s", exc) - return jsonify({"status": "error", "message": str(exc)}), 500 - - -@app.route("/api/ir/live-stats") -def ir_live_stats(): - """Composite live stats tile pulled directly from the UNILAG DSpace IR.""" - from uraas.services.ir_client import DSpaceClient - try: - client = DSpaceClient() - stats = client.get_live_stats() - return jsonify({"status": "success", "data": stats}) - except Exception as exc: - logger.error("ir_live_stats: %s", exc) - return jsonify({"status": "error", "message": str(exc)}), 500 - - -@app.route("/api/ir/deposit/queue", methods=["POST"]) -def ir_queue_batch(): - """Queue a batch of local items for deposit to the real IR. - - Body JSON: - item_ids – list of local Item.id values to deposit - collection_uuid – DSpace collection UUID (from /api/ir/collections) - collection_name – display name (optional, for the email) - approval_email – address to send the approve/reject email to - """ - from uraas.services.batch_approval import queue_batch - - data = request.get_json(silent=True) or {} - item_ids = data.get("item_ids", []) - collection_uuid = (data.get("collection_uuid") or "").strip() - collection_name = (data.get("collection_name") or "").strip() - approval_email = (data.get("approval_email") or "").strip() - requested_by = session.get("user", "admin") - - if not item_ids: - return jsonify({"status": "error", "message": "item_ids required"}), 400 - if not isinstance(item_ids, list) or not all(isinstance(i, int) for i in item_ids): - return jsonify({"status": "error", "message": "item_ids must be a list of integers"}), 400 - if len(item_ids) > 500: - return jsonify({"status": "error", "message": "Maximum 500 items per batch"}), 400 - if not approval_email or "@" not in approval_email: - return jsonify({"status": "error", "message": "A valid approval_email is required"}), 400 - if not collection_uuid: - return jsonify({"status": "error", "message": "collection_uuid required"}), 400 - - try: - result = queue_batch( - item_ids=item_ids, - collection_uuid=collection_uuid, - collection_name=collection_name, - approval_email=approval_email, - requested_by=requested_by, - ) - return jsonify(result), 201 - except Exception as exc: - logger.error("ir_queue_batch: %s", exc) - return jsonify({"status": "error", "message": str(exc)}), 500 - - -@app.route("/api/ir/batch//approve", methods=["GET"]) -def ir_approve_batch(token): - """Email approval link — no login required; token is the credential. - - Renders a plain HTML confirmation page so it works directly in a browser - after the approver clicks the link in their email. - """ - from uraas.services.batch_approval import approve_batch - - # Minimal token sanity-check (URL-safe base64 chars only) - import re as _re - if not token or not _re.match(r'^[A-Za-z0-9_\-]{10,128}$', token): - return _approval_html("Invalid Link", "The approval link is malformed.", ok=False), 400 - - result = approve_batch(token) - - if result["status"] == "approved": - return _approval_html( - "Deposit Approved", - result["message"], - ok=True, - ), 200 - elif result["status"] == "already_actioned": - return _approval_html( - "Already Actioned", - result["message"], - ok=True, - ), 200 - elif result["status"] == "expired": - return _approval_html("Link Expired", result["message"], ok=False), 410 - else: - return _approval_html("Not Found", result["message"], ok=False), 404 - - -@app.route("/api/ir/batch//reject", methods=["GET"]) -def ir_reject_batch(token): - """Email rejection link — no login required; token is the credential.""" - from uraas.services.batch_approval import reject_batch - - import re as _re - if not token or not _re.match(r'^[A-Za-z0-9_\-]{10,128}$', token): - return _approval_html("Invalid Link", "The rejection link is malformed.", ok=False), 400 - - reason = request.args.get("reason", "Rejected via email link") - result = reject_batch(token, reason=reason) - - if result["status"] == "rejected": - return _approval_html( - "Batch Rejected", - result["message"], - ok=False, - ), 200 - elif result["status"] == "already_actioned": - return _approval_html("Already Actioned", result["message"], ok=True), 200 - else: - return _approval_html("Not Found", result["message"], ok=False), 404 - - -@app.route("/api/ir/batches") -def ir_list_batches(): - """List all deposit batches (admin panel).""" - from uraas.services.batch_approval import get_batches - try: - limit = min(int(request.args.get("limit", 50)), 200) - batches = get_batches(limit=limit) - return jsonify({"status": "success", "batches": batches}) - except Exception as exc: - logger.error("ir_list_batches: %s", exc) - return jsonify({"status": "error", "message": str(exc)}), 500 - - -@app.route("/api/ir/batch//status") -def ir_batch_status(token): - """Get status of a specific batch (admin polling).""" - from uraas.services.batch_approval import get_batch - import re as _re - if not token or not _re.match(r'^[A-Za-z0-9_\-]{10,128}$', token): - return jsonify({"status": "error", "message": "Invalid token"}), 400 - batch = get_batch(token) - if not batch: - return jsonify({"status": "error", "message": "Batch not found"}), 404 - return jsonify({"status": "success", "batch": batch}) - - -def _approval_html(title: str, message: str, ok: bool) -> str: - colour = "#1a7a4a" if ok else "#c0392b" - icon = "✓" if ok else "✗" - return f""" - - - - - URAAS — {title} - - - -
-
{icon}
-

{title}

-

{message}

- Return to Dashboard -
- -""" - - -# Error Handlers - - -@app.errorhandler(404) -def not_found_error(error): - """Custom 404 error handler.""" - if request.path.startswith("/api/"): - return jsonify({"error": "Resource not found"}), 404 - return render_template("index.html"), 404 # SPA fallback - - -@app.errorhandler(500) -def internal_error(error): - """Custom 500 error handler.""" - logger.error(f"Internal server error: {str(error)}") - if request.path.startswith("/api/"): - return jsonify({"error": "Internal server error"}), 500 - return jsonify({"error": "Internal server error"}), 500 - - -# Phase 8: /api/version endpoint - - -@app.route("/api/version") -def api_version(): - """Version manifest — commit hash + phase badges for the dashboard UI.""" - return jsonify( - { - "version": os.getenv("RENDER_GIT_COMMIT", "dev")[:8], - "env": "production" if config.is_production() else "development", - "phases_completed": [ - "P4_citation_velocity", - "P5_ark_pids", - "P6_credibility", - "P7_cleanup", - "P8_render_prep", - ], - } - ) - - - -# Run - -if __name__ == "__main__": - # Apply production configuration if on Render - from uraas.production_config import ProductionConfig - - ProductionConfig.apply_config(app) - - # Get port from environment (Render provides this) - port = int(os.getenv("PORT", config.DASHBOARD_PORT)) - - # Determine if running in production - is_production = ProductionConfig.is_production() - - if is_production: - logger.info("=" * 70) - logger.info("URAAS Dashboard Starting (Production Mode)") - logger.info("=" * 70) - logger.info(f"Port: {port}") - logger.info(f"Database: {os.getenv('DATABASE_URL', 'Not configured')[:50]}...") - logger.info(f"Storage: {config.STORAGE_PATH}") - logger.info(f"Health check: http://0.0.0.0:{port}/health") - logger.info("=" * 70) - else: - logger.info("=" * 70) - logger.info("URAAS Dashboard Starting (Development Mode)") - logger.info("=" * 70) - logger.info(f"Dashboard URL: http://localhost:{port}") - logger.info("Press Ctrl+C to stop") - logger.info("=" * 70) - - # Run with SocketIO - socketio.run( - app, - host="0.0.0.0", - port=port, - debug=not is_production, - use_reloader=not is_production, - ) +import csv +import io +import json +import logging +import os +import re +import subprocess +import threading + +from flask import ( + Flask, + Response, + jsonify, + redirect, + render_template, + request, + send_file, + session, + url_for, +) +from flask_socketio import SocketIO +from sqlalchemy import desc, extract, func, or_ +from sqlalchemy.orm import selectinload + +from uraas.analytics.engine import analytics +from uraas.config import config +from uraas.database import ( + Author, + Collection, + Community, + File, + Item, + SessionLocal, + db_year, + db_year_month, +) +from uraas.dashboard.auth import ( + ADMIN, + check_credentials, + clamped_int, + current_role, +) +from flask_limiter import Limiter +from flask_limiter.util import get_remote_address +from uraas.dashboard.responses import api_error, api_ok, csv_response +from uraas.production_config import ProductionConfig +from uraas.utils.analytics_cache import analytics_cache + +# Fail fast on insecure production config before the app even binds. +config.validate() + +app = Flask(__name__) +app.config["SECRET_KEY"] = config.DASHBOARD_SECRET_KEY +# Session-cookie hardening applies in every environment (SECURE only where TLS +# is present, i.e. production) so it is not Render-specific. +app.config.update( + SESSION_COOKIE_HTTPONLY=True, + SESSION_COOKIE_SAMESITE="Lax", + SESSION_COOKIE_SECURE=config.is_production(), +) +# Apply production hardening at import time so it also runs under gunicorn +# (the __main__ block below never executes in a WSGI deployment). +ProductionConfig.apply_config(app) + +# SocketIO: restrict the handshake to an explicit origin allowlist (no wildcard). +socketio = SocketIO( + app, + cors_allowed_origins=config.cors_origins(), + async_mode="threading", +) +logger = logging.getLogger(__name__) +crawler_process = None +crawler_lock = threading.Lock() + +# Rate limiter — in-memory storage (no Redis dep). Limits login to 10 attempts +# per minute to prevent brute-force attacks on admin/viewer credentials. +limiter = Limiter( + key_func=get_remote_address, + app=app, + default_limits=[], # No global limit; only login is rate-limited. + storage_uri="memory://", +) + +# ── Access control (fail-closed) ─────────────────────────────────────────── +# Everything is gated by default. Endpoints are authorised by *endpoint name* +# (function name) so path params don't matter and any NEW route is protected +# until explicitly listed here. +# +# PUBLIC_ENDPOINTS — reachable without a session (login page, health, static). +# ADMIN_ENDPOINTS — require role == admin (crawler, mutations, bulk exports, +# staff directory PII). Everything else needs any login. +PUBLIC_ENDPOINTS = {"login", "logout", "health_check", "api_version", "static"} +ADMIN_ENDPOINTS = { + "start_crawler", + "stop_crawler", + "crawler_status", + "flush_analytics_cache", + "prune_non_sc", + "test_smtp", + "recompute_alignment", + "update_citations", + "bulk_update_citations", + "export_csv", + "export_bibtex", + "export_special_collections_csv", + "alignment_export_csv", + "collaboration_export_csv", + "citations_velocity_csv", + "staff_directory", +} + + +@app.before_request +def _enforce_authentication(): + endpoint = request.endpoint + # Unknown endpoint (404s) and explicit public routes pass through. + if endpoint is None or endpoint in PUBLIC_ENDPOINTS: + return None + role = current_role() + if not role: + if request.path.startswith("/api/"): + return jsonify({"status": "error", "message": "Authentication required"}), 401 + return redirect(url_for("login", next=request.path)) + if endpoint in ADMIN_ENDPOINTS and role != ADMIN: + return jsonify({"status": "error", "message": "Administrator access required"}), 403 + return None + + +def crawler_monitor(process): + global crawler_process + try: + for line in iter(process.stdout.readline, b""): + with crawler_lock: + if crawler_process is None or crawler_process != process: + break + line_decoded = line.decode("utf-8", errors="replace").strip() + if not line_decoded: + continue + if line_decoded.startswith("[INIT]"): + socketio.emit( + "crawl_status", {"status": "initializing", "message": line_decoded} + ) + socketio.emit("terminal_output", {"line": line_decoded}) + elif "URAAS_DOWNLOAD:" in line_decoded: + socketio.emit("crawl_status", {"status": "running"}) + try: + title = line_decoded.split("URAAS_DOWNLOAD:", 1)[-1].strip() + socketio.emit("crawl_progress", {"title": title}) + except Exception: + pass + socketio.emit("terminal_output", {"line": line_decoded}) + else: + socketio.emit("terminal_output", {"line": line_decoded}) + except Exception: + pass + finally: + try: + process.stdout.close() + except Exception: + pass + process.wait() + with crawler_lock: + if crawler_process == process: + crawler_process = None + analytics_cache.invalidate_all() # Flush stale analytics after crawl + socketio.emit("crawl_status", {"status": "stopped"}) + + +@app.context_processor +def inject_asset_version(): + """Cache-bust static assets by their file mtime so browsers always load + the latest JS/CSS after a deploy or edit (no more stale cached app.js).""" + + def asset_version(filename): + try: + path = os.path.join(app.static_folder, filename) + return str(int(os.path.getmtime(path))) + except OSError: + return "1" + + return {"asset_version": asset_version} + + +@app.route("/") +def index(): + return render_template("index.html") + + +@app.route("/login", methods=["GET", "POST"]) +@limiter.limit("10 per minute", methods=["POST"], error_message="Too many login attempts — wait 60 seconds.") +def login(): + if request.method == "POST": + username = (request.form.get("username") or "").strip() + password = request.form.get("password") or "" + role = check_credentials(username, password) + if role: + session.clear() + session["role"] = role + session["user"] = username + session.permanent = True + dest = request.args.get("next") or url_for("index") + # Prevent open redirect: reject any URL with a scheme or netloc + # (this blocks both http://evil.com AND //evil.com style redirects). + from urllib.parse import urlparse as _urlparse + _parsed = _urlparse(dest) + if _parsed.scheme or _parsed.netloc or not dest.startswith("/"): + dest = url_for("index") + return redirect(dest) + # Generic message — no user enumeration. + return render_template("login.html", error="Invalid username or password"), 401 + if current_role(): + return redirect(url_for("index")) + return render_template("login.html", error=None) + + +@app.route("/logout") +def logout(): + session.clear() + return redirect(url_for("login")) + + +@app.after_request +def set_security_headers(response): + """Defense-in-depth headers. CSP allows only the CDNs the dashboard uses.""" + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["X-Frame-Options"] = "DENY" + response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" + response.headers["Content-Security-Policy"] = ( + "default-src 'self'; " + "script-src 'self' 'unsafe-inline' 'unsafe-eval' " + "https://cdn.jsdelivr.net https://cdnjs.cloudflare.com " + "https://unpkg.com https://cdn.tailwindcss.com; " + "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net " + "https://cdnjs.cloudflare.com https://fonts.googleapis.com https://unpkg.com; " + "font-src 'self' https://fonts.gstatic.com https://cdnjs.cloudflare.com data:; " + "img-src 'self' data: blob: https:; " + "connect-src 'self' https://api.openalex.org https://api.crossref.org " + "https://ror.org https://*.basemaps.cartocdn.com; " + "worker-src 'self' blob:; " + "frame-ancestors 'none'" + ) + if config.is_production(): + response.headers["Strict-Transport-Security"] = ( + "max-age=31536000; includeSubDomains" + ) + return response + + +@socketio.on("connect") +def _socket_auth(): + """Reject WebSocket connections that are not from a logged-in session, + so the crawler event stream can't be driven anonymously.""" + if not session.get("role"): + return False + return True + + +@app.route("/ark://") +def resolve_ark(naan, name): + """Local ARK resolver (ARK spec: resolver base + '/' + ark). + + Redirects to the dashboard with the paper modal auto-opened; the ARK + 'inflection' suffix '?' / '?info' returns the metadata record as JSON.""" + from flask import redirect + + ark = f"ark:/{naan}/{name}" + session = SessionLocal() + try: + item = session.query(Item).filter_by(ark=ark).first() + if not item: + return jsonify({"error": "ARK not found", "ark": ark}), 404 + # ARK inflection: '?info' / '?json' returns the metadata record. (A bare + # '?' is the spec's brief-metadata inflection, but Flask's full_path + # always appends '?', so it can't be told apart from a plain resolve — + # we require the explicit suffix.) + wants_info = "info" in request.args or "json" in request.args + if wants_info: + return jsonify( + { + "ark": ark, + "docid": item.docid or "", + "doi": item.doi or "", + "title": item.title or "", + "institution": item.institution or "", + "publication_date": ( + item.publication_date.isoformat() + if item.publication_date + else None + ), + "authors": [a.name for a in item.authors], + "resolver": "URAAS / Africa PID Alliance", + } + ) + return redirect(f"/?paper={item.id}") + except Exception as e: + logger.error("resolve_ark %s: %s", ark, e) + return jsonify({"error": "Internal server error"}), 500 + finally: + session.close() + + +@app.route("/api/methodology") +def get_methodology(): + """Full per-metric methodology dictionary — feeds the ⓘ tooltips. + + Every metric on the dashboard documents its formula, data source and + caveats here so each number can be audited (open-methodology principle).""" + from uraas.config.methodology import METHODOLOGY + + return jsonify({"status": "success", "data": METHODOLOGY}) + + +@app.route("/api/stats") +def get_stats(): + try: + return jsonify( + { + "status": "success", + "top_authors": analytics.get_top_authors(limit=5), + "network_edges": analytics.get_department_collaboration_network(), + } + ) + except Exception as e: + logger.error("get_stats: %s", e) + return jsonify({"status": "error", "top_authors": [], "network_edges": []}), 500 + + +@app.route("/api/papers/tree") +def papers_tree(): + try: + institution = request.args.get("institution", None) + return jsonify( + { + "status": "success", + "data": analytics.get_papers_by_faculty_and_department( + institution=institution + ), + } + ) + except Exception as e: + logger.error("papers_tree: %s", e) + return jsonify({"status": "error", "data": []}), 500 + + +@app.route("/api/papers/") +def get_paper(item_id): + session = SessionLocal() + try: + item = session.query(Item).filter_by(id=item_id).first() + if not item: + return jsonify({"error": "Paper not found"}), 404 + file_record = session.query(File).filter_by(item_id=item_id).first() + collections = [ + { + "id": c.id, + "name": c.name, + "faculty": c.community.name if c.community else "Unknown", + } + for c in item.collections + ] + return jsonify( + { + "id": item.id, + "docid": item.docid or "", + "ark": item.ark or "", + "ark_url": f"/{item.ark}" if item.ark else "", + "cited_by_count": item.cited_by_count or 0, + "african_citation_share": item.african_citation_share, + "counts_by_year": ( + json.loads(item.counts_by_year) if item.counts_by_year else [] + ), + "coauthor_countries": item.coauthor_countries or "", + "is_intra_african": bool(item.is_intra_african), + "title": item.title or "Untitled", + "abstract": item.abstract or "", + "doi": item.doi or "", + "url": item.url or "", + "pdf_url": item.pdf_url or "", + "publication_date": ( + item.publication_date.isoformat() if item.publication_date else None + ), + "source_repository": item.source_repository or "", + "authors": [{"name": a.name} for a in item.authors], + "collections": collections, + "dc": { + "title": item.dc_title or "", + "date_issued": item.dc_date_issued or "", + "identifier_uri": item.dc_identifier_uri or "", + "identifier_doi": item.dc_identifier_doi or "", + "description_provenance": item.dc_description_provenance or "", + "rights": item.dc_rights or "", + }, + "file": ( + { + "has_local_pdf": file_record is not None, + "access_policy": ( + file_record.access_policy if file_record else None + ), + "download_url": ( + f"/api/papers/{item_id}/download" if file_record else None + ), + "sha256": file_record.sha256_hash if file_record else None, + } + if file_record + else {"has_local_pdf": False} + ), + "created_at": item.created_at.isoformat() if item.created_at else None, + } + ) + except Exception as e: + logger.error("get_paper %s: %s", item_id, e) + return jsonify({"error": "Internal server error"}), 500 + finally: + session.close() + + +def _is_open_access(item, file_record) -> bool: + """Open-access when the item's rights say so OR the file is policy-Public.""" + rights = (item.dc_rights or "").lower() if item else "" + if "openaccess" in rights.replace("/", "").replace("-", ""): + return True + if file_record and (file_record.access_policy or "").lower() == "public": + return True + return False + + +@app.route("/api/papers//download") +def download_paper(item_id): + session = SessionLocal() + try: + file_record = session.query(File).filter_by(item_id=item_id).first() + if not file_record: + return jsonify({"error": "PDF not found"}), 404 + item = session.query(Item).filter_by(id=item_id).first() + + # Copyright gate: viewers may only download verified open-access files; + # restricted items are admin-only (see PRIVACY_NOTICE / readiness doc). + if not _is_open_access(item, file_record) and current_role() != ADMIN: + return ( + jsonify( + { + "error": "This item is not open access.", + "message": ( + "Full text is restricted by the publisher's licence. " + "Use the publisher or open-access link on the record." + ), + } + ), + 403, + ) + + # Resolve and contain the path under the project/storage root so a stored + # path can never escape via traversal into arbitrary files. + project_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + file_path = file_record.file_path + if not os.path.isabs(file_path): + file_path = os.path.join(project_root, file_path) + real_path = os.path.realpath(file_path) + allowed_roots = [ + os.path.realpath(project_root), + os.path.realpath(config.STORAGE_PATH), + ] + if not any( + os.path.commonpath([real_path, root]) == root for root in allowed_roots + ): + logger.warning("download_paper %s: path escape blocked: %s", item_id, real_path) + return jsonify({"error": "Access denied"}), 403 + if not os.path.exists(real_path): + return jsonify({"error": "PDF file missing from storage"}), 404 + + filename = ( + f"{item.title[:50]}.pdf" if item and item.title else f"paper_{item_id}.pdf" + ) + filename = "".join( + c for c in filename if c.isalnum() or c in (" ", "-", "_", ".") + ).strip() + return send_file( + real_path, + mimetype="application/pdf", + as_attachment=True, + download_name=filename, + ) + except Exception as e: + logger.error("download_paper %s: %s", item_id, e) + return jsonify({"error": "Failed to download PDF"}), 500 + finally: + session.close() + + +@app.route("/api/papers//bibtex") +def export_single_bibtex(item_id): + session = SessionLocal() + try: + item = session.query(Item).filter_by(id=item_id).first() + if not item: + return jsonify({"error": "Paper not found"}), 404 + authors = [a.name for a in item.authors] + first_last = authors[0].split()[-1] if authors else "Unknown" + year = str(item.publication_date.year) if item.publication_date else "nd" + key = re.sub(r"[^a-zA-Z0-9]", "", f"{first_last}{year}") + author_str = " and ".join(authors) if authors else "Unknown" + title = (item.title or "Untitled").replace("{", "").replace("}", "") + doi_line = f" doi = {{{item.doi}}},\n" if item.doi else "" + url_line = f" url = {{{item.url}}},\n" if item.url else "" + # Persistent identifiers — ARK is resolvable even without a DOI. + note_bits = [] + if item.ark: + note_bits.append(f"ARK: {item.ark}") + if item.docid: + note_bits.append(f"DocID: {item.docid}") + note_line = f" note = {{{'; '.join(note_bits)}}},\n" if note_bits else "" + institution = (item.institution or "").strip() or "Unknown" + bibtex = ( + f"@article{{{key},\n title = {{{title}}},\n author = {{{author_str}}},\n year = {{{year}}},\n institution = {{{institution}}},\n" + + doi_line + + url_line + + note_line + + "}" + ) + return Response( + bibtex, + mimetype="text/plain", + headers={ + "Content-Disposition": f"attachment; filename=paper_{item_id}.bib" + }, + ) + except Exception as e: + logger.error("bibtex %s: %s", item_id, e) + return jsonify({"error": "Export failed"}), 500 + finally: + session.close() + + +@app.route("/api/analytics/overview") +def analytics_overview(): + session = SessionLocal() + institution = request.args.get("institution") + try: + q_item = session.query(Item) + if institution and institution.lower() != "all": + q_item = q_item.filter(Item.institution == institution) + + total = q_item.count() + if total == 0: + return jsonify( + { + "total_papers": 0, + "total_authors": 0, + "total_faculties": 0, + "open_access_papers": 0, + "papers_with_local_pdf": 0, + "oa_percentage": 0, + } + ) + + item_ids_query = q_item.with_entities(Item.id) + + q_author = session.query(Author).join(Author.items).filter(Item.id.in_(item_ids_query)) + q_comm = session.query(Community).filter( + Community.collections.any(Collection.items.any(Item.id.in_(item_ids_query))) + ) + q_file = session.query(File).filter(File.item_id.in_(item_ids_query)) + + authors = q_author.distinct().count() + faculties = q_comm.distinct().count() + oa = q_item.filter(Item.dc_rights.like("%openAccess%")).count() + pdfs = q_file.count() + + return jsonify( + { + "total_papers": total, + "total_authors": authors, + "total_faculties": faculties, + "open_access_papers": oa, + "papers_with_local_pdf": pdfs, + "oa_percentage": round((oa / total * 100) if total else 0, 1), + } + ) + except Exception as e: + logger.error("analytics_overview: %s", e) + return ( + jsonify( + { + "total_papers": 0, + "total_authors": 0, + "total_faculties": 0, + "open_access_papers": 0, + "papers_with_local_pdf": 0, + "oa_percentage": 0, + } + ), + 500, + ) + finally: + session.close() + + +@app.route("/api/analytics/publications-by-year") +def publications_by_year(): + institution = request.args.get("institution") + return jsonify(analytics.get_publications_by_year(institution=institution)) + + +@app.route("/api/analytics/papers-by-faculty") +def papers_by_faculty(): + institution = request.args.get("institution") + return jsonify(analytics.get_papers_by_faculty(institution=institution)) + + +@app.route("/api/analytics/top-authors") +def top_authors_analytics(): + limit = clamped_int("limit", 15, 1, 200) + institution = request.args.get("institution") + return jsonify( + analytics.get_authors_by_papers(limit=limit, institution=institution) + ) + + +@app.route("/api/analytics/open-access-breakdown") +def oa_breakdown(): + institution = request.args.get("institution") + return jsonify(analytics.get_open_access_breakdown(institution=institution)) + + +@app.route("/api/analytics/recent-papers") +def recent_papers(): + limit = clamped_int("limit", 10, 1, 50) + session = SessionLocal() + try: + items = session.query(Item).order_by(desc(Item.created_at)).limit(limit).all() + return jsonify( + [ + { + "id": i.id, + "title": i.title, + "doi": i.doi, + "authors": [a.name for a in i.authors[:3]], + "created_at": i.created_at.isoformat() if i.created_at else None, + "is_oa": "openAccess" in (i.dc_rights or ""), + } + for i in items + ] + ) + finally: + session.close() + + +@app.route("/api/analytics/growth-rate") +def growth_rate(): + institution = request.args.get("institution") + # Backward compatibility with JS which expects 'session' instead of 'month' + data = analytics.get_institutional_growth(institution=institution) + return jsonify([{"session": d["month"], "count": d["count"]} for d in data]) + + +@app.route("/api/analytics/timeline") +def timeline(): + institution = request.args.get("institution") + return jsonify(analytics.get_timeline_data(institution=institution)) + + +@app.route("/api/analytics/papers-by-year-faculty") +def papers_by_year_faculty(): + session = SessionLocal() + try: + rows = ( + session.query( + db_year(Item.publication_date).label("year"), + Community.name.label("faculty"), + func.count(Item.id).label("count"), + ) + .join(Item.collections) + .join(Collection.community) + .filter(Item.publication_date.isnot(None)) + .group_by("year", Community.name) + .order_by("year") + .all() + ) + return jsonify( + [ + {"year": int(r.year), "faculty": r.faculty, "count": r.count} + for r in rows + if r.year + ] + ) + finally: + session.close() + + +@app.route("/api/analytics/faculty-oa-breakdown") +def faculty_oa_breakdown(): + """Phase 7 fix: group-by aggregate, no per-faculty item loads.""" + institution = request.args.get("institution") + session = SessionLocal() + try: + # Two aggregate queries instead of N item fetches + base_q = ( + session.query( + Community.name, + func.count(Item.id).label("total"), + ) + .join(Item.collections) + .join(Collection.community) + ) + oa_q = ( + session.query( + Community.name, + func.count(Item.id).label("oa"), + ) + .join(Item.collections) + .join(Collection.community) + .filter(Item.dc_rights.like("%openAccess%")) + ) + if institution: + inst_name = analytics._resolve_institution_name(institution) + if inst_name: + base_q = base_q.filter(Item.institution.ilike(f"%{inst_name}%")) + oa_q = oa_q.filter(Item.institution.ilike(f"%{inst_name}%")) + + totals = {row.name: row.total for row in base_q.group_by(Community.name).all()} + oas = {row.name: row.oa for row in oa_q.group_by(Community.name).all()} + + return jsonify( + [ + { + "faculty": name, + "oa": oas.get(name, 0), + "restricted": totals[name] - oas.get(name, 0), + } + for name in totals + ] + ) + except Exception as e: + logger.error("faculty_oa_breakdown: %s", e) + return jsonify([]) + finally: + session.close() + + +@app.route("/api/analytics/impact-metrics") +def impact_metrics(): + session = SessionLocal() + institution = request.args.get("institution") + try: + inst_name = ( + analytics._resolve_institution_name(institution) if institution else None + ) + + q_item = session.query(Item) + if inst_name: + q_item = q_item.filter(Item.institution.ilike(f"%{inst_name}%")) + + total = q_item.count() + oa = q_item.filter(Item.dc_rights.like("%openAccess%")).count() + with_doi = q_item.filter(Item.doi.isnot(None)).count() + + q_file = session.query(File) + if inst_name: + q_file = q_file.join(File.item).filter( + Item.institution.ilike(f"%{inst_name}%") + ) + with_pdf = q_file.count() + + q_years = session.query(db_year(Item.publication_date)).filter( + Item.publication_date.isnot(None) + ) + if inst_name: + q_years = q_years.filter(Item.institution.ilike(f"%{inst_name}%")) + years = q_years.distinct().count() + + return jsonify( + { + "total_papers": total, + "open_access_papers": oa, + "oa_rate": round(oa / total * 100, 1) if total else 0, + "papers_with_doi": with_doi, + "doi_rate": round(with_doi / total * 100, 1) if total else 0, + "papers_with_local_pdf": with_pdf, + "pdf_rate": round(with_pdf / total * 100, 1) if total else 0, + "years_covered": years, + } + ) + except Exception as e: + logger.error("impact_metrics: %s", e) + return jsonify({"error": str(e)}), 500 + finally: + session.close() + + +@app.route("/api/analytics/search") +def analytics_search(): + q = request.args.get("q", "").strip() + faculty = request.args.get("faculty", "").strip() + year_from = request.args.get("year_from", type=int) + year_to = request.args.get("year_to", type=int) + oa_only = request.args.get("oa_only", "").lower() == "true" + limit = clamped_int("limit", 50, 1, 200) + session = SessionLocal() + try: + q_obj = session.query(Item) + if q: + author_subq = ( + session.query(Item.id) + .join(Item.authors) + .filter(Author.name.ilike(f"%{q}%")) + .subquery() + ) + q_obj = q_obj.filter( + or_( + Item.title.ilike(f"%{q}%"), + Item.abstract.ilike(f"%{q}%"), + Item.doi.ilike(f"%{q}%"), + Item.id.in_(author_subq), + ) + ) + if faculty: + q_obj = ( + q_obj.join(Item.collections) + .join(Collection.community) + .filter(Community.name.ilike(f"%{faculty}%")) + ) + if year_from: + q_obj = q_obj.filter(db_year(Item.publication_date) >= str(year_from)) + if year_to: + q_obj = q_obj.filter(db_year(Item.publication_date) <= str(year_to)) + if oa_only: + q_obj = q_obj.filter(Item.dc_rights.like("%openAccess%")) + items = q_obj.order_by(desc(Item.created_at)).limit(limit).all() + return jsonify( + [ + { + "id": i.id, + "title": i.title, + "doi": i.doi, + "abstract_snippet": ( + (i.abstract or "")[:200] + if q and i.abstract and q.lower() in (i.abstract or "").lower() + else None + ), + "authors": [a.name for a in i.authors[:4]], + "year": i.publication_date.year if i.publication_date else None, + "is_oa": "openAccess" in (i.dc_rights or ""), + "faculty": ( + i.collections[0].community.name if i.collections else None + ), + } + for i in items + ] + ) + finally: + session.close() + + +@app.route("/api/analytics/faculties") +def list_faculties(): + session = SessionLocal() + institution = request.args.get("institution") + try: + inst_name = ( + analytics._resolve_institution_name(institution) if institution else None + ) + q = session.query(Community.name).order_by(Community.name) + if inst_name: + q = q.filter( + Community.institution.ilike(f"%{inst_name}%") + | Community.name.ilike(f"%{inst_name}%") + ) + rows = q.all() + return jsonify([r[0] for r in rows]) + finally: + session.close() + + +@app.route("/api/institutions") +def list_institutions(): + """List all configured institutions with their staff counts and OAI support flag.""" + from uraas.config.institutions import get_registry + + registry = get_registry() + results = [] + for inst in registry.list_all(): + results.append( + { + "name": inst.name, + "short_name": inst.short_name, + "ror": inst.ror, + "sub_region": inst.sub_region, + "staff_count": len(inst.staff_names), + "has_oai": bool(inst.oai_endpoint), + "oai_endpoint": inst.oai_endpoint or None, + } + ) + return jsonify(results) + + +@app.route("/api/analytics/authors-search") +def authors_search(): + q = request.args.get("q", "") + limit = clamped_int("limit", 10, 1, 100) + institution = request.args.get("institution") + session = SessionLocal() + try: + inst_name = ( + analytics._resolve_institution_name(institution) if institution else None + ) + query = ( + session.query(Author.name, func.count(Item.id)) + .join(Author.items) + .filter(Author.name.ilike(f"%{q}%")) + ) + if inst_name: + query = query.filter(Item.institution.ilike(f"%{inst_name}%")) + authors = ( + query.group_by(Author.name) + .order_by(desc(func.count(Item.id))) + .limit(limit) + .all() + ) + return jsonify([{"name": a[0], "papers": a[1]} for a in authors]) + finally: + session.close() + + +@app.route("/api/analytics/author-network") +def author_network(): + author_name = request.args.get("author", "").strip() + return jsonify(analytics.get_author_network(author_name=author_name or None)) + + +@app.route("/api/analytics/keyword-cloud") +def keyword_cloud(): + institution = request.args.get("institution", None) + top_n = clamped_int("top_n", 60, 1, 150) + return jsonify(analytics.get_keyword_cloud(top_n=top_n, institution=institution)) + + +@app.route("/api/analytics/institution-leaderboard") +def institution_leaderboard(): + return jsonify(analytics.get_institution_leaderboard()) + + +@app.route("/api/analytics/cache-flush", methods=["POST"]) +def flush_analytics_cache(): + """Manual cache flush endpoint (admin use).""" + analytics_cache.invalidate_all() + return jsonify({"status": "success", "message": "Analytics cache flushed"}) + + +@app.route("/api/admin/prune-non-sc", methods=["POST"]) +def prune_non_sc(): + """Re-classify all items and delete non-SC papers (admin only). + + POST body: {"apply": true} — actually prune + POST body: {"apply": false} — dry run (default), just return counts + """ + from uraas.database import Author, Collection, Community, SessionLocal as _SL, item_authors + from uraas.services.sc_engine import is_special_collection + from sqlalchemy import text as _text + + data = request.get_json() or {} + apply = bool(data.get("apply", False)) + + session = _SL() + try: + items = session.query(Item).all() + total = len(items) + keep_ids, drop_ids = [], [] + samples = [] + + for it in items: + is_sc, score, cats = is_special_collection( + it.title or "", it.abstract or "", it.dc_subject or "" + ) + if apply: + it.special_collection_score = float(score) + it.special_collection_categories = ",".join(cats) if is_sc else "" + if is_sc: + keep_ids.append(it.id) + else: + drop_ids.append(it.id) + if len(samples) < 20: + samples.append({ + "id": it.id, + "title": (it.title or "")[:80], + "institution": it.institution or "", + }) + + if apply: + session.commit() + # Delete non-SC items in chunks + session.execute(_text("PRAGMA foreign_keys=ON")) + deleted = 0 + for i in range(0, len(drop_ids), 500): + chunk = drop_ids[i:i + 500] + for it in session.query(Item).filter(Item.id.in_(chunk)).all(): + session.delete(it) + deleted += 1 + session.commit() + + # Sweep stray association rows + session.execute(_text( + "DELETE FROM item_authors WHERE item_id NOT IN (SELECT id FROM items) " + "OR author_id NOT IN (SELECT id FROM authors)" + )) + session.execute(_text( + "DELETE FROM item_collections WHERE item_id NOT IN (SELECT id FROM items) " + "OR collection_id NOT IN (SELECT id FROM collections)" + )) + session.commit() + + # Orphan authors + orphan_authors = ( + session.query(Author) + .filter(~Author.id.in_(session.query(item_authors.c.author_id))) + .all() + ) + for a in orphan_authors: + session.delete(a) + empty_colls = [c for c in session.query(Collection).all() if not c.items] + for c in empty_colls: + session.delete(c) + session.commit() + empty_comms = [c for c in session.query(Community).all() if not c.collections] + for c in empty_comms: + session.delete(c) + session.commit() + + analytics_cache.invalidate_all() + + return jsonify({ + "status": "success", + "applied": apply, + "total_before": total, + "kept_sc": len(keep_ids), + "pruned_non_sc": len(drop_ids), + "sample_dropped": samples, + }) + except Exception as e: + logger.error("prune_non_sc: %s", e) + session.rollback() + return jsonify({"status": "error", "message": str(e)}), 500 + finally: + session.close() + + +@app.route("/api/admin/test-smtp", methods=["POST"]) +def test_smtp(): + """Send a test email to verify SMTP configuration (admin only).""" + from uraas.services.email_service import _is_smtp_configured + from uraas.config import config + import smtplib + from email.mime.text import MIMEText + from email.utils import parseaddr + + if not _is_smtp_configured(): + return jsonify({ + "status": "error", + "message": "SMTP not configured — set SMTP_HOST, SMTP_USER, SMTP_PASSWORD in environment/secrets.", + "smtp_host": config.SMTP_HOST or "(not set)", + "smtp_user": config.SMTP_USER or "(not set)", + }), 400 + + to_email = (request.get_json(silent=True) or {}).get("to", config.SMTP_USER) + try: + msg = MIMEText("URAAS SMTP test — configuration is working correctly.", "plain", "utf-8") + msg["Subject"] = "[URAAS] SMTP Test" + msg["From"] = config.SMTP_FROM + msg["To"] = to_email + if config.SMTP_USE_TLS: + server = smtplib.SMTP(config.SMTP_HOST, config.SMTP_PORT, timeout=15) + server.ehlo(); server.starttls(); server.ehlo() + else: + server = smtplib.SMTP_SSL(config.SMTP_HOST, config.SMTP_PORT, timeout=15) + server.login(config.SMTP_USER, config.SMTP_PASSWORD) + envelope_from = parseaddr(config.SMTP_FROM)[1] or config.SMTP_USER + server.sendmail(envelope_from, [to_email], msg.as_bytes()) + server.quit() + logger.info("SMTP test email sent to %s", to_email) + return jsonify({"status": "success", "message": f"Test email sent to {to_email}"}) + except Exception as exc: + logger.error("SMTP test failed: %s", exc) + return jsonify({"status": "error", "message": str(exc)}), 500 + + +@app.route("/api/admin/recompute-alignment", methods=["POST"]) +def recompute_alignment(): + """Score all SC items against AU framework pillars and rebuild AlignmentAggregate (admin only). + + This is the web-UI equivalent of running scripts/backfill_alignment.py. + Runs synchronously; expect 5-30 s for 100-200 papers.""" + from uraas.services.alignment_engine import recompute_aggregates, score_item_alignment + + session = SessionLocal() + try: + q = session.query(Item).filter(Item.special_collection_score > 0) + items = q.all() + scored = 0 + for it in items: + try: + al_json, al_ver = score_item_alignment( + it.title or "", it.abstract or "", it.dc_subject or "" + ) + it.alignment_scores = al_json + it.alignment_version = al_ver + scored += 1 + except Exception as e: + logger.warning("alignment scoring failed for item %s: %s", it.id, e) + session.commit() + + rows = recompute_aggregates(session) + analytics_cache.invalidate_all() + logger.info("recompute_alignment: scored=%d, aggregate_rows=%d", scored, rows) + return jsonify({"status": "success", "scored": scored, "aggregate_rows": rows}) + except Exception as e: + logger.error("recompute_alignment: %s", e) + session.rollback() + return jsonify({"status": "error", "message": str(e)}), 500 + finally: + session.close() + + +@app.route("/api/analytics/faculty-comparison") +def faculty_comparison(): + faculties = request.args.getlist("faculty") + institution = request.args.get("institution", None) + session = SessionLocal() + try: + result = {} + # If no specific faculties given, return all + if not faculties: + q = session.query(Community) + if institution: + q = q.filter(Community.institution.ilike(f"%{institution}%")) + comms = q.all() + else: + comms = [] + for fac_name in faculties: + comm = ( + session.query(Community) + .filter(Community.name.ilike(f"%{fac_name}%")) + .first() + ) + if comm: + comms.append(comm) + + for comm in comms: + # Skip vague community names that are actually institution names + if comm.name and len(comm.name) < 5: + continue + items = ( + session.query(Item) + .join(Item.collections) + .join(Collection.community) + .filter(Community.id == comm.id) + .options(selectinload(Item.authors)) + .all() + ) + if not items: + continue + oa_count = sum(1 for i in items if "openAccess" in (i.dc_rights or "")) + years = [i.publication_date.year for i in items if i.publication_date] + authors_set = set(a.name for i in items for a in i.authors) + # Top 3 keywords across all papers + from collections import Counter + + kw_counter = Counter() + for i in items: + for kw in (i.ai_keywords or "").split(","): + kw = kw.strip() + if len(kw) > 3: + kw_counter[kw] += 1 + top_keywords = [k for k, _ in kw_counter.most_common(5)] + result[comm.name] = { + "total_papers": len(items), + "open_access": oa_count, + "restricted": len(items) - oa_count, + "oa_rate": round(oa_count / len(items) * 100, 1) if items else 0, + "unique_authors": len(authors_set), + "year_range": [min(years), max(years)] if years else [], + "peak_year": max(set(years), key=years.count) if years else None, + "departments": len(comm.collections), + "top_keywords": top_keywords, + "institution": comm.institution or "Unknown", + } + return jsonify(result) + finally: + session.close() + + +@app.route("/api/analytics/department-comparison") +def department_comparison(): + faculty = request.args.get("faculty", "") + session = SessionLocal() + try: + comm = ( + session.query(Community) + .filter(Community.name.ilike(f"%{faculty}%")) + .first() + ) + if not comm: + return jsonify({}) + result = {} + for coll in comm.collections: + items = ( + session.query(Item) + .join(Item.collections) + .filter(Collection.id == coll.id) + .options(selectinload(Item.authors)) + .all() + ) + if not items: + continue + oa = sum(1 for i in items if "openAccess" in (i.dc_rights or "")) + years = [i.publication_date.year for i in items if i.publication_date] + result[coll.name] = { + "total": len(items), + "open_access": oa, + "restricted": len(items) - oa, + "oa_rate": round(oa / len(items) * 100, 1) if items else 0, + "unique_authors": len(set(a.name for i in items for a in i.authors)), + "years": sorted(set(years)), + } + return jsonify(result) + finally: + session.close() + + +@app.route("/api/analytics/lecturer-profile") +def lecturer_profile(): + name = request.args.get("name", "").strip() + session = SessionLocal() + try: + author = session.query(Author).filter(Author.name.ilike(f"%{name}%")).first() + if not author: + return jsonify({"error": "Author not found"}), 404 + # Eager-load the relationships walked below (collections→community, authors) + # so the profile renders in a handful of queries instead of one-per-paper. + items = ( + session.query(Item) + .filter(Item.authors.any(Author.id == author.id)) + .options( + selectinload(Item.authors), + selectinload(Item.collections).selectinload(Collection.community), + ) + .all() + ) + oa = sum(1 for i in items if "openAccess" in (i.dc_rights or "")) + years = sorted( + set(i.publication_date.year for i in items if i.publication_date) + ) + faculties = list( + set(c.community.name for i in items for c in i.collections if c.community) + ) + depts = list(set(c.name for i in items for c in i.collections)) + co_authors = {} + for item in items: + for a in item.authors: + if a.name != author.name: + co_authors[a.name] = co_authors.get(a.name, 0) + 1 + top_co = sorted(co_authors.items(), key=lambda x: -x[1])[:10] + return jsonify( + { + "name": author.name, + "total_papers": len(items), + "open_access": oa, + "oa_rate": round(oa / len(items) * 100, 1) if items else 0, + "active_years": years, + "faculties": faculties, + "departments": depts, + "top_collaborators": [{"name": n, "papers": c} for n, c in top_co], + "papers": [ + { + "id": i.id, + "title": i.title, + "doi": i.doi, + "year": i.publication_date.year if i.publication_date else None, + "is_oa": "openAccess" in (i.dc_rights or ""), + } + for i in sorted( + items, + key=lambda x: x.publication_date + or __import__("datetime").datetime.min, + reverse=True, + )[:20] + ], + } + ) + finally: + session.close() + + +@app.route("/api/analytics/language-research") +def language_research(): + """Language & Culture research — returns SC papers matched by language keywords.""" + from uraas.config.language_research import score_item + from uraas.services.sc_engine import SC_FILTER + + session = SessionLocal() + try: + institution = request.args.get("institution", "").strip() + q = session.query(Item).filter(SC_FILTER) + if institution: + q = q.filter(Item.institution.ilike(f"%{institution}%")) + items = q.all() + + matches = [] + keyword_counts: dict = {} + for item in items: + try: + score, matched = score_item(item.title or "", item.abstract or "") + if not score: + continue + # Count only the actual matched language terms (not all words) + for term in matched: + t = term.lower() + keyword_counts[t] = keyword_counts.get(t, 0) + 1 + matches.append( + { + "id": item.id, + "title": item.title, + "year": ( + item.publication_date.year if item.publication_date else None + ), + "authors": [a.name for a in item.authors[:4]], + "is_oa": "openAccess" in (item.dc_rights or ""), + "score": score, + "matched_terms": matched[:6], + } + ) + except Exception as e: + logger.error("language_research item %s: %s", item.id, e) + continue + + matches.sort(key=lambda x: (-x["score"], -(x.get("year") or 0))) + top_keywords = sorted(keyword_counts.items(), key=lambda x: -x[1])[:20] + return jsonify( + { + "total_language_papers": len(matches), + "top_keywords": [{"keyword": k, "count": v} for k, v in top_keywords], + "papers": matches[:50], + } + ) + except Exception as e: + logger.error("language_research: %s", e) + return ( + jsonify( + { + "error": "Internal server error", + "total_language_papers": 0, + "top_keywords": [], + "papers": [], + } + ), + 500, + ) + finally: + session.close() + + + + +# Multi-Institution Comparator (APA Core Feature) + + +@app.route("/api/comparator/compare", methods=["POST"]) +def compare_institutions(): + """ + Compare multiple institutions across all metrics. + Request body: {"ror_ids": ["ror1", "ror2", "ror3"]} + """ + try: + from uraas.services.comparator_engine import ComparatorEngine + + data = request.get_json() + ror_ids = data.get("ror_ids", []) + + if not ror_ids or len(ror_ids) < 2: + return jsonify({"error": "Provide at least 2 ROR IDs"}), 400 + + if len(ror_ids) > 15: + return jsonify({"error": "Maximum 15 institutions"}), 400 + + comparison = ComparatorEngine.compare_institutions(ror_ids) + return jsonify(comparison) + + except Exception as e: + logger.error(f"compare_institutions: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route("/api/comparator/collaboration-mesh", methods=["POST"]) +def collaboration_mesh(): + """ + Get collaboration network data for geographic visualization. + Request body: {"ror_ids": ["ror1", "ror2", "ror3"]} + """ + try: + from uraas.services.comparator_engine import ComparatorEngine + + data = request.get_json() + ror_ids = data.get("ror_ids", []) + + if not ror_ids: + return jsonify({"error": "Provide ROR IDs"}), 400 + + mesh = ComparatorEngine.get_collaboration_matrix(ror_ids) + return jsonify(mesh) + + except Exception as e: + logger.error(f"collaboration_mesh: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route("/api/comparator/senate-report", methods=["POST"]) +def generate_senate_report(): + """ + Generate comprehensive senate report. + Request body: {"ror_ids": ["ror1", "ror2"], "format": "json"} + """ + try: + from uraas.services.comparator_engine import ComparatorEngine + + data = request.get_json() + ror_ids = data.get("ror_ids", []) + format_type = data.get("format", "json") + + if not ror_ids: + return jsonify({"error": "Provide ROR IDs"}), 400 + + report = ComparatorEngine.generate_senate_report(ror_ids, format_type) + + if format_type == "json": + return jsonify(report) + elif format_type == "csv": + output = io.StringIO() + writer = csv.writer(output) + writer.writerow( + [ + "Institution", + "ROR", + "SC Papers", + "SC Authors", + "OA Rate %", + "Indigenous Knowledge", + "African Literature", + "Papers/Author", + ] + ) + for inst in report["detailed_comparison"]["institutions"]: + m = inst["metrics"] + writer.writerow( + [ + inst["name"], + inst["ror_id"], + m.get("total_papers", 0), + m.get("total_authors", 0), + m.get("oa_rate", 0), + m.get("indigenous_knowledge", 0), + m.get("african_literature", 0), + m.get("papers_per_author", 0), + ] + ) + output.seek(0) + return Response( + output.getvalue(), + mimetype="text/csv", + headers={ + "Content-Disposition": "attachment; filename=senate_report.csv" + }, + ) + elif format_type == "pdf": + # Plain-text report (PDF rendering would need reportlab; keep deps minimal) + lines = [ + report["title"], + "=" * len(report["title"]), + f"Generated: {report['generated_at']}", + f"Institutions: {report['institutions_analyzed']}", + "", + "Executive Summary:", + ] + for k, v in report["executive_summary"].items(): + lines.append(f" {k}: {v}") + lines += ["", "Recommendations:"] + for r in report.get("recommendations", []): + lines.append(f" - {r}") + return Response( + "\n".join(lines), + mimetype="text/plain", + headers={ + "Content-Disposition": "attachment; filename=senate_report.txt" + }, + ) + else: + return jsonify({"error": "Invalid format"}), 400 + + except Exception as e: + logger.error(f"generate_senate_report: {e}") + return jsonify({"error": str(e)}), 500 + + +# Citation Tracking & Bibliometrics + + +@app.route("/api/citations/") +def get_citations(item_id): + """Get citation data for a paper — enriched with ARK + Pan-African share (Phase 5).""" + session = SessionLocal() + try: + item = session.query(Item).filter_by(id=item_id).first() + # Base citation count from DB (fast, no API call) + base = { + "item_id": item_id, + "citation_count": item.cited_by_count or 0 if item else 0, + "ark": item.ark or "" if item else "", + "docid": item.docid or "" if item else "", + "african_citation_share": item.african_citation_share if item else None, + "openalex_id": item.openalex_id or "" if item else "", + } + except Exception: + base = {"item_id": item_id, "citation_count": 0} + finally: + session.close() + + # Supplement with live CitationMetrics record if available + try: + from uraas.services.citation_tracker import get_paper_citations + live = get_paper_citations(item_id) + base.update(live) + except Exception as e: + logger.debug(f"get_citations live lookup {item_id}: {e}") + return jsonify(base) + + +@app.route("/api/citations/velocity/export.csv") +def citations_velocity_csv(): + """Phase 4 — Streaming CSV export of the citation velocity time-series. + + Columns: year, citations_received, pan_african_share_pct, items_covered + Query param: ?institution= + """ + try: + institution = request.args.get("institution", "").strip().lower() or None + data = analytics.get_citation_velocity(institution) + series = data.get("by_year", []) + share = data.get("pan_african_share_pct") + covered = data.get("pan_african_share_items", 0) + + def gen(): + yield [ + "Year", + "Citations Received", + "Pan-African Share %", + "Items Covered (pan-African)", + ] + for row in series: + yield [ + row["year"], + row["citations"], + share if share is not None else "", + covered, + ] + + return csv_response(gen(), "citation_velocity.csv") + except Exception as e: + logger.error(f"citations_velocity_csv: {e}") + return api_error(str(e)) + + +@app.route("/api/citations/update/", methods=["POST"]) +def update_citations(item_id): + """Manually trigger citation update for a paper.""" + try: + from uraas.services.citation_tracker import CitationTracker + + success = CitationTracker.update_paper_citations(item_id) + return jsonify({"success": success, "item_id": item_id}) + except Exception as e: + logger.error(f"update_citations {item_id}: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route("/api/author//metrics") +def get_author_metrics(author_id): + """Get bibliometric indicators for an author (h-index, citations, etc.).""" + try: + from uraas.services.citation_tracker import get_author_bibliometrics + + return jsonify(get_author_bibliometrics(author_id)) + except Exception as e: + logger.error(f"get_author_metrics {author_id}: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route("/api/citations/bulk-update", methods=["POST"]) +def bulk_update_citations(): + """Bulk update citations for papers (admin endpoint).""" + try: + from uraas.services.citation_tracker import CitationTracker + + limit = clamped_int("limit", 50, 1, 200) + force = request.args.get("force", "false").lower() == "true" + stats = CitationTracker.bulk_update_citations(limit=limit, force=force) + return jsonify(stats) + except Exception as e: + logger.error(f"bulk_update_citations: {e}") + return jsonify({"error": str(e)}), 500 + + +# Advanced Search + + +@app.route("/api/search/advanced") +def advanced_search(): + """ + Advanced search with Boolean operators and field-specific queries. + + Query examples: + ?q="machine learning" AND author:smith + ?q=(covid OR pandemic) AND year:2020 + ?q=title:cancer NOT lung + ?q=author:okonkwo AND faculty:science + + Supported fields: + title, abstract, author, year, doi, faculty, department, keyword, language, oa + + Operators: + AND, OR, NOT, parentheses for grouping, "quotes" for phrases + """ + try: + from uraas.services.advanced_search import SearchQuery + + query = request.args.get("q", "").strip() + limit = clamped_int("limit", 50, 1, 200) + offset = clamped_int("offset", 0, 0, 100000) + sort_by = request.args.get( + "sort", "relevance" + ) # relevance, date, citations, title + + filters = { + "year_from": request.args.get("year_from", type=int), + "year_to": request.args.get("year_to", type=int), + "oa_only": request.args.get("oa_only", "false").lower() == "true", + "faculty": request.args.get("faculty"), + "has_pdf": request.args.get("has_pdf", "false").lower() == "true", + } + + results = SearchQuery.execute_search( + query=query, limit=limit, offset=offset, sort_by=sort_by, filters=filters + ) + + return jsonify(results) + + except Exception as e: + logger.error(f"advanced_search: {e}") + return jsonify({"error": str(e), "total": 0, "results": []}), 500 + + +@app.route("/api/search/suggest") +def search_suggestions(): + """Get autocomplete suggestions for search queries.""" + try: + from uraas.services.advanced_search import SearchQuery + + partial = request.args.get("q", "").strip() + field = request.args.get("field", "all") + + suggestions = SearchQuery.get_search_suggestions(partial, field) + return jsonify({"suggestions": suggestions}) + + except Exception as e: + logger.error(f"search_suggestions: {e}") + return jsonify({"suggestions": []}), 500 + + +# APA Novel Metrics + + +@app.route("/api/analytics/tk-vitality-score") +def tk_vitality_score(): + institution = request.args.get("institution", None) + return jsonify(analytics.get_tk_vitality_score(institution=institution)) + + +@app.route("/api/analytics/linguistic-diversity-index") +def linguistic_diversity_index(): + return jsonify(analytics.get_linguistic_diversity_index()) + + +@app.route("/api/analytics/pid-coverage") +def pid_coverage(): + institution = request.args.get("institution", None) + return jsonify(analytics.get_pid_coverage(institution=institution)) + + +@app.route("/api/analytics/knowledge-repatriation") +def knowledge_repatriation(): + institution = request.args.get("institution", None) + return jsonify(analytics.get_knowledge_repatriation(institution=institution)) + + +@app.route("/api/analytics/research-diversity") +def research_diversity(): + institution = request.args.get("institution", None) + return jsonify(analytics.get_research_diversity(institution=institution)) + + +@app.route("/api/analytics/open-science-health") +def open_science_health(): + institution = request.args.get("institution", None) + return jsonify(analytics.get_open_science_health(institution=institution)) + + +@app.route("/api/analytics/special-collections") +def special_collections(): + institution = request.args.get("institution", None) + return jsonify(analytics.get_special_collections_metrics(institution=institution)) + + +@app.route("/api/analytics/special-collections/overview") +def special_collections_overview(): + institution = request.args.get("institution", None) + return jsonify(analytics.get_special_collections_overview(institution=institution)) + + +@app.route("/api/analytics/special-collections/export.csv") +def export_special_collections_csv(): + """Download special collections data as CSV.""" + try: + rows = analytics.get_special_collections_csv_data() + output = io.StringIO() + writer = csv.writer(output) + for row in rows: + writer.writerow(row) + output.seek(0) + return Response( + output.getvalue(), + mimetype="text/csv", + headers={ + "Content-Disposition": "attachment; filename=uraas_special_collections.csv" + }, + ) + except Exception as e: + logger.error(f"export_special_collections_csv: {e}") + return jsonify({"error": str(e)}), 500 + + +# ── Framework Alignment endpoints (AU charters / Agenda 2063 / blocs) ──────── +# Scores are precomputed at ingest / by scripts/backfill_alignment.py and read +# from alignment_aggregates — these endpoints never score at request time. + + +def _resolve_inst_name(): + institution = request.args.get("institution", "").strip().lower() + return analytics._resolve_institution_name(institution) if institution else None + + +def _alignment_top_papers(session, top_item_ids, framework, pillar): + """Evidence chips: resolve top item ids to titles + matched keywords.""" + from uraas.services.alignment_engine import get_alignment + + ids = [int(i) for i in (top_item_ids or "").split(",") if i] + papers = [] + for item in session.query(Item).filter(Item.id.in_(ids)).all(): + pdata = ( + get_alignment(item).get(framework, {}).get("pillars", {}).get(pillar, {}) + ) + papers.append( + { + "id": item.id, + "title": item.title or "Untitled", + "score": pdata.get("score", 0), + "matched_keywords": pdata.get("matched_keywords", [])[:4], + } + ) + papers.sort(key=lambda p: -p["score"]) + return papers + + +@app.route("/api/alignment/frameworks") +def alignment_frameworks(): + """All frameworks + pillar metadata for the selector UI.""" + from uraas.config.alignment_frameworks import FRAMEWORK_GROUPS, FRAMEWORKS + from uraas.services.alignment_engine import scoring_mode + + frameworks = [ + { + "key": fkey, + "name": f["name"], + "type": f["type"], + "year": f.get("year"), + "color": f.get("color", "#3b82f6"), + "pillars": [ + {"key": pk, "name": p["name"], "description": p["description"]} + for pk, p in f["pillars"].items() + ], + } + for fkey, f in FRAMEWORKS.items() + ] + return api_ok( + {"frameworks": frameworks, "groups": FRAMEWORK_GROUPS, "mode": scoring_mode()} + ) + + +@app.route("/api/alignment/profile") +def alignment_profile(): + """Radar-chart profile for one framework: per-pillar avg score + evidence.""" + from uraas.config.alignment_frameworks import FRAMEWORKS, GAP_THRESHOLD + from uraas.database import AlignmentAggregate + from uraas.services.narratives import narrate + + framework = request.args.get("framework", "agenda2063") + if framework not in FRAMEWORKS: + return api_error(f"Unknown framework: {framework}", 400) + inst_name = _resolve_inst_name() + + cache_key = f"align_profile_{framework}_{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached: + return jsonify(cached) + + session = SessionLocal() + try: + fdef = FRAMEWORKS[framework] + rows = { + r.pillar: r + for r in session.query(AlignmentAggregate) + .filter_by(institution=inst_name or "", framework=framework) + .all() + } + pillars = [] + for pkey, pdef in fdef["pillars"].items(): + r = rows.get(pkey) + avg = r.avg_score if r else 0.0 + pillars.append( + { + "key": pkey, + "name": pdef["name"], + "avg_score": avg, + "paper_count": r.paper_count if r else 0, + "is_gap": avg < GAP_THRESHOLD, + "top_papers": ( + _alignment_top_papers(session, r.top_item_ids, framework, pkey) + if r + else [] + ), + } + ) + + scores = [p["avg_score"] for p in pillars] + top = max(pillars, key=lambda p: p["avg_score"]) if pillars else None + data = { + "framework": framework, + "framework_name": fdef["name"], + "color": fdef.get("color", "#3b82f6"), + "institution": inst_name or "All Institutions", + "pillars": pillars, + "overall_score": round(sum(scores) / len(scores), 1) if scores else 0, + "gap_threshold": GAP_THRESHOLD, + } + narrative = ( + narrate( + "alignment_profile", + institution=inst_name or "The repository", + top_pillar=top["name"], + top_score=top["avg_score"], + top_count=top["paper_count"], + gap_count=sum(1 for p in pillars if p["is_gap"]), + pillar_count=len(pillars), + threshold=GAP_THRESHOLD, + ) + if top + else "" + ) + resp = api_ok(data, narrative=narrative, methodology_key="alignment_score") + analytics_cache.set(cache_key, resp.get_json(), ttl=1800) + return resp + except Exception as e: + logger.error(f"alignment_profile: {e}") + return api_error(str(e)) + finally: + session.close() + + +@app.route("/api/alignment/matrix") +def alignment_matrix(): + """Heatmap cells: every (framework, pillar) avg score for one institution.""" + from uraas.config.alignment_frameworks import FRAMEWORKS + from uraas.database import AlignmentAggregate + + inst_name = _resolve_inst_name() + cache_key = f"align_matrix_{inst_name or 'all'}" + cached = analytics_cache.get(cache_key) + if cached: + return jsonify(cached) + + session = SessionLocal() + try: + rows = ( + session.query(AlignmentAggregate) + .filter_by(institution=inst_name or "") + .all() + ) + scored = {(r.framework, r.pillar): r for r in rows} + cells = [] + for fkey, f in FRAMEWORKS.items(): + for pkey, pdef in f["pillars"].items(): + r = scored.get((fkey, pkey)) + cells.append( + { + "framework": fkey, + "framework_name": f["name"], + "pillar": pkey, + "pillar_name": pdef["name"], + "avg_score": r.avg_score if r else 0.0, + "paper_count": r.paper_count if r else 0, + } + ) + data = { + "institution": inst_name or "All Institutions", + "frameworks": [ + {"key": k, "name": f["name"], "color": f.get("color")} + for k, f in FRAMEWORKS.items() + ], + "cells": cells, + } + resp = api_ok(data, methodology_key="alignment_score") + analytics_cache.set(cache_key, resp.get_json(), ttl=1800) + return resp + except Exception as e: + logger.error(f"alignment_matrix: {e}") + return api_error(str(e)) + finally: + session.close() + + +@app.route("/api/alignment/gaps") +def alignment_gaps(): + """Research Gap Radar: pillars below threshold, ascending by score.""" + from uraas.config.alignment_frameworks import FRAMEWORKS, GAP_THRESHOLD + from uraas.database import AlignmentAggregate + from uraas.services.narratives import narrate + + inst_name = _resolve_inst_name() + try: + threshold = float(request.args.get("threshold", GAP_THRESHOLD)) + except ValueError: + threshold = GAP_THRESHOLD + + session = SessionLocal() + try: + rows = { + (r.framework, r.pillar): r + for r in session.query(AlignmentAggregate) + .filter_by(institution=inst_name or "") + .all() + } + gaps = [] + for fkey, f in FRAMEWORKS.items(): + for pkey, pdef in f["pillars"].items(): + r = rows.get((fkey, pkey)) + avg = r.avg_score if r else 0.0 + if avg < threshold: + gaps.append( + { + "framework": fkey, + "framework_name": f["name"], + "pillar": pkey, + "pillar_name": pdef["name"], + "avg_score": avg, + "paper_count": r.paper_count if r else 0, + } + ) + gaps.sort(key=lambda g: g["avg_score"]) + worst = gaps[0] if gaps else None + narrative = ( + narrate( + "alignment_gaps", + gap_count=len(gaps), + framework_count=len({g["framework"] for g in gaps}), + worst_pillar=worst["pillar_name"], + worst_framework=worst["framework_name"], + worst_score=worst["avg_score"], + ) + if worst + else "" + ) + return api_ok( + { + "gaps": gaps, + "threshold": threshold, + "institution": inst_name or "All Institutions", + }, + narrative=narrative, + methodology_key="alignment_gap", + ) + except Exception as e: + logger.error(f"alignment_gaps: {e}") + return api_error(str(e)) + finally: + session.close() + + +@app.route("/api/alignment/export.csv") +def alignment_export_csv(): + """CSV of the full alignment matrix for the selected institution.""" + from uraas.config.alignment_frameworks import FRAMEWORKS + from uraas.database import AlignmentAggregate + + inst_name = _resolve_inst_name() + session = SessionLocal() + try: + rows = { + (r.framework, r.pillar): r + for r in session.query(AlignmentAggregate) + .filter_by(institution=inst_name or "") + .all() + } + + def generate(): + yield ["Framework", "Pillar", "Avg Score (0-100)", "Aligned Papers"] + for fkey, f in FRAMEWORKS.items(): + for pkey, pdef in f["pillars"].items(): + r = rows.get((fkey, pkey)) + yield [ + f["name"], + pdef["name"], + r.avg_score if r else 0.0, + r.paper_count if r else 0, + ] + + return csv_response(generate(), "alignment_matrix.csv") + except Exception as e: + logger.error(f"alignment_export_csv: {e}") + return api_error(str(e)) + finally: + session.close() + + +# ── Intra-African collaboration endpoints ───────────────────────────────────── + + +@app.route("/api/collaboration/overview") +def collaboration_overview(): + """Intra-African Collaboration Index + benchmark context.""" + from uraas.services.narratives import narrate + + try: + institution = request.args.get("institution", "").strip().lower() or None + data = analytics.get_collaboration_overview(institution) + partners = data.get("top_partner_countries", []) + narrative = narrate( + "intra_african" if partners else "intra_african_no_partner", + pct=data.get("intra_african_pct", 0), + ratio=data.get("ratio_vs_baseline", 0), + baseline=data.get("baseline_pct", 8.4), + top_partner=partners[0]["name"] if partners else "", + ) + return api_ok( + data, + narrative=narrative, + methodology_key="intra_african_collaboration", + ) + except Exception as e: + logger.error(f"collaboration_overview: {e}") + return api_error(str(e)) + + +@app.route("/api/collaboration/matrix") +def collaboration_matrix(): + """African country-pair co-publication matrix.""" + from uraas.services.narratives import narrate + + try: + institution = request.args.get("institution", "").strip().lower() or None + data = analytics.get_country_pair_matrix(institution) + top = data["pairs"][0] if data["pairs"] else None + narrative = ( + narrate( + "country_pairs", + pair_count=len(data["pairs"]), + country_a=top["source_name"], + country_b=top["target_name"], + count=top["count"], + ) + if top + else "" + ) + return api_ok(data, narrative=narrative, methodology_key="country_pair_matrix") + except Exception as e: + logger.error(f"collaboration_matrix: {e}") + return api_error(str(e)) + + +@app.route("/api/collaboration/countries") +def collaboration_countries(): + """Per-country paper counts for the Africa choropleth.""" + try: + institution = request.args.get("institution", "").strip().lower() or None + return api_ok( + {"countries": analytics.get_country_aggregates(institution)}, + methodology_key="intra_african_collaboration", + ) + except Exception as e: + logger.error(f"collaboration_countries: {e}") + return api_error(str(e)) + + +@app.route("/api/collaboration/arcs") +def collaboration_arcs(): + """GeoJSON great-circle arcs for the collaboration map (bare GeoJSON).""" + try: + institution = request.args.get("institution", "").strip().lower() or None + return jsonify(analytics.get_collaboration_arcs(institution)) + except Exception as e: + logger.error(f"collaboration_arcs: {e}") + return api_error(str(e)) + + +@app.route("/api/collaboration/network") +def collaboration_network(): + """Author collaboration network with Louvain communities + centrality.""" + try: + author = request.args.get("author", "").strip() or None + limit = clamped_int("limit", 30, 1, 100) + return api_ok(analytics.get_author_network(author, limit)) + except Exception as e: + logger.error(f"collaboration_network: {e}") + return api_error(str(e)) + + +@app.route("/api/citations/velocity") +def citations_velocity(): + """Repository citation velocity + Pan-African citation share.""" + from uraas.services.narratives import narrate + + try: + institution = request.args.get("institution", "").strip().lower() or None + data = analytics.get_citation_velocity(institution) + series = data.get("by_year", []) + recent = series[-1]["citations"] if series else 0 + parts = [] + if series: + parts.append( + narrate( + "citation_velocity", + recent_rate=recent, + avg_first2y=data.get("avg_first2y", 0), + ) + ) + if data.get("pan_african_share_pct") is not None: + parts.append( + narrate( + "pan_african_share", + share=data["pan_african_share_pct"], + covered=data.get("pan_african_share_items", 0), + ) + ) + return api_ok( + data, + narrative=" ".join(p for p in parts if p), + methodology_key="citation_velocity", + ) + except Exception as e: + logger.error(f"citations_velocity: {e}") + return api_error(str(e)) + + +@app.route("/api/collaboration/export.csv") +def collaboration_export_csv(): + """CSV export — ?view=matrix (default) or ?view=countries.""" + try: + institution = request.args.get("institution", "").strip().lower() or None + view = request.args.get("view", "matrix") + if view == "countries": + rows_data = analytics.get_country_aggregates(institution) + + def gen_countries(): + yield ["Country", "ISO2", "Papers", "Intra-African Papers"] + for r in rows_data: + yield [r["name"], r["code"], r["papers"], r["intra_african"]] + + return csv_response(gen_countries(), "collaboration_countries.csv") + + matrix = analytics.get_country_pair_matrix(institution) + + def gen_matrix(): + yield ["Country A", "Country B", "Co-publications"] + for p in matrix["pairs"]: + yield [p["source_name"], p["target_name"], p["count"]] + + return csv_response(gen_matrix(), "collaboration_matrix.csv") + except Exception as e: + logger.error(f"collaboration_export_csv: {e}") + return api_error(str(e)) + + +@app.route("/api/analytics/staff-directory") +def staff_directory(): + """ + Returns real staff records with name, department, faculty, ORCID for each institution. + Query params: ?institution=unilag (optional; returns all if omitted) + """ + from uraas.config.institutions import get_registry + + registry = get_registry() + institution_filter = request.args.get("institution", "").strip().lower() + + result = [] + insts = ( + [registry.get(institution_filter)] + if institution_filter + else registry.list_all() + ) + insts = [i for i in insts if i] # filter None + + for inst in insts: + # Get dynamic authors from database + dynamic_authors = analytics.get_top_authors( + limit=5000, institution=inst.short_name + ) + + # Merge dynamic ORCIDs/RORs with static departments + staff_data = [] + static_lookup = {r["name"].lower(): r for r in inst.staff_records} + + for author in dynamic_authors: + a_name = author.get("author", "") + static_rec = static_lookup.get(a_name.lower(), {}) + + staff_data.append( + { + "name": a_name, + "orcid": author.get("orcid") or static_rec.get("orcid"), + "ror": author.get("ror"), + "department": static_rec.get("department"), + "faculty": static_rec.get("faculty"), + "paper_count": author.get("count", 0), + } + ) + + result.append( + { + "institution": inst.name, + "short_name": inst.short_name, + "country": inst.country, + "staff": staff_data, + "staff_count": len(staff_data), + "staff_with_orcid": sum(1 for s in staff_data if s.get("orcid")), + "departments": inst.departments, + } + ) + + return jsonify(result) + + +# Export + + +@app.route("/api/export/papers.csv") +def export_csv(): + """Streaming CSV of every paper (yield_per avoids loading all rows).""" + from sqlalchemy.orm import selectinload + + def generate(): + session = SessionLocal() + try: + yield [ + "ID", + "Title", + "Authors", + "DOI", + "ARK", + "DocID", + "Year", + "Faculty", + "Open Access", + "Source", + ] + # selectinload (not joinedload) is required with yield_per — joined + # eager loads against collections need row-uniquing, which yield_per + # forbids. + q = ( + session.query(Item) + .options( + selectinload(Item.authors), + selectinload(Item.collections).selectinload(Collection.community), + ) + .order_by(desc(Item.created_at)) + ) + for i in q.yield_per(200): + authors = "; ".join(a.name for a in i.authors) + faculty = i.collections[0].community.name if i.collections else "" + year = i.publication_date.year if i.publication_date else "" + yield [ + i.id, + i.title or "", + authors, + i.doi or "", + i.ark or "", + i.docid or "", + year, + faculty, + "Yes" if "openAccess" in (i.dc_rights or "") else "No", + i.source_repository or "", + ] + finally: + session.close() + + return csv_response(generate(), "uraas_papers.csv") + + +@app.route("/api/export/papers.bibtex") +def export_bibtex(): + session = SessionLocal() + try: + items = session.query(Item).order_by(desc(Item.created_at)).all() + entries = [] + for i in items: + authors = [a.name for a in i.authors] + first_last = authors[0].split()[-1] if authors else "Unknown" + year = str(i.publication_date.year) if i.publication_date else "nd" + key = __import__("re").sub(r"[^a-zA-Z0-9]", "", f"{first_last}{year}") + author_str = " and ".join(authors) if authors else "Unknown" + title = (i.title or "Untitled").replace("{", "").replace("}", "") + doi_line = f" doi = {{{i.doi}}},\n" if i.doi else "" + url_line = f" url = {{{i.url}}},\n" if i.url else "" + institution = (i.institution or "").strip() or "Unknown" + entries.append( + f"@article{{{key},\n title = {{{title}}},\n author = {{{author_str}}},\n year = {{{year}}},\n institution = {{{institution}}},\n" + + doi_line + + url_line + + "}" + ) + return Response( + "\n\n".join(entries), + mimetype="text/plain", + headers={"Content-Disposition": "attachment; filename=uraas_papers.bib"}, + ) + finally: + session.close() + + +# Crawler control + + +@app.route("/api/crawler/start", methods=["POST"]) +def start_crawler(): + global crawler_process + with crawler_lock: + if crawler_process and crawler_process.poll() is None: + return ( + jsonify({"status": "error", "message": "Crawler already running"}), + 400, + ) + data = request.get_json() or {} + target = min(max(int(data.get("target", 20)), 1), 250) + institution = data.get("institution", "unilag") + # Validate institution against registry before passing to subprocess. + if institution != "all": + from uraas.config.institutions import get_registry as _get_reg + _reg = _get_reg() + if not _reg.get(institution): + return jsonify({"status": "error", "message": f"Unknown institution: {institution}"}), 400 + # Default ON — heavy bias toward Special Collections in every crawl. + boost_special = bool(data.get("boost_special", True)) + sc_only = bool(data.get("sc_only", False)) + # Optional spider selection (allowlisted). "oai" = read-only harvest of + # the institution's own repository (theses/grey literature). + spider = data.get("spider", "openalex") + _allowed_spiders = ("openalex", "crossref", "arxiv", "orcid", "oai", + "semantic_scholar", "europepmc", "core", "pubmed", + "openaire", "doaj", "ajol", "all") + if spider not in _allowed_spiders: + return jsonify({"status": "error", "message": f"Unknown spider: {spider}"}), 400 + # OAI date window — accept only a safe YYYY-MM-DD shape; ignore anything else. + _date_re = re.compile(r"^\d{4}-\d{2}-\d{2}$") + from_date = data.get("from_date") + until_date = data.get("until_date") + from_date = from_date if (from_date and _date_re.match(str(from_date))) else None + until_date = until_date if (until_date and _date_re.match(str(until_date))) else None + try: + # Derive project root and script path + project_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + script_path = os.path.join( + project_root, "scripts", "crawl_multi_institution.py" + ) + + cmd = [__import__("sys").executable, script_path, "--target", str(target)] + if institution != "all": + cmd.extend(["--institutions", institution]) + cmd.extend(["--spider", spider]) + if spider == "oai": + if from_date: + cmd.extend(["--from-date", from_date]) + if until_date: + cmd.extend(["--until-date", until_date]) + else: + if not boost_special: + cmd.append("--no-boost-special") + if sc_only: + cmd.append("--sc-only") + + logger.info(f"Executing crawler command: {' '.join(cmd)}") + + # Pass PYTHONUNBUFFERED so terminal output appears in real-time order + env = dict(os.environ, PYTHONUNBUFFERED="1") + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, + env=env, + ) + crawler_process = process + thread = threading.Thread( + target=crawler_monitor, args=(process,), daemon=True + ) + thread.start() + return jsonify( + { + "status": "success", + "message": f"Crawler started target {target} papers", + } + ) + except FileNotFoundError: + return ( + jsonify( + { + "status": "error", + "message": f"Crawler script not found at {script_path}", + } + ), + 500, + ) + except Exception as e: + logger.error("start_crawler: %s", e) + return jsonify({"status": "error", "message": str(e)}), 500 + + +@app.route("/api/crawler/stop", methods=["POST"]) +def stop_crawler(): + global crawler_process + with crawler_lock: + if crawler_process and crawler_process.poll() is None: + crawler_process.terminate() + crawler_process = None + return jsonify({"status": "success", "message": "Crawler stopped"}) + return jsonify({"status": "warning", "message": "No crawler running"}) + + +@app.route("/api/crawler/status") +def crawler_status(): + with crawler_lock: + running = crawler_process is not None and crawler_process.poll() is None + return jsonify({"status": "running" if running else "idle"}) + + +# Health Check Endpoint for Render + + +@app.route("/health") +def health_check(): + """Render readiness probe (Phase 8 enhanced). + + Checks: DB connectivity + latency, disk space, analytics cache, methodology. + Returns 200 if all checks pass, 503 if any critical check fails. + """ + import time + from datetime import datetime + + from sqlalchemy import text + from uraas.config.methodology import METHODOLOGY + + t_start = time.monotonic() + health_status = { + "status": "healthy", + "timestamp": datetime.utcnow().isoformat(), + "version": os.getenv("RENDER_GIT_COMMIT", "dev")[:8], + "checks": {}, + } + + # DB check (with latency) + t0 = time.monotonic() + try: + session = SessionLocal() + session.execute(text("SELECT 1")) + session.close() + health_status["checks"]["database"] = { + "status": "ok", + "latency_ms": round((time.monotonic() - t0) * 1000, 1), + } + except Exception as e: + health_status["checks"]["database"] = {"status": f"error: {str(e)}"} + health_status["status"] = "unhealthy" + logger.error(f"Database health check failed: {str(e)}") + + # Analytics cache probe + try: + analytics_cache.get("__health__") + health_status["checks"]["cache"] = {"status": "ok"} + except Exception as e: + health_status["checks"]["cache"] = {"status": f"error: {str(e)}"} + + # Methodology sanity + health_status["checks"]["methodology"] = { + "status": "ok", + "metric_count": len(METHODOLOGY), + } + + # Disk space check on persistent volume + try: + storage_path = config.STORAGE_PATH + if os.path.exists(storage_path): + import shutil + stat = shutil.disk_usage(storage_path) + free_gb = stat.free / (1024**3) + health_status["checks"]["disk_space_gb"] = round(free_gb, 2) + if free_gb < 1: + health_status["status"] = "unhealthy" + health_status["checks"]["disk_space"] = "critical" + else: + health_status["checks"]["disk_space"] = "storage path not found" + except Exception as e: + health_status["checks"]["disk_space"] = f"error: {str(e)}" + + health_status["response_ms"] = round((time.monotonic() - t_start) * 1000, 1) + status_code = 200 if health_status["status"] == "healthy" else 503 + return jsonify(health_status), status_code + + +@app.route("/api/university-registry", methods=["GET"]) +def get_university_registry(): + """ + Get the comprehensive 52-country African university registry. + """ + try: + import json + + registry_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "data", + "university_registry.json", + ) + if not os.path.exists(registry_path): + registry_path = "data/university_registry.json" + + with open(registry_path, "r", encoding="utf-8") as f: + data = json.load(f) + return jsonify(data) + except Exception as e: + logger.error(f"get_university_registry: {e}") + return jsonify({"error": str(e)}), 500 + + +@app.route("/api/reports/unilag-subregion", methods=["GET"]) +def get_unilag_report(): + """ + Generate draft report of UNILAG contributions to African languages in West Africa. + """ + session = SessionLocal() + try: + import json + from datetime import datetime + + from uraas.utils.ai_classifier import AU_CHARTER_TARGETS, classify_au_targets + + # UNILAG ROR + unilag_ror = "https://ror.org/05rk03822" + + # 1. Fetch UNILAG items + unilag_items = session.query(Item).filter(Item.ror == unilag_ror).all() + total_unilag = len(unilag_items) + + # 2. Fetch West Africa items (excluding UNILAG, e.g. UI and Covenant) + west_africa_rors = [ + "https://ror.org/01js2sh04", + "https://ror.org/0545s4788", + ] # UI and Covenant + wa_items = session.query(Item).filter(Item.ror.in_(west_africa_rors)).all() + total_wa = len(wa_items) + + # 3. Analyze UNILAG papers against AU Charter Target 2 (African Languages) + target2_compliant = 0 + keywords_found = set() + by_year = {} + + for item in unilag_items: + results = classify_au_targets( + item.title or "", item.abstract or "", item.dc_subject or "" + ) + for r in results: + if r["target_number"] == 2: + target2_compliant += 1 + keywords_found.update(r["matched_keywords"]) + year = item.publication_date.year if item.publication_date else None + if year: + by_year[year] = by_year.get(year, 0) + 1 + + # Determine gaps + all_t2_keywords = AU_CHARTER_TARGETS[2]["keywords"] + keywords_gap = [kw for kw in all_t2_keywords if kw not in keywords_found] + + # 4. Compare with West African average + wa_target2_compliant = 0 + for item in wa_items: + results = classify_au_targets( + item.title or "", item.abstract or "", item.dc_subject or "" + ) + for r in results: + if r["target_number"] == 2: + wa_target2_compliant += 1 + + unilag_compliance_rate = ( + round(target2_compliant / total_unilag * 100, 1) if total_unilag else 0.0 + ) + wa_compliance_rate = ( + round(wa_target2_compliant / total_wa * 100, 1) if total_wa else 0.0 + ) + + report_data = { + "title": "Decolonizing Knowledge: UNILAG Contributions to African Languages & Cultural Renaissance in West Africa", + "metadata": { + "institution": "University of Lagos (UNILAG)", + "subregion": "West Africa", + "generated_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), + }, + "introduction": ( + "This report evaluates the academic contributions of the University of Lagos (UNILAG) " + "toward the development of African languages and the decolonization of science in the West African sub-region. " + "Aligned with the African Union Charter for African Cultural Renaissance, specifically Target 2 (Development of African Languages), " + "this analysis highlights the intersection of linguistic preservation, local knowledge systems, and active " + "institutional engagement in decolonial research." + ), + "statistics": { + "total_curated": total_unilag, + "compliant_count": target2_compliant, + "compliance_rate": unilag_compliance_rate, + "gaps_count": total_unilag - target2_compliant, + "keywords_found": list(keywords_found), + "keywords_gap": keywords_gap, + }, + "scores_and_trends": { + "timeline": [ + {"year": y, "count": by_year[y]} for y in sorted(by_year.keys()) + ], + "comparison": { + "unilag_rate": unilag_compliance_rate, + "west_africa_rate": wa_compliance_rate, + "unilag_compliant": target2_compliant, + "west_africa_compliant": wa_target2_compliant, + }, + }, + "conclusion": ( + f"UNILAG shows solid alignment with the African Union Charter targets, with a decolonial compliance rate of {unilag_compliance_rate}%. " + f"Linguistic preservation is robust, particularly with keywords like '{', '.join(list(keywords_found)[:4])}' being highly active. " + f"However, critical gaps remain in the development of scientific literature in local languages. " + f"To bridge this gap, future research should focus on areas like '{', '.join(keywords_gap[:4])}' to ensure a more comprehensive " + "contribution to the African Union's Renaissance targets." + ), + } + return jsonify(report_data) + except Exception as e: + logger.error(f"get_unilag_report: {e}") + return jsonify({"error": str(e)}), 500 + finally: + session.close() + + +# ── Live IR connection + batch deposit ─────────────────────────────────────── +# All write endpoints are admin-only (ADMIN_ENDPOINTS list at the top of this +# file gates them automatically). The approve/reject token endpoints are +# intentionally PUBLIC — the token itself is the credential. + +ADMIN_ENDPOINTS.update({ + "ir_status", + "ir_collections", + "ir_live_stats", + "ir_queue_batch", + "ir_list_batches", + "ir_batch_status", + "ir_test_harvest", +}) + + +@app.route("/api/ir/test-harvest", methods=["POST"]) +def ir_test_harvest(): + """Dry-run OAI harvest: collect SC papers, send preview email, save JSON. + + Body JSON: + institution – short name (default: "unilag") + count – max SC papers to collect (default: 50, max: 100) + email – confirmation address (required) + from_date – OAI from date YYYY-MM-DD (optional) + + DOES NOT save to DB. DOES NOT deposit to IR. + Runs in a background thread; returns immediately with a job ID. + """ + data = request.get_json(silent=True) or {} + institution = (data.get("institution") or "unilag").strip().lower() + count = min(max(int(data.get("count") or 50), 1), 100) + email = (data.get("email") or "").strip() + from_date = (data.get("from_date") or "").strip() or None + + if not email or "@" not in email: + return jsonify({"status": "error", "message": "A valid email address is required"}), 400 + + from uraas.config.institutions import get_registry as _get_reg + _reg = _get_reg() + inst_cfg = _reg.get(institution) + if not inst_cfg: + return jsonify({"status": "error", "message": f"Unknown institution: {institution}"}), 400 + if not inst_cfg.oai_endpoint: + return jsonify({"status": "error", "message": f"'{institution}' has no OAI endpoint configured"}), 400 + + import subprocess, sys as _sys + script_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + "scripts", "test_harvest_50.py", + ) + cmd = [_sys.executable, script_path, + "--institution", institution, + "--count", str(count), + "--email", email] + if from_date: + cmd.extend(["--from-date", from_date]) + + try: + env = dict(os.environ, PYTHONUNBUFFERED="1") + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, + env=env, + ) + # Stream output via existing SocketIO terminal + def _monitor(): + for line in iter(process.stdout.readline, b""): + line_text = line.decode("utf-8", errors="replace").strip() + if line_text: + socketio.emit("terminal_output", {"line": line_text}) + process.wait() + socketio.emit("terminal_output", {"line": "[TEST HARVEST] Complete."}) + threading.Thread(target=_monitor, daemon=True).start() + + return jsonify({ + "status": "started", + "message": f"Dry-run harvest started for {inst_cfg.name}. Preview will be emailed to {email}.", + "institution": inst_cfg.name, + "count": count, + "email": email, + }), 202 + except Exception as exc: + logger.error("ir_test_harvest: %s", exc) + return jsonify({"status": "error", "message": str(exc)}), 500 + + +@app.route("/api/ir/status") +def ir_status(): + """Check connectivity to the live DSpace IR (no credentials needed for read).""" + from uraas.services.ir_client import DSpaceClient + client = DSpaceClient() + result = client.probe() + return jsonify(result), 200 if result.get("ok") else 503 + + +@app.route("/api/ir/collections") +def ir_collections(): + """List DSpace collections the configured account can submit to.""" + from uraas.services.ir_client import DSpaceClient + try: + client = DSpaceClient() + client.login() + cols = client.get_submittable_collections() + return jsonify({"status": "success", "collections": cols}) + except Exception as exc: + logger.error("ir_collections: %s", exc) + return jsonify({"status": "error", "message": str(exc)}), 500 + + +@app.route("/api/ir/live-stats") +def ir_live_stats(): + """Composite live stats tile pulled directly from the UNILAG DSpace IR.""" + from uraas.services.ir_client import DSpaceClient + try: + client = DSpaceClient() + stats = client.get_live_stats() + return jsonify({"status": "success", "data": stats}) + except Exception as exc: + logger.error("ir_live_stats: %s", exc) + return jsonify({"status": "error", "message": str(exc)}), 500 + + +@app.route("/api/ir/deposit/queue", methods=["POST"]) +def ir_queue_batch(): + """Queue a batch of local items for deposit to the real IR. + + Body JSON: + item_ids – list of local Item.id values to deposit + collection_uuid – DSpace collection UUID (from /api/ir/collections) + collection_name – display name (optional, for the email) + approval_email – address to send the approve/reject email to + """ + from uraas.services.batch_approval import queue_batch + + data = request.get_json(silent=True) or {} + item_ids = data.get("item_ids", []) + collection_uuid = (data.get("collection_uuid") or "").strip() + collection_name = (data.get("collection_name") or "").strip() + approval_email = (data.get("approval_email") or "").strip() + requested_by = session.get("user", "admin") + + if not item_ids: + return jsonify({"status": "error", "message": "item_ids required"}), 400 + if not isinstance(item_ids, list) or not all(isinstance(i, int) for i in item_ids): + return jsonify({"status": "error", "message": "item_ids must be a list of integers"}), 400 + if len(item_ids) > 500: + return jsonify({"status": "error", "message": "Maximum 500 items per batch"}), 400 + if not approval_email or "@" not in approval_email: + return jsonify({"status": "error", "message": "A valid approval_email is required"}), 400 + if not collection_uuid: + return jsonify({"status": "error", "message": "collection_uuid required"}), 400 + + try: + result = queue_batch( + item_ids=item_ids, + collection_uuid=collection_uuid, + collection_name=collection_name, + approval_email=approval_email, + requested_by=requested_by, + ) + return jsonify(result), 201 + except Exception as exc: + logger.error("ir_queue_batch: %s", exc) + return jsonify({"status": "error", "message": str(exc)}), 500 + + +@app.route("/api/ir/batch//approve", methods=["GET"]) +def ir_approve_batch(token): + """Email approval link — no login required; token is the credential. + + Renders a plain HTML confirmation page so it works directly in a browser + after the approver clicks the link in their email. + """ + from uraas.services.batch_approval import approve_batch + + # Minimal token sanity-check (URL-safe base64 chars only) + import re as _re + if not token or not _re.match(r'^[A-Za-z0-9_\-]{10,128}$', token): + return _approval_html("Invalid Link", "The approval link is malformed.", ok=False), 400 + + result = approve_batch(token) + + if result["status"] == "approved": + return _approval_html( + "Deposit Approved", + result["message"], + ok=True, + ), 200 + elif result["status"] == "already_actioned": + return _approval_html( + "Already Actioned", + result["message"], + ok=True, + ), 200 + elif result["status"] == "expired": + return _approval_html("Link Expired", result["message"], ok=False), 410 + else: + return _approval_html("Not Found", result["message"], ok=False), 404 + + +@app.route("/api/ir/batch//reject", methods=["GET"]) +def ir_reject_batch(token): + """Email rejection link — no login required; token is the credential.""" + from uraas.services.batch_approval import reject_batch + + import re as _re + if not token or not _re.match(r'^[A-Za-z0-9_\-]{10,128}$', token): + return _approval_html("Invalid Link", "The rejection link is malformed.", ok=False), 400 + + reason = request.args.get("reason", "Rejected via email link") + result = reject_batch(token, reason=reason) + + if result["status"] == "rejected": + return _approval_html( + "Batch Rejected", + result["message"], + ok=False, + ), 200 + elif result["status"] == "already_actioned": + return _approval_html("Already Actioned", result["message"], ok=True), 200 + else: + return _approval_html("Not Found", result["message"], ok=False), 404 + + +@app.route("/api/ir/batches") +def ir_list_batches(): + """List all deposit batches (admin panel).""" + from uraas.services.batch_approval import get_batches + try: + limit = min(int(request.args.get("limit", 50)), 200) + batches = get_batches(limit=limit) + return jsonify({"status": "success", "batches": batches}) + except Exception as exc: + logger.error("ir_list_batches: %s", exc) + return jsonify({"status": "error", "message": str(exc)}), 500 + + +@app.route("/api/ir/batch//status") +def ir_batch_status(token): + """Get status of a specific batch (admin polling).""" + from uraas.services.batch_approval import get_batch + import re as _re + if not token or not _re.match(r'^[A-Za-z0-9_\-]{10,128}$', token): + return jsonify({"status": "error", "message": "Invalid token"}), 400 + batch = get_batch(token) + if not batch: + return jsonify({"status": "error", "message": "Batch not found"}), 404 + return jsonify({"status": "success", "batch": batch}) + + +def _approval_html(title: str, message: str, ok: bool) -> str: + colour = "#1a7a4a" if ok else "#c0392b" + icon = "✓" if ok else "✗" + return f""" + + + + + URAAS — {title} + + + +
+
{icon}
+

{title}

+

{message}

+ Return to Dashboard +
+ +""" + + +# Error Handlers + + +@app.errorhandler(404) +def not_found_error(error): + """Custom 404 error handler.""" + if request.path.startswith("/api/"): + return jsonify({"error": "Resource not found"}), 404 + return render_template("index.html"), 404 # SPA fallback + + +@app.errorhandler(500) +def internal_error(error): + """Custom 500 error handler.""" + logger.error(f"Internal server error: {str(error)}") + if request.path.startswith("/api/"): + return jsonify({"error": "Internal server error"}), 500 + return jsonify({"error": "Internal server error"}), 500 + + +# Phase 8: /api/version endpoint + + +@app.route("/api/version") +def api_version(): + """Version manifest — commit hash + phase badges for the dashboard UI.""" + return jsonify( + { + "version": os.getenv("RENDER_GIT_COMMIT", "dev")[:8], + "env": "production" if config.is_production() else "development", + "phases_completed": [ + "P4_citation_velocity", + "P5_ark_pids", + "P6_credibility", + "P7_cleanup", + "P8_render_prep", + ], + } + ) + + + +# Run + +if __name__ == "__main__": + # Apply production configuration if on Render + from uraas.production_config import ProductionConfig + + ProductionConfig.apply_config(app) + + # Get port from environment (Render provides this) + port = int(os.getenv("PORT", config.DASHBOARD_PORT)) + + # Determine if running in production + is_production = ProductionConfig.is_production() + + if is_production: + logger.info("=" * 70) + logger.info("URAAS Dashboard Starting (Production Mode)") + logger.info("=" * 70) + logger.info(f"Port: {port}") + logger.info(f"Database: {os.getenv('DATABASE_URL', 'Not configured')[:50]}...") + logger.info(f"Storage: {config.STORAGE_PATH}") + logger.info(f"Health check: http://0.0.0.0:{port}/health") + logger.info("=" * 70) + else: + logger.info("=" * 70) + logger.info("URAAS Dashboard Starting (Development Mode)") + logger.info("=" * 70) + logger.info(f"Dashboard URL: http://localhost:{port}") + logger.info("Press Ctrl+C to stop") + logger.info("=" * 70) + + # Run with SocketIO + socketio.run( + app, + host="0.0.0.0", + port=port, + debug=not is_production, + use_reloader=not is_production, + ) diff --git a/uraas/dashboard/auth.py b/uraas/dashboard/auth.py index a117649e998f2697487bcc916cf0a76123d98c25..aaaa94888423d06e831489fa03fe4205aaef6a42 100644 --- a/uraas/dashboard/auth.py +++ b/uraas/dashboard/auth.py @@ -1,103 +1,103 @@ -""" -Lightweight authentication for the URAAS dashboard. - -Zero external auth dependencies: Flask `session` cookies (signed with -DASHBOARD_SECRET_KEY) + Werkzeug password hashing. Two roles: - - admin — full control: crawler, mutations, bulk exports, staff directory, - and download of any stored file regardless of rights. - viewer — read access to the dashboard and analytics; may download only - open-access files. - -Credentials come from the environment (see uraas.config.Config). Passwords are -stored as Werkzeug hashes, never plaintext. This is an interim layer; the -deployment plan replaces/augments it with institutional SSO (Shibboleth/LDAP). -""" - -import functools - -from flask import jsonify, redirect, request, session, url_for -from werkzeug.security import check_password_hash - -from uraas.config import config - -ADMIN = "admin" -VIEWER = "viewer" - - -def check_credentials(username: str, password: str): - """Return the role string for valid credentials, else None. - - Constant-ish: always runs a hash comparison against the matching user's - stored hash. Unknown users / empty hashes return None. - """ - if not username or not password: - return None - candidates = ( - (config.ADMIN_USERNAME, config.ADMIN_PASSWORD_HASH, ADMIN), - (config.VIEWER_USERNAME, config.VIEWER_PASSWORD_HASH, VIEWER), - ) - for user, pw_hash, role in candidates: - if username == user and pw_hash and check_password_hash(pw_hash, password): - return role - return None - - -def current_role(): - """Role of the logged-in user, or None if unauthenticated.""" - return session.get("role") - - -def _wants_json() -> bool: - """True for API/XHR requests, which should get a 401 rather than a redirect.""" - return request.path.startswith("/api/") or request.accept_mimetypes.best == ( - "application/json" - ) - - -def login_required(view): - """Require any authenticated user (viewer or admin).""" - - @functools.wraps(view) - def wrapped(*args, **kwargs): - if not current_role(): - if _wants_json(): - return jsonify({"status": "error", "message": "Authentication required"}), 401 - return redirect(url_for("login", next=request.path)) - return view(*args, **kwargs) - - return wrapped - - -def admin_required(view): - """Require an authenticated admin. 401 if anonymous, 403 if a viewer.""" - - @functools.wraps(view) - def wrapped(*args, **kwargs): - role = current_role() - if not role: - if _wants_json(): - return jsonify({"status": "error", "message": "Authentication required"}), 401 - return redirect(url_for("login", next=request.path)) - if role != ADMIN: - return jsonify({"status": "error", "message": "Administrator access required"}), 403 - return view(*args, **kwargs) - - return wrapped - - -def clamped_int(name: str, default: int, lo: int, hi: int) -> int: - """Read an int query param, defaulting and clamping to [lo, hi]. - - Never raises on bad input (returns default), so endpoints can't be DoS'd - with huge limits or 500'd with non-numeric values. - """ - raw = request.args.get(name) - if raw is None or raw == "": - value = default - else: - try: - value = int(raw) - except (TypeError, ValueError): - value = default - return max(lo, min(value, hi)) +""" +Lightweight authentication for the URAAS dashboard. + +Zero external auth dependencies: Flask `session` cookies (signed with +DASHBOARD_SECRET_KEY) + Werkzeug password hashing. Two roles: + + admin — full control: crawler, mutations, bulk exports, staff directory, + and download of any stored file regardless of rights. + viewer — read access to the dashboard and analytics; may download only + open-access files. + +Credentials come from the environment (see uraas.config.Config). Passwords are +stored as Werkzeug hashes, never plaintext. This is an interim layer; the +deployment plan replaces/augments it with institutional SSO (Shibboleth/LDAP). +""" + +import functools + +from flask import jsonify, redirect, request, session, url_for +from werkzeug.security import check_password_hash + +from uraas.config import config + +ADMIN = "admin" +VIEWER = "viewer" + + +def check_credentials(username: str, password: str): + """Return the role string for valid credentials, else None. + + Constant-ish: always runs a hash comparison against the matching user's + stored hash. Unknown users / empty hashes return None. + """ + if not username or not password: + return None + candidates = ( + (config.ADMIN_USERNAME, config.ADMIN_PASSWORD_HASH, ADMIN), + (config.VIEWER_USERNAME, config.VIEWER_PASSWORD_HASH, VIEWER), + ) + for user, pw_hash, role in candidates: + if username == user and pw_hash and check_password_hash(pw_hash, password): + return role + return None + + +def current_role(): + """Role of the logged-in user, or None if unauthenticated.""" + return session.get("role") + + +def _wants_json() -> bool: + """True for API/XHR requests, which should get a 401 rather than a redirect.""" + return request.path.startswith("/api/") or request.accept_mimetypes.best == ( + "application/json" + ) + + +def login_required(view): + """Require any authenticated user (viewer or admin).""" + + @functools.wraps(view) + def wrapped(*args, **kwargs): + if not current_role(): + if _wants_json(): + return jsonify({"status": "error", "message": "Authentication required"}), 401 + return redirect(url_for("login", next=request.path)) + return view(*args, **kwargs) + + return wrapped + + +def admin_required(view): + """Require an authenticated admin. 401 if anonymous, 403 if a viewer.""" + + @functools.wraps(view) + def wrapped(*args, **kwargs): + role = current_role() + if not role: + if _wants_json(): + return jsonify({"status": "error", "message": "Authentication required"}), 401 + return redirect(url_for("login", next=request.path)) + if role != ADMIN: + return jsonify({"status": "error", "message": "Administrator access required"}), 403 + return view(*args, **kwargs) + + return wrapped + + +def clamped_int(name: str, default: int, lo: int, hi: int) -> int: + """Read an int query param, defaulting and clamping to [lo, hi]. + + Never raises on bad input (returns default), so endpoints can't be DoS'd + with huge limits or 500'd with non-numeric values. + """ + raw = request.args.get(name) + if raw is None or raw == "": + value = default + else: + try: + value = int(raw) + except (TypeError, ValueError): + value = default + return max(lo, min(value, hi)) diff --git a/uraas/dashboard/responses.py b/uraas/dashboard/responses.py index 616ae0147859ed14579b47e76368fbc3ad1366a5..8ecc9d1037db02f674ebbf2882c8f44c03f84240 100644 --- a/uraas/dashboard/responses.py +++ b/uraas/dashboard/responses.py @@ -1,72 +1,72 @@ -""" -Standard API response helpers for new URAAS endpoints. - -Envelope shape: - {"status": "success", "data": ..., "narrative": "...", - "methodology": {...}, "benchmark": {...}} - -Older endpoints keep their bare payloads until migrated (their JS consumers -parse those shapes directly); all NEW endpoints must use api_ok/api_error. -""" - -import csv -import io -from typing import Iterable, Optional - -from flask import Response, jsonify, stream_with_context - - -def api_ok( - data, - *, - narrative: Optional[str] = None, - methodology_key: Optional[str] = None, - benchmark: Optional[dict] = None, - **extra, -) -> Response: - """Standard success envelope. methodology_key is resolved from - uraas.config.methodology.METHODOLOGY so the chart tooltip and the - endpoint always agree.""" - payload = {"status": "success", "data": data} - if narrative: - payload["narrative"] = narrative - if methodology_key: - from uraas.config.methodology import METHODOLOGY - - method = METHODOLOGY.get(methodology_key) - if method: - payload["methodology"] = {"key": methodology_key, **method} - if benchmark is None: - benchmark = method.get("benchmark") - if benchmark: - payload["benchmark"] = benchmark - payload.update(extra) - return jsonify(payload) - - -def api_error(message: str, status: int = 500, **extra) -> Response: - payload = {"status": "error", "message": message} - payload.update(extra) - resp = jsonify(payload) - resp.status_code = status - return resp - - -def csv_response(rows: Iterable[list], filename: str) -> Response: - """Streaming CSV download. `rows` is any iterable of lists (header first); - pass a generator for large exports so nothing is held in memory.""" - - def generate(): - buf = io.StringIO() - writer = csv.writer(buf) - for row in rows: - writer.writerow(row) - yield buf.getvalue() - buf.seek(0) - buf.truncate(0) - - return Response( - stream_with_context(generate()), - mimetype="text/csv", - headers={"Content-Disposition": f"attachment; filename={filename}"}, - ) +""" +Standard API response helpers for new URAAS endpoints. + +Envelope shape: + {"status": "success", "data": ..., "narrative": "...", + "methodology": {...}, "benchmark": {...}} + +Older endpoints keep their bare payloads until migrated (their JS consumers +parse those shapes directly); all NEW endpoints must use api_ok/api_error. +""" + +import csv +import io +from typing import Iterable, Optional + +from flask import Response, jsonify, stream_with_context + + +def api_ok( + data, + *, + narrative: Optional[str] = None, + methodology_key: Optional[str] = None, + benchmark: Optional[dict] = None, + **extra, +) -> Response: + """Standard success envelope. methodology_key is resolved from + uraas.config.methodology.METHODOLOGY so the chart tooltip and the + endpoint always agree.""" + payload = {"status": "success", "data": data} + if narrative: + payload["narrative"] = narrative + if methodology_key: + from uraas.config.methodology import METHODOLOGY + + method = METHODOLOGY.get(methodology_key) + if method: + payload["methodology"] = {"key": methodology_key, **method} + if benchmark is None: + benchmark = method.get("benchmark") + if benchmark: + payload["benchmark"] = benchmark + payload.update(extra) + return jsonify(payload) + + +def api_error(message: str, status: int = 500, **extra) -> Response: + payload = {"status": "error", "message": message} + payload.update(extra) + resp = jsonify(payload) + resp.status_code = status + return resp + + +def csv_response(rows: Iterable[list], filename: str) -> Response: + """Streaming CSV download. `rows` is any iterable of lists (header first); + pass a generator for large exports so nothing is held in memory.""" + + def generate(): + buf = io.StringIO() + writer = csv.writer(buf) + for row in rows: + writer.writerow(row) + yield buf.getvalue() + buf.seek(0) + buf.truncate(0) + + return Response( + stream_with_context(generate()), + mimetype="text/csv", + headers={"Content-Disposition": f"attachment; filename={filename}"}, + ) diff --git a/uraas/database.py b/uraas/database.py index 1854b524dca28b90f246c355c1cd1277e496c415..c98810cff4338ededa430f3eb23eabccd6e2be03 100644 --- a/uraas/database.py +++ b/uraas/database.py @@ -1,370 +1,370 @@ -""" -URAAS Database Models -Supports the APA Intelligence & Analytics Platform: -- Dublin Core metadata (DSpace-compatible) -- DocID™ persistent identifiers (Africa PID Alliance) -- ORCID / ROR integration -- TK (Traditional Knowledge) labels for indigenous content -- Linguistic metadata for Diversity Index -""" - -from datetime import datetime - -from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Index, Integer -from sqlalchemy import String -from sqlalchemy import String as SAString -from sqlalchemy import Table, Text, cast, create_engine, extract, func -from sqlalchemy.orm import declarative_base, relationship, sessionmaker - -from uraas.config import config - - -# Cache the dialect at import time — URL cannot change at runtime. -_IS_SQLITE: bool = (config.DATABASE_URL or "").lower().startswith("sqlite") - - -def db_year(col): - """Cross-dialect YEAR() extraction returning a string ('2024'). - - SQLite has no extract(year) for TEXT-stored datetimes when the column - was populated from ISO strings, so we use strftime there. Postgres - rejects strftime, so we use extract(year). - """ - if _IS_SQLITE: - return func.strftime("%Y", col) - return cast(extract("year", col), SAString) - - -def db_year_month(col): - """Cross-dialect YEAR-MONTH ('2024-03').""" - if _IS_SQLITE: - return func.strftime("%Y-%m", col) - return func.to_char(col, "YYYY-MM") - - -Base = declarative_base() - -# ── Association Tables ──────────────────────────────────────────────────────── - -item_authors = Table( - "item_authors", - Base.metadata, - Column( - "item_id", Integer, ForeignKey("items.id", ondelete="CASCADE"), primary_key=True - ), - Column( - "author_id", - Integer, - ForeignKey("authors.id", ondelete="CASCADE"), - primary_key=True, - ), -) - -item_collections = Table( - "item_collections", - Base.metadata, - Column( - "item_id", Integer, ForeignKey("items.id", ondelete="CASCADE"), primary_key=True - ), - Column( - "collection_id", - Integer, - ForeignKey("collections.id", ondelete="CASCADE"), - primary_key=True, - ), - Column("confidence_score", Float, default=1.0), -) - -# ── Core Models ─────────────────────────────────────────────────────────────── - - -class Community(Base): - """Faculty / School — top-level organisational unit.""" - - __tablename__ = "communities" - - id = Column(Integer, primary_key=True) - name = Column(String(255), unique=True, nullable=False) - - # ── APA / ROR ───────────────────────────────────────────────────────────── - ror_id = Column(String(128)) # e.g. https://ror.org/03qcnxw14 - institution = Column(String(255)) # parent institution name - ror = Column(String(128)) # Institution ROR for multi-tenant comparison - - collections = relationship("Collection", back_populates="community") - - -class Collection(Base): - """Department / Research Group — second-level unit.""" - - __tablename__ = "collections" - - id = Column(Integer, primary_key=True) - community_id = Column(Integer, ForeignKey("communities.id"), nullable=False) - name = Column(String(255), unique=True, nullable=False) - email_domains = Column(Text) # comma-separated - keywords = Column(Text) # comma-separated - - community = relationship("Community", back_populates="collections") - items = relationship( - "Item", secondary=item_collections, back_populates="collections" - ) - - -class Author(Base): - """Researcher / Creator.""" - - __tablename__ = "authors" - - id = Column(Integer, primary_key=True) - name = Column(String(255), nullable=False) - normalized_name = Column(String(255), nullable=False, index=True) - profile_url = Column(String(512)) - - # PID integrations - orcid = Column(String(64)) # e.g. 0000-0002-1825-0097 - ror = Column(String(128)) # institutional ROR - - items = relationship("Item", secondary=item_authors, back_populates="authors") - - -class Item(Base): - """ - Research output — paper, thesis, dataset, cultural artefact, etc. - Stores full Dublin Core + DocID™ + APA-specific metadata. - """ - - __tablename__ = "items" - - id = Column(Integer, primary_key=True) - title = Column(String(512), nullable=False) - abstract = Column(Text) - doi = Column(String(255), unique=True) - publication_date = Column(DateTime) - url = Column(String(512), unique=True) - source_repository = Column(String(100)) - pdf_url = Column(String(512)) - - # ── Dublin Core ─────────────────────────────────────────────────────────── - dc_title = Column(String(512)) - dc_date_issued = Column(String(50)) - dc_identifier_uri = Column(String(512)) - dc_identifier_doi = Column(String(255)) - dc_description_provenance = Column(Text) - dc_rights = Column(String(255), default="info:eu-repo/semantics/restrictedAccess") - dc_type = Column(String(100)) # Article, Thesis, Dataset, CulturalHeritage … - dc_language = Column(String(50)) # ISO 639-1 code, e.g. "en", "yo", "ig" - dc_subject = Column(Text) # comma-separated subject tags - - # ── DocID™ (Africa PID Alliance) ───────────────────────────────────────── - docid = Column(String(128), unique=True, index=True) # 20.500.14351/[hash] - docid_assigned_at = Column(DateTime) - - # ── APA-specific fields ─────────────────────────────────────────────────── - # Institution ROR for multi-tenant comparison - ror = Column(String(128), index=True) # e.g. https://ror.org/03qcnxw14 - institution = Column(String(255)) # Institution name - - # Content type for TK Vitality Score - content_type = Column(String(50), default="research_paper") - # Values: research_paper | thesis | patent | indigenous_knowledge | - # cultural_heritage | oral_tradition | dataset | grey_literature - - # Traditional Knowledge labels (CARE principles) - tk_label = Column(String(100)) # e.g. "TK Attribution", "TK Non-Commercial" - tk_community = Column(String(255)) # originating community - - # Patent linkage (Patent-to-Paper Velocity) - patent_id = Column(String(128)) - patent_date = Column(DateTime) - - # Language metadata (Linguistic Diversity Index) - language_code = Column(String(10)) # ISO 639-1: "en", "yo", "ig", "ha", "sw" … - is_african_language = Column(Boolean, default=False) - - # SDG alignment (comma-separated SDG numbers, e.g. "3,4,13") - sdg_tags = Column(Text) - - # AI-extracted keywords (comma-separated) - ai_keywords = Column(Text) - - # Special Collections weighting (computed by classify_special_collections). - # score = sum of (matched_keywords * 3) across all SC categories; 0 = not SC. - # categories = comma-separated category names with hits, e.g. "Indigenous Knowledge,Cultural Heritage". - special_collection_score = Column(Float, default=0.0, index=True) - special_collection_categories = Column(Text) - - # ── Framework alignment (AU charters / Agenda 2063 / regional blocs) ───── - # JSON: {"banjul": {"overall": 42.1, "pillars": {"civil_political_rights": - # {"score": 61.0, "semantic": 0.55, "keyword": 0.71, - # "matched_keywords": ["human rights", ...]}}}, ...} - alignment_scores = Column(Text) - alignment_version = Column(Integer, default=0) - - # ── Intra-African collaboration (from OpenAlex authorships) ────────────── - coauthor_countries = Column(Text) # sorted ISO2 csv, e.g. "KE,NG,ZA" - african_country_count = Column(Integer, default=0) - is_intra_african = Column(Boolean, default=False, index=True) - - # ── Citation velocity (OpenAlex counts_by_year) ────────────────────────── - openalex_id = Column(String(64)) # e.g. "W2741809807" - counts_by_year = Column(Text) # JSON [{"year": 2023, "cited_by_count": 4}, ...] - cited_by_count = Column(Integer, default=0) - african_citation_share = Column(Float) # 0-100; NULL = not yet computed - - # ── ARK persistent identifier (Archival Resource Key) ──────────────────── - ark = Column(String(128)) # e.g. "ark:/99999/u1x7kq2m9b4cz" (unique index below) - ark_assigned_at = Column(DateTime) - - created_at = Column(DateTime, default=datetime.utcnow) - - authors = relationship("Author", secondary=item_authors, back_populates="items") - collections = relationship( - "Collection", secondary=item_collections, back_populates="items" - ) - files = relationship("File", back_populates="item", cascade="all, delete-orphan") - - -class File(Base): - """Local PDF bitstream.""" - - __tablename__ = "files" - - id = Column(Integer, primary_key=True) - item_id = Column( - Integer, ForeignKey("items.id", ondelete="CASCADE"), nullable=False - ) - file_path = Column(String(512), nullable=False) - sha256_hash = Column(String(128)) - access_policy = Column(String(50), default="Private") - downloaded_at = Column(DateTime, default=datetime.utcnow) - - item = relationship("Item", back_populates="files") - - -class ItemAffiliation(Base): - """One row per (item, institution) authorship affiliation from OpenAlex. - - Feeds the country-pair collaboration matrix, the Africa choropleth and - institution-level collaboration networks. - """ - - __tablename__ = "item_affiliations" - - id = Column(Integer, primary_key=True) - item_id = Column( - Integer, ForeignKey("items.id", ondelete="CASCADE"), index=True, nullable=False - ) - ror = Column(String(128), index=True) # short form, e.g. "05rk03822" - institution_name = Column(String(255)) - country_code = Column(String(2), index=True) # ISO2 from OpenAlex - author_count = Column(Integer, default=1) # authors at this institution on this paper - - -class AlignmentAggregate(Base): - """Precomputed per-institution framework/pillar alignment averages. - - Recomputed by scripts/backfill_alignment.py and the post-crawl hook; - read directly by the radar/matrix/gap endpoints. - """ - - __tablename__ = "alignment_aggregates" - - id = Column(Integer, primary_key=True) - institution = Column(String(255), index=True) # full name; "" = all institutions - framework = Column(String(64), index=True) # e.g. "banjul", "agenda2063" - pillar = Column(String(64)) # pillar key - avg_score = Column(Float, default=0.0) - paper_count = Column(Integer, default=0) # papers with pillar score >= threshold - top_item_ids = Column(Text) # csv of top-5 item ids (evidence chips) - computed_at = Column(DateTime, default=datetime.utcnow) - - -class CrawlJob(Base): - """Tracks crawler sessions for provenance and growth-rate charts.""" - - __tablename__ = "crawl_jobs" - - id = Column(Integer, primary_key=True) - source_name = Column(String(100), nullable=False) - status = Column(String(50), default="PENDING") - items_scraped = Column(Integer, default=0) - started_at = Column(DateTime, default=datetime.utcnow) - ended_at = Column(DateTime) - - -class DepositBatch(Base): - """Tracks a staged batch of items queued for deposit to the real DSpace IR. - - Flow: pending_approval → approved/rejected → depositing → completed/failed. - An approval token is emailed to the address the admin typed in; only - clicking the link in that email advances the batch to 'approved'. - """ - - __tablename__ = "deposit_batches" - - id = Column(Integer, primary_key=True) - # Cryptographically random URL-safe token that authorises this specific batch. - # Stored as-is (64 hex chars); treated as a one-time-use secret. - token = Column(String(128), unique=True, index=True, nullable=False) - - # Lifecycle status - status = Column(String(30), default="pending_approval", nullable=False, index=True) - # Values: pending_approval | approved | rejected | depositing | completed | failed - - approval_email = Column(String(255), nullable=False) - collection_uuid = Column(String(128)) # DSpace target collection UUID - collection_name = Column(String(255)) # for display only - - # JSON array of local Item.id values to deposit, e.g. [1, 7, 42] - item_ids_json = Column(Text, nullable=False, default="[]") - item_count = Column(Integer, default=0) - - deposited_count = Column(Integer, default=0) - failed_count = Column(Integer, default=0) - - notes = Column(Text) # rejection reason, or first fatal error - requested_by = Column(String(100)) # session username - - created_at = Column(DateTime, default=datetime.utcnow) - updated_at = Column(DateTime, default=datetime.utcnow) - expires_at = Column(DateTime) # approval link expires after 48 h - approved_at = Column(DateTime) - completed_at = Column(DateTime) - - # JSON array of per-item results: [{"item_id": 1, "status": "ok", "dspace_id": "..."}, ...] - deposit_log = Column(Text, default="[]") - - -# ── Indexes for query performance ───────────────────────────────────────────── -# ix_items_docid is auto-created by index=True on Item.docid — no duplicate needed -Index("ix_items_language", Item.language_code) -Index("ix_items_content_type", Item.content_type) -Index("ix_items_created_at", Item.created_at) -Index("ix_authors_orcid", Author.orcid) -Index("ux_items_ark", Item.ark, unique=True) -Index( - "ix_item_aff_item_country", - ItemAffiliation.item_id, - ItemAffiliation.country_code, -) - - -# ── Engine & Session ────────────────────────────────────────────────────────── -def _build_engine(): - """SQLite needs check_same_thread=False; Postgres rejects that arg.""" - url = config.DATABASE_URL - # Render exposes postgres:// but SQLAlchemy 2.x wants postgresql:// - if url.startswith("postgres://"): - url = url.replace("postgres://", "postgresql://", 1) - if url.startswith("sqlite"): - return create_engine(url, connect_args={"check_same_thread": False}) - return create_engine(url, pool_pre_ping=True, pool_recycle=3600) - - -engine = _build_engine() -SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) - - -def init_db(): - Base.metadata.create_all(bind=engine, checkfirst=True) +""" +URAAS Database Models +Supports the APA Intelligence & Analytics Platform: +- Dublin Core metadata (DSpace-compatible) +- DocID™ persistent identifiers (Africa PID Alliance) +- ORCID / ROR integration +- TK (Traditional Knowledge) labels for indigenous content +- Linguistic metadata for Diversity Index +""" + +from datetime import datetime + +from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Index, Integer +from sqlalchemy import String +from sqlalchemy import String as SAString +from sqlalchemy import Table, Text, cast, create_engine, extract, func +from sqlalchemy.orm import declarative_base, relationship, sessionmaker + +from uraas.config import config + + +# Cache the dialect at import time — URL cannot change at runtime. +_IS_SQLITE: bool = (config.DATABASE_URL or "").lower().startswith("sqlite") + + +def db_year(col): + """Cross-dialect YEAR() extraction returning a string ('2024'). + + SQLite has no extract(year) for TEXT-stored datetimes when the column + was populated from ISO strings, so we use strftime there. Postgres + rejects strftime, so we use extract(year). + """ + if _IS_SQLITE: + return func.strftime("%Y", col) + return cast(extract("year", col), SAString) + + +def db_year_month(col): + """Cross-dialect YEAR-MONTH ('2024-03').""" + if _IS_SQLITE: + return func.strftime("%Y-%m", col) + return func.to_char(col, "YYYY-MM") + + +Base = declarative_base() + +# ── Association Tables ──────────────────────────────────────────────────────── + +item_authors = Table( + "item_authors", + Base.metadata, + Column( + "item_id", Integer, ForeignKey("items.id", ondelete="CASCADE"), primary_key=True + ), + Column( + "author_id", + Integer, + ForeignKey("authors.id", ondelete="CASCADE"), + primary_key=True, + ), +) + +item_collections = Table( + "item_collections", + Base.metadata, + Column( + "item_id", Integer, ForeignKey("items.id", ondelete="CASCADE"), primary_key=True + ), + Column( + "collection_id", + Integer, + ForeignKey("collections.id", ondelete="CASCADE"), + primary_key=True, + ), + Column("confidence_score", Float, default=1.0), +) + +# ── Core Models ─────────────────────────────────────────────────────────────── + + +class Community(Base): + """Faculty / School — top-level organisational unit.""" + + __tablename__ = "communities" + + id = Column(Integer, primary_key=True) + name = Column(String(255), unique=True, nullable=False) + + # ── APA / ROR ───────────────────────────────────────────────────────────── + ror_id = Column(String(128)) # e.g. https://ror.org/03qcnxw14 + institution = Column(String(255)) # parent institution name + ror = Column(String(128)) # Institution ROR for multi-tenant comparison + + collections = relationship("Collection", back_populates="community") + + +class Collection(Base): + """Department / Research Group — second-level unit.""" + + __tablename__ = "collections" + + id = Column(Integer, primary_key=True) + community_id = Column(Integer, ForeignKey("communities.id"), nullable=False) + name = Column(String(255), unique=True, nullable=False) + email_domains = Column(Text) # comma-separated + keywords = Column(Text) # comma-separated + + community = relationship("Community", back_populates="collections") + items = relationship( + "Item", secondary=item_collections, back_populates="collections" + ) + + +class Author(Base): + """Researcher / Creator.""" + + __tablename__ = "authors" + + id = Column(Integer, primary_key=True) + name = Column(String(255), nullable=False) + normalized_name = Column(String(255), nullable=False, index=True) + profile_url = Column(String(512)) + + # PID integrations + orcid = Column(String(64)) # e.g. 0000-0002-1825-0097 + ror = Column(String(128)) # institutional ROR + + items = relationship("Item", secondary=item_authors, back_populates="authors") + + +class Item(Base): + """ + Research output — paper, thesis, dataset, cultural artefact, etc. + Stores full Dublin Core + DocID™ + APA-specific metadata. + """ + + __tablename__ = "items" + + id = Column(Integer, primary_key=True) + title = Column(String(512), nullable=False) + abstract = Column(Text) + doi = Column(String(255), unique=True) + publication_date = Column(DateTime) + url = Column(String(512), unique=True) + source_repository = Column(String(100)) + pdf_url = Column(String(512)) + + # ── Dublin Core ─────────────────────────────────────────────────────────── + dc_title = Column(String(512)) + dc_date_issued = Column(String(50)) + dc_identifier_uri = Column(String(512)) + dc_identifier_doi = Column(String(255)) + dc_description_provenance = Column(Text) + dc_rights = Column(String(255), default="info:eu-repo/semantics/restrictedAccess") + dc_type = Column(String(100)) # Article, Thesis, Dataset, CulturalHeritage … + dc_language = Column(String(50)) # ISO 639-1 code, e.g. "en", "yo", "ig" + dc_subject = Column(Text) # comma-separated subject tags + + # ── DocID™ (Africa PID Alliance) ───────────────────────────────────────── + docid = Column(String(128), unique=True, index=True) # 20.500.14351/[hash] + docid_assigned_at = Column(DateTime) + + # ── APA-specific fields ─────────────────────────────────────────────────── + # Institution ROR for multi-tenant comparison + ror = Column(String(128), index=True) # e.g. https://ror.org/03qcnxw14 + institution = Column(String(255)) # Institution name + + # Content type for TK Vitality Score + content_type = Column(String(50), default="research_paper") + # Values: research_paper | thesis | patent | indigenous_knowledge | + # cultural_heritage | oral_tradition | dataset | grey_literature + + # Traditional Knowledge labels (CARE principles) + tk_label = Column(String(100)) # e.g. "TK Attribution", "TK Non-Commercial" + tk_community = Column(String(255)) # originating community + + # Patent linkage (Patent-to-Paper Velocity) + patent_id = Column(String(128)) + patent_date = Column(DateTime) + + # Language metadata (Linguistic Diversity Index) + language_code = Column(String(10)) # ISO 639-1: "en", "yo", "ig", "ha", "sw" … + is_african_language = Column(Boolean, default=False) + + # SDG alignment (comma-separated SDG numbers, e.g. "3,4,13") + sdg_tags = Column(Text) + + # AI-extracted keywords (comma-separated) + ai_keywords = Column(Text) + + # Special Collections weighting (computed by classify_special_collections). + # score = sum of (matched_keywords * 3) across all SC categories; 0 = not SC. + # categories = comma-separated category names with hits, e.g. "Indigenous Knowledge,Cultural Heritage". + special_collection_score = Column(Float, default=0.0, index=True) + special_collection_categories = Column(Text) + + # ── Framework alignment (AU charters / Agenda 2063 / regional blocs) ───── + # JSON: {"banjul": {"overall": 42.1, "pillars": {"civil_political_rights": + # {"score": 61.0, "semantic": 0.55, "keyword": 0.71, + # "matched_keywords": ["human rights", ...]}}}, ...} + alignment_scores = Column(Text) + alignment_version = Column(Integer, default=0) + + # ── Intra-African collaboration (from OpenAlex authorships) ────────────── + coauthor_countries = Column(Text) # sorted ISO2 csv, e.g. "KE,NG,ZA" + african_country_count = Column(Integer, default=0) + is_intra_african = Column(Boolean, default=False, index=True) + + # ── Citation velocity (OpenAlex counts_by_year) ────────────────────────── + openalex_id = Column(String(64)) # e.g. "W2741809807" + counts_by_year = Column(Text) # JSON [{"year": 2023, "cited_by_count": 4}, ...] + cited_by_count = Column(Integer, default=0) + african_citation_share = Column(Float) # 0-100; NULL = not yet computed + + # ── ARK persistent identifier (Archival Resource Key) ──────────────────── + ark = Column(String(128)) # e.g. "ark:/99999/u1x7kq2m9b4cz" (unique index below) + ark_assigned_at = Column(DateTime) + + created_at = Column(DateTime, default=datetime.utcnow) + + authors = relationship("Author", secondary=item_authors, back_populates="items") + collections = relationship( + "Collection", secondary=item_collections, back_populates="items" + ) + files = relationship("File", back_populates="item", cascade="all, delete-orphan") + + +class File(Base): + """Local PDF bitstream.""" + + __tablename__ = "files" + + id = Column(Integer, primary_key=True) + item_id = Column( + Integer, ForeignKey("items.id", ondelete="CASCADE"), nullable=False + ) + file_path = Column(String(512), nullable=False) + sha256_hash = Column(String(128)) + access_policy = Column(String(50), default="Private") + downloaded_at = Column(DateTime, default=datetime.utcnow) + + item = relationship("Item", back_populates="files") + + +class ItemAffiliation(Base): + """One row per (item, institution) authorship affiliation from OpenAlex. + + Feeds the country-pair collaboration matrix, the Africa choropleth and + institution-level collaboration networks. + """ + + __tablename__ = "item_affiliations" + + id = Column(Integer, primary_key=True) + item_id = Column( + Integer, ForeignKey("items.id", ondelete="CASCADE"), index=True, nullable=False + ) + ror = Column(String(128), index=True) # short form, e.g. "05rk03822" + institution_name = Column(String(255)) + country_code = Column(String(2), index=True) # ISO2 from OpenAlex + author_count = Column(Integer, default=1) # authors at this institution on this paper + + +class AlignmentAggregate(Base): + """Precomputed per-institution framework/pillar alignment averages. + + Recomputed by scripts/backfill_alignment.py and the post-crawl hook; + read directly by the radar/matrix/gap endpoints. + """ + + __tablename__ = "alignment_aggregates" + + id = Column(Integer, primary_key=True) + institution = Column(String(255), index=True) # full name; "" = all institutions + framework = Column(String(64), index=True) # e.g. "banjul", "agenda2063" + pillar = Column(String(64)) # pillar key + avg_score = Column(Float, default=0.0) + paper_count = Column(Integer, default=0) # papers with pillar score >= threshold + top_item_ids = Column(Text) # csv of top-5 item ids (evidence chips) + computed_at = Column(DateTime, default=datetime.utcnow) + + +class CrawlJob(Base): + """Tracks crawler sessions for provenance and growth-rate charts.""" + + __tablename__ = "crawl_jobs" + + id = Column(Integer, primary_key=True) + source_name = Column(String(100), nullable=False) + status = Column(String(50), default="PENDING") + items_scraped = Column(Integer, default=0) + started_at = Column(DateTime, default=datetime.utcnow) + ended_at = Column(DateTime) + + +class DepositBatch(Base): + """Tracks a staged batch of items queued for deposit to the real DSpace IR. + + Flow: pending_approval → approved/rejected → depositing → completed/failed. + An approval token is emailed to the address the admin typed in; only + clicking the link in that email advances the batch to 'approved'. + """ + + __tablename__ = "deposit_batches" + + id = Column(Integer, primary_key=True) + # Cryptographically random URL-safe token that authorises this specific batch. + # Stored as-is (64 hex chars); treated as a one-time-use secret. + token = Column(String(128), unique=True, index=True, nullable=False) + + # Lifecycle status + status = Column(String(30), default="pending_approval", nullable=False, index=True) + # Values: pending_approval | approved | rejected | depositing | completed | failed + + approval_email = Column(String(255), nullable=False) + collection_uuid = Column(String(128)) # DSpace target collection UUID + collection_name = Column(String(255)) # for display only + + # JSON array of local Item.id values to deposit, e.g. [1, 7, 42] + item_ids_json = Column(Text, nullable=False, default="[]") + item_count = Column(Integer, default=0) + + deposited_count = Column(Integer, default=0) + failed_count = Column(Integer, default=0) + + notes = Column(Text) # rejection reason, or first fatal error + requested_by = Column(String(100)) # session username + + created_at = Column(DateTime, default=datetime.utcnow) + updated_at = Column(DateTime, default=datetime.utcnow) + expires_at = Column(DateTime) # approval link expires after 48 h + approved_at = Column(DateTime) + completed_at = Column(DateTime) + + # JSON array of per-item results: [{"item_id": 1, "status": "ok", "dspace_id": "..."}, ...] + deposit_log = Column(Text, default="[]") + + +# ── Indexes for query performance ───────────────────────────────────────────── +# ix_items_docid is auto-created by index=True on Item.docid — no duplicate needed +Index("ix_items_language", Item.language_code) +Index("ix_items_content_type", Item.content_type) +Index("ix_items_created_at", Item.created_at) +Index("ix_authors_orcid", Author.orcid) +Index("ux_items_ark", Item.ark, unique=True) +Index( + "ix_item_aff_item_country", + ItemAffiliation.item_id, + ItemAffiliation.country_code, +) + + +# ── Engine & Session ────────────────────────────────────────────────────────── +def _build_engine(): + """SQLite needs check_same_thread=False; Postgres rejects that arg.""" + url = config.DATABASE_URL + # Render exposes postgres:// but SQLAlchemy 2.x wants postgresql:// + if url.startswith("postgres://"): + url = url.replace("postgres://", "postgresql://", 1) + if url.startswith("sqlite"): + return create_engine(url, connect_args={"check_same_thread": False}) + return create_engine(url, pool_pre_ping=True, pool_recycle=3600) + + +engine = _build_engine() +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + +def init_db(): + Base.metadata.create_all(bind=engine, checkfirst=True) diff --git a/uraas/middlewares/__init__.py b/uraas/middlewares/__init__.py index 6fb2ee526ee615c1d57d3034c12613dda8bb6adb..8620ef2f6450493318715b048e0fece7c36519f6 100644 --- a/uraas/middlewares/__init__.py +++ b/uraas/middlewares/__init__.py @@ -1 +1 @@ -# Scrapy middlewares +# Scrapy middlewares diff --git a/uraas/middlewares/retry_middleware.py b/uraas/middlewares/retry_middleware.py index 4895bbdd2dfd57c2f3ef82ba577257c26b5492ca..8562f37aaa2bb23976f23926f2e11510db8b6cb9 100644 --- a/uraas/middlewares/retry_middleware.py +++ b/uraas/middlewares/retry_middleware.py @@ -1,32 +1,32 @@ -""" -Enhanced retry middleware with exponential backoff. -""" - -import time - -from scrapy.downloadermiddlewares.retry import RetryMiddleware -from scrapy.utils.response import response_status_message - - -class EnhancedRetryMiddleware(RetryMiddleware): - """Retry middleware with exponential backoff and better logging.""" - - def process_response(self, request, response, spider): - if request.meta.get("dont_retry", False): - return response - - if response.status in self.retry_http_codes: - reason = response_status_message(response.status) - retry_times = request.meta.get("retry_times", 0) + 1 - - # Exponential backoff - delay = min(2**retry_times, 60) # Max 60 seconds - spider.logger.warning( - f"Retrying {request.url} (attempt {retry_times}/{self.max_retry_times}) " - f"after {delay}s delay. Reason: {reason}" - ) - - time.sleep(delay) - return self._retry(request, reason, spider) or response - - return response +""" +Enhanced retry middleware with exponential backoff. +""" + +import time + +from scrapy.downloadermiddlewares.retry import RetryMiddleware +from scrapy.utils.response import response_status_message + + +class EnhancedRetryMiddleware(RetryMiddleware): + """Retry middleware with exponential backoff and better logging.""" + + def process_response(self, request, response, spider): + if request.meta.get("dont_retry", False): + return response + + if response.status in self.retry_http_codes: + reason = response_status_message(response.status) + retry_times = request.meta.get("retry_times", 0) + 1 + + # Exponential backoff + delay = min(2**retry_times, 60) # Max 60 seconds + spider.logger.warning( + f"Retrying {request.url} (attempt {retry_times}/{self.max_retry_times}) " + f"after {delay}s delay. Reason: {reason}" + ) + + time.sleep(delay) + return self._retry(request, reason, spider) or response + + return response diff --git a/uraas/pipelines/affiliation_filter.py b/uraas/pipelines/affiliation_filter.py index f9781c6212e294451a4540c8578e2b6331e17a18..a6820d1f33670135f1de623093541d50d973e457 100644 --- a/uraas/pipelines/affiliation_filter.py +++ b/uraas/pipelines/affiliation_filter.py @@ -1,153 +1,153 @@ -import os -import re -import sys - -from scrapy.exceptions import DropItem - -# Add project root to path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) - -from uraas.config.institutions import get_registry -from uraas.utils.staff_validator import StaffValidator - - -class AffiliationFilterPipeline: - """ - Multi-institution affiliation filter. - Only allows papers with at least ONE confirmed staff member from the target institution. - Prevents false positives from papers that just mention the institution name. - """ - - def __init__(self): - self.registry = get_registry() - self.validators = {} # Cache validators per institution - self.institution_patterns = {} # Cache compiled patterns per institution - - def _ensure_institution(self, spider): - """Lazily initialize institution config from the spider (called in process_item).""" - institution_name = getattr(spider, "institution", "unilag") - if getattr(self, "_initialized_for", None) == institution_name: - return # Already set up for this institution - - institution_config = self.registry.get(institution_name) - if not institution_config: - spider.logger.warning( - f"Institution '{institution_name}' not found, using default (UNILAG)" - ) - institution_config = self.registry.get("unilag") - - self.current_institution = institution_config - self.current_validator = StaffValidator(institution_config=institution_config) - self.current_patterns = [ - re.compile(pattern, re.IGNORECASE) - for pattern in institution_config.affiliation_patterns - ] - self._initialized_for = institution_name - spider.logger.info( - f"Affiliation filter initialized for {institution_config.name}" - ) - spider.logger.info(f"Staff count: {len(self.current_validator.staff_names)}") - spider.logger.info(f"Affiliation patterns: {len(self.current_patterns)}") - - def is_institution_affiliated(self, text: str) -> bool: - """Check if unstructured text contains institution references""" - if not text: - return False - return any(pattern.search(text) for pattern in self.current_patterns) - - def process_item(self, item, spider): - """ - Validate that paper has at least one confirmed staff member. - - Steps: - 1. Check if at least ONE author is a confirmed staff member (MANDATORY) - 2. Optionally verify affiliation text as secondary confirmation - 3. Tag item with institution ROR - """ - try: - # Initialize institution config lazily (spider is available here) - self._ensure_institution(spider) - - authors = item.get("authors", []) - - # Handle empty or invalid authors list - if not authors or not isinstance(authors, list): - raise DropItem( - f"Paper '{item.get('title', 'Unknown')[:60]}...' has no valid authors list." - ) - - # CRITICAL: Validate against staff list with RELAXED threshold (75%) - matching_staff = [] - for author in authors: - try: - if self.current_validator.is_staff_member( - author, fuzzy_threshold=75 - ): - matching_staff.append(author) - except Exception as e: - spider.logger.error(f"Error validating author '{author}': {str(e)}") - continue - - if not matching_staff: - raise DropItem( - f"Paper '{item.get('title', 'Unknown')[:60]}...' has NO confirmed " - f"{self.current_institution.name} staff authors. " - f"Authors: {', '.join(str(a) for a in authors[:3])}" - ) - - # Store which authors are staff for metadata - item["staff_authors"] = matching_staff - item["unilag_staff_authors"] = ( - matching_staff # Legacy field for backward compatibility - ) - - # Tag with institution ROR - item["institution"] = self.current_institution.name - item["institution_ror"] = self.current_institution.ror - - # Secondary validation: Check affiliation text (optional but recommended) - has_affiliation = False - - try: - # Check explicit affiliations list - affiliations = item.get("affiliations", []) - if isinstance(affiliations, list): - if any( - self.is_institution_affiliated(aff) - for aff in affiliations - if aff - ): - has_affiliation = True - - # Check raw string summary - if not has_affiliation: - raw_text = ( - " ".join(str(a) for a in affiliations if a) - if isinstance(affiliations, list) - else "" - ) - raw_text += " " + str(item.get("raw_affiliation", "")) - if self.is_institution_affiliated(raw_text): - has_affiliation = True - - except Exception as e: - spider.logger.error( - f"Error checking affiliations for paper '{item.get('title', 'Unknown')[:60]}': {str(e)}" - ) - - # Log warning if staff member found but no affiliation text - if not has_affiliation: - spider.logger.warning( - f"Paper has {self.current_institution.short_name} staff ({item['staff_authors']}) " - f"but no affiliation text. Proceeding with caution." - ) - - return item - - except DropItem: - raise - except Exception as e: - spider.logger.error( - f"Unexpected error in affiliation filter for paper '{item.get('title', 'Unknown')[:60]}': {str(e)}" - ) - raise DropItem(f"Failed to process item due to error: {str(e)}") +import os +import re +import sys + +from scrapy.exceptions import DropItem + +# Add project root to path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + +from uraas.config.institutions import get_registry +from uraas.utils.staff_validator import StaffValidator + + +class AffiliationFilterPipeline: + """ + Multi-institution affiliation filter. + Only allows papers with at least ONE confirmed staff member from the target institution. + Prevents false positives from papers that just mention the institution name. + """ + + def __init__(self): + self.registry = get_registry() + self.validators = {} # Cache validators per institution + self.institution_patterns = {} # Cache compiled patterns per institution + + def _ensure_institution(self, spider): + """Lazily initialize institution config from the spider (called in process_item).""" + institution_name = getattr(spider, "institution", "unilag") + if getattr(self, "_initialized_for", None) == institution_name: + return # Already set up for this institution + + institution_config = self.registry.get(institution_name) + if not institution_config: + spider.logger.warning( + f"Institution '{institution_name}' not found, using default (UNILAG)" + ) + institution_config = self.registry.get("unilag") + + self.current_institution = institution_config + self.current_validator = StaffValidator(institution_config=institution_config) + self.current_patterns = [ + re.compile(pattern, re.IGNORECASE) + for pattern in institution_config.affiliation_patterns + ] + self._initialized_for = institution_name + spider.logger.info( + f"Affiliation filter initialized for {institution_config.name}" + ) + spider.logger.info(f"Staff count: {len(self.current_validator.staff_names)}") + spider.logger.info(f"Affiliation patterns: {len(self.current_patterns)}") + + def is_institution_affiliated(self, text: str) -> bool: + """Check if unstructured text contains institution references""" + if not text: + return False + return any(pattern.search(text) for pattern in self.current_patterns) + + def process_item(self, item, spider): + """ + Validate that paper has at least one confirmed staff member. + + Steps: + 1. Check if at least ONE author is a confirmed staff member (MANDATORY) + 2. Optionally verify affiliation text as secondary confirmation + 3. Tag item with institution ROR + """ + try: + # Initialize institution config lazily (spider is available here) + self._ensure_institution(spider) + + authors = item.get("authors", []) + + # Handle empty or invalid authors list + if not authors or not isinstance(authors, list): + raise DropItem( + f"Paper '{item.get('title', 'Unknown')[:60]}...' has no valid authors list." + ) + + # CRITICAL: Validate against staff list with RELAXED threshold (75%) + matching_staff = [] + for author in authors: + try: + if self.current_validator.is_staff_member( + author, fuzzy_threshold=75 + ): + matching_staff.append(author) + except Exception as e: + spider.logger.error(f"Error validating author '{author}': {str(e)}") + continue + + if not matching_staff: + raise DropItem( + f"Paper '{item.get('title', 'Unknown')[:60]}...' has NO confirmed " + f"{self.current_institution.name} staff authors. " + f"Authors: {', '.join(str(a) for a in authors[:3])}" + ) + + # Store which authors are staff for metadata + item["staff_authors"] = matching_staff + item["unilag_staff_authors"] = ( + matching_staff # Legacy field for backward compatibility + ) + + # Tag with institution ROR + item["institution"] = self.current_institution.name + item["institution_ror"] = self.current_institution.ror + + # Secondary validation: Check affiliation text (optional but recommended) + has_affiliation = False + + try: + # Check explicit affiliations list + affiliations = item.get("affiliations", []) + if isinstance(affiliations, list): + if any( + self.is_institution_affiliated(aff) + for aff in affiliations + if aff + ): + has_affiliation = True + + # Check raw string summary + if not has_affiliation: + raw_text = ( + " ".join(str(a) for a in affiliations if a) + if isinstance(affiliations, list) + else "" + ) + raw_text += " " + str(item.get("raw_affiliation", "")) + if self.is_institution_affiliated(raw_text): + has_affiliation = True + + except Exception as e: + spider.logger.error( + f"Error checking affiliations for paper '{item.get('title', 'Unknown')[:60]}': {str(e)}" + ) + + # Log warning if staff member found but no affiliation text + if not has_affiliation: + spider.logger.warning( + f"Paper has {self.current_institution.short_name} staff ({item['staff_authors']}) " + f"but no affiliation text. Proceeding with caution." + ) + + return item + + except DropItem: + raise + except Exception as e: + spider.logger.error( + f"Unexpected error in affiliation filter for paper '{item.get('title', 'Unknown')[:60]}': {str(e)}" + ) + raise DropItem(f"Failed to process item due to error: {str(e)}") diff --git a/uraas/pipelines/database.py b/uraas/pipelines/database.py index ada39ceeb5ce9dc51732b2c7106b161dfdd5a36b..9cb68841a9c2bc212c1bda96da3265a2d824fb07 100644 --- a/uraas/pipelines/database.py +++ b/uraas/pipelines/database.py @@ -1,330 +1,330 @@ -# Define your item pipelines here -import re -from datetime import date - -from scrapy.exceptions import DropItem - -from uraas.database import Author, Collection, Community, File, Item, SessionLocal -from uraas.utils.ai_classifier import ( - _clean_text, - classify_special_collections, - extract_keywords, -) -from uraas.utils.analytics_cache import analytics_cache -from uraas.utils.ark_generator import ark_generator -from uraas.utils.pdf_downloader import pdf_downloader -from uraas.utils.unilag_classifier import classifier - -_DOI_RE = re.compile(r"^10\.\d{4,}/") - - -def _validate_doi(doi: str) -> bool: - """Returns True if the DOI has a valid format.""" - if not doi: - return False - # Clean common prefixes - doi = doi.replace("https://doi.org/", "").replace("http://dx.doi.org/", "").strip() - return bool(_DOI_RE.match(doi)) - - -class DatabaseStoragePipeline: - def open_spider(self, spider): - self.session = SessionLocal() - self._cache_invalidated = False - - def close_spider(self, spider): - try: - self.session.close() - except Exception: - pass - # Invalidate analytics cache so fresh data shows immediately - if self._cache_invalidated: - analytics_cache.invalidate_all() - - def process_item(self, item, spider): - try: - # Validate item has required fields - if not item.get("title"): - spider.logger.error("Item missing title, skipping") - return item - - doi = item.get("doi") or None - - # Validate DOI format — reject malformed ones - if doi and not _validate_doi(doi): - spider.logger.warning(f"Malformed DOI rejected: {doi!r}") - doi = None - - # Deduplicate by DOI first, then by URL, then by title - if doi: - doi = ( - doi.replace("https://doi.org/", "") - .replace("http://dx.doi.org/", "") - .strip() - ) - existing = self.session.query(Item).filter_by(doi=doi).first() - if existing: - spider.logger.debug(f"Duplicate DOI skipped: {doi}") - return item - - url = item.get("url") - if url: - existing = self.session.query(Item).filter_by(url=url).first() - if existing: - spider.logger.debug(f"Duplicate URL skipped: {url}") - return item - - # Deduplicate by normalised title (avoid same title from multiple sources) - norm_title = (item.get("title") or "").strip().lower()[:200] - if norm_title: - existing = ( - self.session.query(Item) - .filter(Item.title.ilike(norm_title[:100])) - .first() - ) - if existing: - spider.logger.debug(f"Duplicate title skipped: {norm_title[:60]}") - return item - - # Classify the document using enhanced classifier - try: - text_corpus = f"{item.get('title', '')} {item.get('abstract', '')} {item.get('raw_affiliation', '')}" - classifications = classifier.classify(text_corpus, threshold=0.5) - except Exception as e: - spider.logger.error(f"Classification error: {e}") - classifications = [] - - provenance = f"Harvested via URAAS Crawler - {date.today().isoformat()}" - - # Extract AI keywords from title+abstract using the proper classifier - try: - ai_kws = extract_keywords( - item.get("title", ""), item.get("abstract", ""), top_n=20 - ) - tags = [k["word"] for k in ai_kws] - except Exception as e: - spider.logger.error(f"Keyword extraction error: {e}") - tags = [] - - # Determine institution from spider context - institution_name = getattr(spider, "institution_name", None) - institution_ror = getattr(spider, "ror_id", None) - - # Special Collections scoring — heavy weight on indigenous knowledge, - # cultural heritage, African literature, etc. Score>0 marks the item as - # part of a special collection; drives ranking on the dashboard. - sc_score = 0.0 - sc_categories = "" - try: - sc_hits = classify_special_collections( - item.get("title", ""), - item.get("abstract", ""), - item.get("dc_subject", "") or ", ".join(tags[:15]), - ) - if sc_hits: - sc_score = float(sum(h["score"] for h in sc_hits)) - sc_categories = ",".join(h["category"] for h in sc_hits) - spider.logger.info( - f"SC HIT (score={sc_score:.1f}, cats={sc_categories}): " - f"{(item.get('title') or '')[:80]}" - ) - except Exception as e: - spider.logger.error(f"Special-collections scoring error: {e}") - - # SC gate: only store papers classified as Special Collections - if sc_score <= 0.0: - raise DropItem(f"Not a special collection: {(item.get('title') or '')[:60]}") - - # Parse publication date — accept YYYY, YYYY-MM-DD, or full ISO timestamps. - pub_date_raw = item.get("publication_date") or "" - pub_date = None - if pub_date_raw: - try: - from datetime import datetime as _dt - pub_date_str = str(pub_date_raw).strip() - for fmt in ("%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d", "%Y-%m", "%Y"): - try: - pub_date = _dt.strptime(pub_date_str[:len(fmt)], fmt) - break - except ValueError: - continue - except Exception: - pass - - # Normalise content/doc type - raw_type = (item.get("content_type") or item.get("dc_type") or "").strip() - # Map verbose DSpace dc:type values to our controlled vocabulary - _type_map = { - "thesis": "Thesis", "dissertation": "Thesis", - "article": "Article", "journal article": "Article", - "report": "Report", "technical report": "Report", - "conference paper": "Article", "book chapter": "Article", - "dataset": "Dataset", "preprint": "Article", - } - doc_type = _type_map.get(raw_type.lower(), raw_type) if raw_type else None - - # URL: fall back to None — never use a generic domain as unique URL - item_url = item.get("url") or None - - # Create Item with Dublin Core metadata - doc = Item( - title=item.get("title"), - dc_title=item.get("title"), - dc_identifier_uri=doi or item_url, - dc_identifier_doi=doi, - dc_date_issued=pub_date_raw[:10] if pub_date_raw else None, - dc_description_provenance=provenance, - dc_rights=item.get( - "dc_rights", "info:eu-repo/semantics/restrictedAccess" - ), - dc_type=doc_type, - dc_language=item.get("dc_language") or item.get("language_code") or None, - abstract=item.get("abstract") or None, - doi=doi, - url=item_url, - publication_date=pub_date, - source_repository=item.get("source_repository"), - pdf_url=item.get("pdf_url"), - content_type=doc_type, - language_code=item.get("language_code") or item.get("dc_language") or None, - is_african_language=bool(item.get("is_african_language", False)), - # AI keywords (comma-separated) - dc_subject=item.get("dc_subject") or ", ".join(tags[:15]), - ai_keywords=", ".join(tags), - sdg_tags=item.get("sdg_tags"), - coauthor_countries=item.get("coauthor_countries") or None, - african_country_count=int(item.get("african_country_count") or 0), - is_intra_african=bool(item.get("is_intra_african", False)), - openalex_id=item.get("openalex_id") or None, - cited_by_count=int(item.get("cited_by_count") or 0), - # Institution tracking for multi-institution analytics - institution=institution_name, - ror=institution_ror, - # Special Collections weighting - special_collection_score=sc_score, - special_collection_categories=sc_categories, - ) - - # Log to stdout for dashboard terminal - try: - safe_title = ( - (item.get("title") or "") - .encode("ascii", errors="replace") - .decode("ascii") - ) - print(f"URAAS_DOWNLOAD: {safe_title}", flush=True) - except Exception: - print(f"URAAS_DOWNLOAD: [Title encoding error]", flush=True) - - # Create Authors - authors_full = item.get("authors_full", []) - if not authors_full: - # Fallback to simple list if authors_full is missing - for a in item.get("authors", []): - authors_full.append({"name": a, "orcid": "", "ror": ""}) - - for auth in authors_full: - author_name = auth.get("name", "") - try: - if not author_name or not isinstance(author_name, str): - continue - author_obj = ( - self.session.query(Author) - .filter_by(normalized_name=author_name.lower().strip()) - .first() - ) - - if not author_obj: - author_obj = Author( - name=author_name, - normalized_name=author_name.lower().strip(), - orcid=auth.get("orcid", ""), - ror=auth.get("ror", ""), - ) - self.session.add(author_obj) - else: - # Update missing IDs if they are newly discovered - if auth.get("orcid") and not author_obj.orcid: - author_obj.orcid = auth["orcid"] - if auth.get("ror") and not author_obj.ror: - author_obj.ror = auth["ror"] - - doc.authors.append(author_obj) - except Exception as e: - spider.logger.error(f"Error processing author '{author_name}': {e}") - continue - - self.session.add(doc) - self.session.flush() # Get doc.id - - # Mint ARK — deterministic from DOI > URL > doc.id; idempotent on re-run. - try: - from datetime import datetime as _dt - ark_seed = doi or item_url or str(doc.id) - doc.ark = ark_generator.mint(ark_seed) - doc.ark_assigned_at = _dt.utcnow() - except Exception as e: - spider.logger.warning(f"ARK mint failed for item {doc.id}: {e}") - - # Map classified collections - try: - for community_name, collection_name, score in classifications[:3]: - try: - coll_obj = ( - self.session.query(Collection) - .filter_by(name=collection_name) - .first() - ) - if coll_obj and coll_obj not in doc.collections: - doc.collections.append(coll_obj) - except Exception as e: - spider.logger.error( - f"Error adding collection '{collection_name}': {e}" - ) - continue - except Exception as e: - spider.logger.error(f"Error processing classifications: {e}") - - # Score framework alignment (AU charters, Agenda 2063, etc.) - try: - from uraas.services.alignment_engine import score_item_alignment - al_json, al_ver = score_item_alignment( - doc.title or "", doc.abstract or "", doc.dc_subject or "" - ) - doc.alignment_scores = al_json - doc.alignment_version = al_ver - except Exception as e: - spider.logger.warning(f"Alignment scoring skipped for item: {e}") - - # Download PDF if available - if doc.pdf_url: - try: - policy = item.get("suggested_access", "Private") - # Cast explicitly to satisfy IDE static type checkers (MyPy) - pdf_metadata = pdf_downloader.download_pdf(str(doc.pdf_url), int(doc.id)) # type: ignore - if pdf_metadata: - bitstream = File( - item_id=doc.id, - file_path=pdf_metadata["file_path"], - sha256_hash=pdf_metadata["sha256_hash"], - access_policy=policy, - ) - self.session.add(bitstream) - except Exception as e: - spider.logger.error(f"PDF download error: {e}") - - self.session.commit() - self._cache_invalidated = True - return item - - except DropItem: - raise - except Exception as e: - spider.logger.error( - f"Database storage error for '{item.get('title', 'Unknown')[:60]}': {e}" - ) - try: - self.session.rollback() - except Exception: - pass - raise +# Define your item pipelines here +import re +from datetime import date + +from scrapy.exceptions import DropItem + +from uraas.database import Author, Collection, Community, File, Item, SessionLocal +from uraas.utils.ai_classifier import ( + _clean_text, + classify_special_collections, + extract_keywords, +) +from uraas.utils.analytics_cache import analytics_cache +from uraas.utils.ark_generator import ark_generator +from uraas.utils.pdf_downloader import pdf_downloader +from uraas.utils.unilag_classifier import classifier + +_DOI_RE = re.compile(r"^10\.\d{4,}/") + + +def _validate_doi(doi: str) -> bool: + """Returns True if the DOI has a valid format.""" + if not doi: + return False + # Clean common prefixes + doi = doi.replace("https://doi.org/", "").replace("http://dx.doi.org/", "").strip() + return bool(_DOI_RE.match(doi)) + + +class DatabaseStoragePipeline: + def open_spider(self, spider): + self.session = SessionLocal() + self._cache_invalidated = False + + def close_spider(self, spider): + try: + self.session.close() + except Exception: + pass + # Invalidate analytics cache so fresh data shows immediately + if self._cache_invalidated: + analytics_cache.invalidate_all() + + def process_item(self, item, spider): + try: + # Validate item has required fields + if not item.get("title"): + spider.logger.error("Item missing title, skipping") + return item + + doi = item.get("doi") or None + + # Validate DOI format — reject malformed ones + if doi and not _validate_doi(doi): + spider.logger.warning(f"Malformed DOI rejected: {doi!r}") + doi = None + + # Deduplicate by DOI first, then by URL, then by title + if doi: + doi = ( + doi.replace("https://doi.org/", "") + .replace("http://dx.doi.org/", "") + .strip() + ) + existing = self.session.query(Item).filter_by(doi=doi).first() + if existing: + spider.logger.debug(f"Duplicate DOI skipped: {doi}") + return item + + url = item.get("url") + if url: + existing = self.session.query(Item).filter_by(url=url).first() + if existing: + spider.logger.debug(f"Duplicate URL skipped: {url}") + return item + + # Deduplicate by normalised title (avoid same title from multiple sources) + norm_title = (item.get("title") or "").strip().lower()[:200] + if norm_title: + existing = ( + self.session.query(Item) + .filter(Item.title.ilike(norm_title[:100])) + .first() + ) + if existing: + spider.logger.debug(f"Duplicate title skipped: {norm_title[:60]}") + return item + + # Classify the document using enhanced classifier + try: + text_corpus = f"{item.get('title', '')} {item.get('abstract', '')} {item.get('raw_affiliation', '')}" + classifications = classifier.classify(text_corpus, threshold=0.5) + except Exception as e: + spider.logger.error(f"Classification error: {e}") + classifications = [] + + provenance = f"Harvested via URAAS Crawler - {date.today().isoformat()}" + + # Extract AI keywords from title+abstract using the proper classifier + try: + ai_kws = extract_keywords( + item.get("title", ""), item.get("abstract", ""), top_n=20 + ) + tags = [k["word"] for k in ai_kws] + except Exception as e: + spider.logger.error(f"Keyword extraction error: {e}") + tags = [] + + # Determine institution from spider context + institution_name = getattr(spider, "institution_name", None) + institution_ror = getattr(spider, "ror_id", None) + + # Special Collections scoring — heavy weight on indigenous knowledge, + # cultural heritage, African literature, etc. Score>0 marks the item as + # part of a special collection; drives ranking on the dashboard. + sc_score = 0.0 + sc_categories = "" + try: + sc_hits = classify_special_collections( + item.get("title", ""), + item.get("abstract", ""), + item.get("dc_subject", "") or ", ".join(tags[:15]), + ) + if sc_hits: + sc_score = float(sum(h["score"] for h in sc_hits)) + sc_categories = ",".join(h["category"] for h in sc_hits) + spider.logger.info( + f"SC HIT (score={sc_score:.1f}, cats={sc_categories}): " + f"{(item.get('title') or '')[:80]}" + ) + except Exception as e: + spider.logger.error(f"Special-collections scoring error: {e}") + + # SC gate: only store papers classified as Special Collections + if sc_score <= 0.0: + raise DropItem(f"Not a special collection: {(item.get('title') or '')[:60]}") + + # Parse publication date — accept YYYY, YYYY-MM-DD, or full ISO timestamps. + pub_date_raw = item.get("publication_date") or "" + pub_date = None + if pub_date_raw: + try: + from datetime import datetime as _dt + pub_date_str = str(pub_date_raw).strip() + for fmt in ("%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d", "%Y-%m", "%Y"): + try: + pub_date = _dt.strptime(pub_date_str[:len(fmt)], fmt) + break + except ValueError: + continue + except Exception: + pass + + # Normalise content/doc type + raw_type = (item.get("content_type") or item.get("dc_type") or "").strip() + # Map verbose DSpace dc:type values to our controlled vocabulary + _type_map = { + "thesis": "Thesis", "dissertation": "Thesis", + "article": "Article", "journal article": "Article", + "report": "Report", "technical report": "Report", + "conference paper": "Article", "book chapter": "Article", + "dataset": "Dataset", "preprint": "Article", + } + doc_type = _type_map.get(raw_type.lower(), raw_type) if raw_type else None + + # URL: fall back to None — never use a generic domain as unique URL + item_url = item.get("url") or None + + # Create Item with Dublin Core metadata + doc = Item( + title=item.get("title"), + dc_title=item.get("title"), + dc_identifier_uri=doi or item_url, + dc_identifier_doi=doi, + dc_date_issued=pub_date_raw[:10] if pub_date_raw else None, + dc_description_provenance=provenance, + dc_rights=item.get( + "dc_rights", "info:eu-repo/semantics/restrictedAccess" + ), + dc_type=doc_type, + dc_language=item.get("dc_language") or item.get("language_code") or None, + abstract=item.get("abstract") or None, + doi=doi, + url=item_url, + publication_date=pub_date, + source_repository=item.get("source_repository"), + pdf_url=item.get("pdf_url"), + content_type=doc_type, + language_code=item.get("language_code") or item.get("dc_language") or None, + is_african_language=bool(item.get("is_african_language", False)), + # AI keywords (comma-separated) + dc_subject=item.get("dc_subject") or ", ".join(tags[:15]), + ai_keywords=", ".join(tags), + sdg_tags=item.get("sdg_tags"), + coauthor_countries=item.get("coauthor_countries") or None, + african_country_count=int(item.get("african_country_count") or 0), + is_intra_african=bool(item.get("is_intra_african", False)), + openalex_id=item.get("openalex_id") or None, + cited_by_count=int(item.get("cited_by_count") or 0), + # Institution tracking for multi-institution analytics + institution=institution_name, + ror=institution_ror, + # Special Collections weighting + special_collection_score=sc_score, + special_collection_categories=sc_categories, + ) + + # Log to stdout for dashboard terminal + try: + safe_title = ( + (item.get("title") or "") + .encode("ascii", errors="replace") + .decode("ascii") + ) + print(f"URAAS_DOWNLOAD: {safe_title}", flush=True) + except Exception: + print(f"URAAS_DOWNLOAD: [Title encoding error]", flush=True) + + # Create Authors + authors_full = item.get("authors_full", []) + if not authors_full: + # Fallback to simple list if authors_full is missing + for a in item.get("authors", []): + authors_full.append({"name": a, "orcid": "", "ror": ""}) + + for auth in authors_full: + author_name = auth.get("name", "") + try: + if not author_name or not isinstance(author_name, str): + continue + author_obj = ( + self.session.query(Author) + .filter_by(normalized_name=author_name.lower().strip()) + .first() + ) + + if not author_obj: + author_obj = Author( + name=author_name, + normalized_name=author_name.lower().strip(), + orcid=auth.get("orcid", ""), + ror=auth.get("ror", ""), + ) + self.session.add(author_obj) + else: + # Update missing IDs if they are newly discovered + if auth.get("orcid") and not author_obj.orcid: + author_obj.orcid = auth["orcid"] + if auth.get("ror") and not author_obj.ror: + author_obj.ror = auth["ror"] + + doc.authors.append(author_obj) + except Exception as e: + spider.logger.error(f"Error processing author '{author_name}': {e}") + continue + + self.session.add(doc) + self.session.flush() # Get doc.id + + # Mint ARK — deterministic from DOI > URL > doc.id; idempotent on re-run. + try: + from datetime import datetime as _dt + ark_seed = doi or item_url or str(doc.id) + doc.ark = ark_generator.mint(ark_seed) + doc.ark_assigned_at = _dt.utcnow() + except Exception as e: + spider.logger.warning(f"ARK mint failed for item {doc.id}: {e}") + + # Map classified collections + try: + for community_name, collection_name, score in classifications[:3]: + try: + coll_obj = ( + self.session.query(Collection) + .filter_by(name=collection_name) + .first() + ) + if coll_obj and coll_obj not in doc.collections: + doc.collections.append(coll_obj) + except Exception as e: + spider.logger.error( + f"Error adding collection '{collection_name}': {e}" + ) + continue + except Exception as e: + spider.logger.error(f"Error processing classifications: {e}") + + # Score framework alignment (AU charters, Agenda 2063, etc.) + try: + from uraas.services.alignment_engine import score_item_alignment + al_json, al_ver = score_item_alignment( + doc.title or "", doc.abstract or "", doc.dc_subject or "" + ) + doc.alignment_scores = al_json + doc.alignment_version = al_ver + except Exception as e: + spider.logger.warning(f"Alignment scoring skipped for item: {e}") + + # Download PDF if available + if doc.pdf_url: + try: + policy = item.get("suggested_access", "Private") + # Cast explicitly to satisfy IDE static type checkers (MyPy) + pdf_metadata = pdf_downloader.download_pdf(str(doc.pdf_url), int(doc.id)) # type: ignore + if pdf_metadata: + bitstream = File( + item_id=doc.id, + file_path=pdf_metadata["file_path"], + sha256_hash=pdf_metadata["sha256_hash"], + access_policy=policy, + ) + self.session.add(bitstream) + except Exception as e: + spider.logger.error(f"PDF download error: {e}") + + self.session.commit() + self._cache_invalidated = True + return item + + except DropItem: + raise + except Exception as e: + spider.logger.error( + f"Database storage error for '{item.get('title', 'Unknown')[:60]}': {e}" + ) + try: + self.session.rollback() + except Exception: + pass + raise diff --git a/uraas/pipelines/gap_analysis.py b/uraas/pipelines/gap_analysis.py index 60379feb3f5a450c4599c6a289e3e991bd7291dd..b182d5db14929f317a9add2be19bf8d79498d3a9 100644 --- a/uraas/pipelines/gap_analysis.py +++ b/uraas/pipelines/gap_analysis.py @@ -1,69 +1,69 @@ -from scrapy.exceptions import DropItem -from thefuzz import fuzz - -from uraas.database import Item, SessionLocal -from uraas.utils.normalizer import normalize_title - -FUZZY_THRESHOLD = 95 # % similarity required to classify as duplicate - - -class GapAnalysisPipeline: - """ - Phase 2: The Gap Analysis (Fuzzy Edition). - 1. Check DOI first (exact, deterministic). - 2. If no DOI, fuzzy-compare normalized title via Levenshtein distance. - If similarity >= 95% → drop as duplicate. - """ - - def open_spider(self): - self.session = SessionLocal() - # Cache existing normalized titles for fast in-memory fuzzy comparison - self._cached_titles = [ - normalize_title(r[0]) - for r in self.session.query(Item.dc_title).all() - if r[0] - ] - - def close_spider(self): - try: - self.session.close() - except Exception: - pass - - def process_item(self, item, spider): - try: - # --- Step 1: DOI exact match --- - if item.get("doi"): - exists = self.session.query(Item).filter_by(doi=item["doi"]).first() - if exists: - spider.logger.info(f"[Gap] DOI duplicate: {item['doi']}") - raise DropItem(f"Duplicate DOI: {item['doi']}") - - # --- Step 2: URL exact match --- - if item.get("url"): - exists = self.session.query(Item).filter_by(url=item["url"]).first() - if exists: - spider.logger.info(f"[Gap] URL duplicate: {item['url']}") - raise DropItem(f"Duplicate URL: {item['url']}") - - # --- Step 3: Fuzzy title match (only when no DOI for deterministic check) --- - if not item.get("doi") and item.get("title"): - needle = normalize_title(item["title"]) - for cached in self._cached_titles: - score = fuzz.ratio(needle, cached) - if score >= FUZZY_THRESHOLD: - spider.logger.info( - f"[Gap] Fuzzy duplicate ({score}%): '{item['title'][:60]}'" - ) - raise DropItem(f"Fuzzy duplicate title ({score}%)") - - # Survives all checks → it's a genuine gap, add to cache for this session - self._cached_titles.append(normalize_title(item.get("title", ""))) - return item - - except DropItem: - raise - except Exception as e: - spider.logger.error(f"[Gap] Error processing item: {e}") - # Don't drop on error, let it through - return item +from scrapy.exceptions import DropItem +from thefuzz import fuzz + +from uraas.database import Item, SessionLocal +from uraas.utils.normalizer import normalize_title + +FUZZY_THRESHOLD = 95 # % similarity required to classify as duplicate + + +class GapAnalysisPipeline: + """ + Phase 2: The Gap Analysis (Fuzzy Edition). + 1. Check DOI first (exact, deterministic). + 2. If no DOI, fuzzy-compare normalized title via Levenshtein distance. + If similarity >= 95% → drop as duplicate. + """ + + def open_spider(self): + self.session = SessionLocal() + # Cache existing normalized titles for fast in-memory fuzzy comparison + self._cached_titles = [ + normalize_title(r[0]) + for r in self.session.query(Item.dc_title).all() + if r[0] + ] + + def close_spider(self): + try: + self.session.close() + except Exception: + pass + + def process_item(self, item, spider): + try: + # --- Step 1: DOI exact match --- + if item.get("doi"): + exists = self.session.query(Item).filter_by(doi=item["doi"]).first() + if exists: + spider.logger.info(f"[Gap] DOI duplicate: {item['doi']}") + raise DropItem(f"Duplicate DOI: {item['doi']}") + + # --- Step 2: URL exact match --- + if item.get("url"): + exists = self.session.query(Item).filter_by(url=item["url"]).first() + if exists: + spider.logger.info(f"[Gap] URL duplicate: {item['url']}") + raise DropItem(f"Duplicate URL: {item['url']}") + + # --- Step 3: Fuzzy title match (only when no DOI for deterministic check) --- + if not item.get("doi") and item.get("title"): + needle = normalize_title(item["title"]) + for cached in self._cached_titles: + score = fuzz.ratio(needle, cached) + if score >= FUZZY_THRESHOLD: + spider.logger.info( + f"[Gap] Fuzzy duplicate ({score}%): '{item['title'][:60]}'" + ) + raise DropItem(f"Fuzzy duplicate title ({score}%)") + + # Survives all checks → it's a genuine gap, add to cache for this session + self._cached_titles.append(normalize_title(item.get("title", ""))) + return item + + except DropItem: + raise + except Exception as e: + spider.logger.error(f"[Gap] Error processing item: {e}") + # Don't drop on error, let it through + return item diff --git a/uraas/pipelines/unpaywall.py b/uraas/pipelines/unpaywall.py index 1a81342463a9940908c393c8589c50a70cd37c36..e3c3f6a1e523ff4226e526541ac749a3685a9cee 100644 --- a/uraas/pipelines/unpaywall.py +++ b/uraas/pipelines/unpaywall.py @@ -1,39 +1,39 @@ -import requests - - -class UnpaywallPipeline: - """Verifies legal Open Access PDF bitstreams via Unpaywall before ingestion.""" - - def process_item(self, item, spider): - doi = item.get("doi") - if not doi: - return item - - # Already have a PDF? Unpaywall might find a better/legal one, but let's check - try: - # Unpaywall requires an email for the API - url = f"https://api.unpaywall.org/v2/{doi}?email=uraas-bot@unilag.edu.ng" - response = requests.get(url, timeout=10) - if response.status_code == 200: - data = response.json() - is_oa = data.get("is_oa", False) - oa_status = data.get("oa_status", "closed") - item["oa_status"] = oa_status - - # Smart Version Detection - if oa_status in ["gold", "hybrid", "green"]: - item["suggested_access"] = "Public" # Safe to share - else: - item["suggested_access"] = "Private" # Restricted/Bronze/Closed - - if is_oa and data.get("best_oa_location"): - legal_pdf = data["best_oa_location"].get("url_for_pdf") - if legal_pdf: - item["pdf_url"] = legal_pdf - spider.logger.info( - f"Unpaywall ({oa_status}): Found legal PDF for {doi}" - ) - except Exception as e: - spider.logger.debug(f"Unpaywall check failed for {doi}: {e}") - - return item +import requests + + +class UnpaywallPipeline: + """Verifies legal Open Access PDF bitstreams via Unpaywall before ingestion.""" + + def process_item(self, item, spider): + doi = item.get("doi") + if not doi: + return item + + # Already have a PDF? Unpaywall might find a better/legal one, but let's check + try: + # Unpaywall requires an email for the API + url = f"https://api.unpaywall.org/v2/{doi}?email=uraas-bot@unilag.edu.ng" + response = requests.get(url, timeout=10) + if response.status_code == 200: + data = response.json() + is_oa = data.get("is_oa", False) + oa_status = data.get("oa_status", "closed") + item["oa_status"] = oa_status + + # Smart Version Detection + if oa_status in ["gold", "hybrid", "green"]: + item["suggested_access"] = "Public" # Safe to share + else: + item["suggested_access"] = "Private" # Restricted/Bronze/Closed + + if is_oa and data.get("best_oa_location"): + legal_pdf = data["best_oa_location"].get("url_for_pdf") + if legal_pdf: + item["pdf_url"] = legal_pdf + spider.logger.info( + f"Unpaywall ({oa_status}): Found legal PDF for {doi}" + ) + except Exception as e: + spider.logger.debug(f"Unpaywall check failed for {doi}: {e}") + + return item diff --git a/uraas/production_config.py b/uraas/production_config.py index aaeafc2585e78ffad5606483b0853dca579dec17..8c95b11a83fed2adfdf802deec3d40030c61783d 100644 --- a/uraas/production_config.py +++ b/uraas/production_config.py @@ -1,106 +1,106 @@ -""" -Production configuration module for Render deployment. -Detects production environment and applies security-hardened settings. -""" - -import logging -import os -from typing import Any, Dict - - -class ProductionConfig: - """Production configuration for Render deployment.""" - - @staticmethod - def is_production() -> bool: - """Detect production: Render, or any host flagged URAAS_ENV=production - (e.g. a self-managed UNILAG DMZ deployment behind nginx).""" - return os.getenv("RENDER") == "true" or os.getenv("URAAS_ENV") == "production" - - @staticmethod - def apply_config(app) -> None: - """ - Apply production configuration to Flask app. - Only applies settings when running on Render. - """ - if not ProductionConfig.is_production(): - return - - # Disable debug mode in production (CRITICAL for security) - app.config["DEBUG"] = False - app.config["TESTING"] = False - - # Security settings for HTTPS - app.config["SESSION_COOKIE_SECURE"] = True # HTTPS only - app.config["SESSION_COOKIE_HTTPONLY"] = True # No JavaScript access - app.config["SESSION_COOKIE_SAMESITE"] = "Lax" # CSRF protection - - # Use production secret key from environment - secret_key = os.getenv("DASHBOARD_SECRET_KEY") - if secret_key: - app.config["SECRET_KEY"] = secret_key - else: - # Fallback - generate random key if not provided - import secrets - - app.config["SECRET_KEY"] = secrets.token_hex(32) - logging.warning("DASHBOARD_SECRET_KEY not set, using generated key") - - # Database configuration - database_url = os.getenv("DATABASE_URL") - if database_url: - # Render provides postgres:// but SQLAlchemy needs postgresql:// - if database_url.startswith("postgres://"): - database_url = database_url.replace("postgres://", "postgresql://", 1) - app.config["SQLALCHEMY_DATABASE_URI"] = database_url - - # SQLAlchemy engine options for production - app.config["SQLALCHEMY_ENGINE_OPTIONS"] = { - "pool_size": 5, - "pool_recycle": 3600, # Recycle connections after 1 hour - "pool_pre_ping": True, # Verify connections before using - "max_overflow": 10, - "pool_timeout": 30, - } - - # Storage configuration - app.config["STORAGE_PATH"] = os.getenv( - "STORAGE_PATH", "/opt/render/project/storage" - ) - - # Ensure storage directory exists - storage_path = app.config["STORAGE_PATH"] - pdf_path = os.path.join(storage_path, "pdfs") - os.makedirs(pdf_path, exist_ok=True) - - # Configure production logging - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - ) - - # Suppress noisy loggers - logging.getLogger("werkzeug").setLevel(logging.WARNING) - logging.getLogger("socketio").setLevel(logging.WARNING) - logging.getLogger("engineio").setLevel(logging.WARNING) - - app.logger.info("=" * 70) - app.logger.info("Production configuration applied") - app.logger.info( - f"Database: {database_url[:50] if database_url else 'Not configured'}..." - ) - app.logger.info(f"Storage: {storage_path}") - app.logger.info(f"Secret key: {'✓ Set' if secret_key else '⚠ Generated'}") - app.logger.info("=" * 70) - - @staticmethod - def get_config_dict() -> Dict[str, Any]: - """Return configuration as dictionary for inspection.""" - return { - "is_production": ProductionConfig.is_production(), - "database_url": os.getenv("DATABASE_URL", "Not set")[:50] + "...", - "storage_path": os.getenv("STORAGE_PATH", "Not set"), - "secret_key_set": bool(os.getenv("DASHBOARD_SECRET_KEY")), - "render_env": os.getenv("RENDER", "Not set"), - } +""" +Production configuration module for Render deployment. +Detects production environment and applies security-hardened settings. +""" + +import logging +import os +from typing import Any, Dict + + +class ProductionConfig: + """Production configuration for Render deployment.""" + + @staticmethod + def is_production() -> bool: + """Detect production: Render, or any host flagged URAAS_ENV=production + (e.g. a self-managed UNILAG DMZ deployment behind nginx).""" + return os.getenv("RENDER") == "true" or os.getenv("URAAS_ENV") == "production" + + @staticmethod + def apply_config(app) -> None: + """ + Apply production configuration to Flask app. + Only applies settings when running on Render. + """ + if not ProductionConfig.is_production(): + return + + # Disable debug mode in production (CRITICAL for security) + app.config["DEBUG"] = False + app.config["TESTING"] = False + + # Security settings for HTTPS + app.config["SESSION_COOKIE_SECURE"] = True # HTTPS only + app.config["SESSION_COOKIE_HTTPONLY"] = True # No JavaScript access + app.config["SESSION_COOKIE_SAMESITE"] = "Lax" # CSRF protection + + # Use production secret key from environment + secret_key = os.getenv("DASHBOARD_SECRET_KEY") + if secret_key: + app.config["SECRET_KEY"] = secret_key + else: + # Fallback - generate random key if not provided + import secrets + + app.config["SECRET_KEY"] = secrets.token_hex(32) + logging.warning("DASHBOARD_SECRET_KEY not set, using generated key") + + # Database configuration + database_url = os.getenv("DATABASE_URL") + if database_url: + # Render provides postgres:// but SQLAlchemy needs postgresql:// + if database_url.startswith("postgres://"): + database_url = database_url.replace("postgres://", "postgresql://", 1) + app.config["SQLALCHEMY_DATABASE_URI"] = database_url + + # SQLAlchemy engine options for production + app.config["SQLALCHEMY_ENGINE_OPTIONS"] = { + "pool_size": 5, + "pool_recycle": 3600, # Recycle connections after 1 hour + "pool_pre_ping": True, # Verify connections before using + "max_overflow": 10, + "pool_timeout": 30, + } + + # Storage configuration + app.config["STORAGE_PATH"] = os.getenv( + "STORAGE_PATH", "/opt/render/project/storage" + ) + + # Ensure storage directory exists + storage_path = app.config["STORAGE_PATH"] + pdf_path = os.path.join(storage_path, "pdfs") + os.makedirs(pdf_path, exist_ok=True) + + # Configure production logging + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + # Suppress noisy loggers + logging.getLogger("werkzeug").setLevel(logging.WARNING) + logging.getLogger("socketio").setLevel(logging.WARNING) + logging.getLogger("engineio").setLevel(logging.WARNING) + + app.logger.info("=" * 70) + app.logger.info("Production configuration applied") + app.logger.info( + f"Database: {database_url[:50] if database_url else 'Not configured'}..." + ) + app.logger.info(f"Storage: {storage_path}") + app.logger.info(f"Secret key: {'✓ Set' if secret_key else '⚠ Generated'}") + app.logger.info("=" * 70) + + @staticmethod + def get_config_dict() -> Dict[str, Any]: + """Return configuration as dictionary for inspection.""" + return { + "is_production": ProductionConfig.is_production(), + "database_url": os.getenv("DATABASE_URL", "Not set")[:50] + "...", + "storage_path": os.getenv("STORAGE_PATH", "Not set"), + "secret_key_set": bool(os.getenv("DASHBOARD_SECRET_KEY")), + "render_env": os.getenv("RENDER", "Not set"), + } diff --git a/uraas/services/advanced_search.py b/uraas/services/advanced_search.py index dee6f9373d34890389a7c407f1bbd833f995eed7..d09399f9237b4cd5c8e4d0ec16c403fe43cc4706 100644 --- a/uraas/services/advanced_search.py +++ b/uraas/services/advanced_search.py @@ -1,549 +1,549 @@ -""" -Advanced Search Service -Implements Boolean operators, field-specific queries, and full-text search. -Comparable to Scopus/Web of Science search capabilities. -""" - -import logging -import re -from typing import Dict, List, Optional, Tuple - -from sqlalchemy import and_, func, not_, or_, text - -from uraas.database import Author, Collection, Community, Item, SessionLocal, db_year - -logger = logging.getLogger(__name__) - - -class SearchQuery: - """Parse and execute advanced search queries.""" - - @staticmethod - def ast_to_string(node) -> str: - if node is None: - return "" - node_type = node[0] - if node_type in ("AND", "OR"): - return f"({SearchQuery.ast_to_string(node[1])} {node_type} {SearchQuery.ast_to_string(node[2])})" - elif node_type == "NOT": - return f"(NOT {SearchQuery.ast_to_string(node[1])})" - elif node_type == "FIELD_VAL": - field, val = node[1] - return f"{field}:{val}" - elif node_type == "PHRASE": - return f'"{node[1]}"' - elif node_type == "TERM": - return node[1] - return str(node) - - @staticmethod - def parse_boolean_query(query: str): - """ - Parse Boolean search query into structured format (AST). - - Supports: - - AND, OR, NOT operators - - Parentheses for grouping - - Field-specific searches: title:machine, author:smith, year:2020 - - Phrase searches: "machine learning" - - Wildcards: machin* (suffix), *learning (prefix) - - Returns: - AST node (tuple) - """ - query = query.strip() - if not query: - return None - - try: - # Tokenize - token_specification = [ - ("LPAR", r"\("), - ("RPAR", r"\)"), - ("AND", r"\bAND\b"), - ("OR", r"\bOR\b"), - ("NOT", r"\bNOT\b"), - ( - "FIELD_VAL", - r'\b(\w+):(?:(?:"([^"]+)")|(?:\'([^\']+)\')|([^\s\)\(]+))', - ), - ("PHRASE", r'"([^"]+)"|\'([^\']+)\''), - ("TERM", r"[^\s\)\(]+"), - ("SKIP", r"\s+"), - ] - tok_regex = "|".join( - f"(?P<{name}>{pattern})" for name, pattern in token_specification - ) - tokens = [] - for mo in re.finditer(tok_regex, query): - kind = mo.lastgroup - if kind == "SKIP": - continue - val = mo.group(kind) - if kind == "FIELD_VAL": - field_match = re.match(r"^(\w+):(.*)$", val) - field = field_match.group(1).lower() - raw_val = field_match.group(2) - if (raw_val.startswith('"') and raw_val.endswith('"')) or ( - raw_val.startswith("'") and raw_val.endswith("'") - ): - raw_val = raw_val[1:-1] - tokens.append(("FIELD_VAL", (field, raw_val))) - elif kind == "PHRASE": - if (val.startswith('"') and val.endswith('"')) or ( - val.startswith("'") and val.endswith("'") - ): - val = val[1:-1] - tokens.append(("PHRASE", val)) - elif kind == "TERM": - tokens.append(("TERM", val)) - else: - tokens.append((kind, val)) - - # Insert implicit ANDs - operand_types = {"TERM", "PHRASE", "FIELD_VAL", "RPAR"} - start_operand_types = {"TERM", "PHRASE", "FIELD_VAL", "LPAR", "NOT"} - tokens_with_ands = [] - for i, tok in enumerate(tokens): - if i > 0: - prev_tok = tokens[i - 1] - if prev_tok[0] in operand_types and tok[0] in start_operand_types: - tokens_with_ands.append(("AND", "AND")) - tokens_with_ands.append(tok) - - # Parse to AST using Shunting-yard - precedence = {"NOT": 3, "AND": 2, "OR": 1} - output_stack = [] - operator_stack = [] - - for tok_type, val in tokens_with_ands: - if tok_type in {"TERM", "PHRASE", "FIELD_VAL"}: - output_stack.append((tok_type, val)) - elif tok_type == "LPAR": - operator_stack.append((tok_type, val)) - elif tok_type == "RPAR": - while operator_stack and operator_stack[-1][0] != "LPAR": - op = operator_stack.pop() - if op[0] == "NOT": - if not output_stack: - raise ValueError("Invalid NOT query") - arg = output_stack.pop() - output_stack.append(("NOT", arg)) - else: - if len(output_stack) < 2: - raise ValueError( - f"Invalid query: missing arguments for {op[0]}" - ) - right = output_stack.pop() - left = output_stack.pop() - output_stack.append((op[0], left, right)) - if not operator_stack: - raise ValueError("Mismatched parentheses") - operator_stack.pop() # pop LPAR - elif tok_type in {"AND", "OR", "NOT"}: - while ( - operator_stack - and operator_stack[-1][0] in precedence - and precedence[operator_stack[-1][0]] >= precedence[tok_type] - ): - op = operator_stack.pop() - if op[0] == "NOT": - if not output_stack: - raise ValueError("Invalid NOT query") - arg = output_stack.pop() - output_stack.append(("NOT", arg)) - else: - if len(output_stack) < 2: - raise ValueError( - f"Invalid query: missing arguments for {op[0]}" - ) - right = output_stack.pop() - left = output_stack.pop() - output_stack.append((op[0], left, right)) - operator_stack.append((tok_type, val)) - - while operator_stack: - op = operator_stack.pop() - if op[0] == "LPAR": - raise ValueError("Mismatched parentheses") - if op[0] == "NOT": - if not output_stack: - raise ValueError("Invalid NOT query") - arg = output_stack.pop() - output_stack.append(("NOT", arg)) - else: - if len(output_stack) < 2: - raise ValueError( - f"Invalid query: missing arguments for {op[0]}" - ) - right = output_stack.pop() - left = output_stack.pop() - output_stack.append((op[0], left, right)) - - if not output_stack: - return None - if len(output_stack) > 1: - raise ValueError("Invalid query expression") - return output_stack[0] - - except Exception as e: - logger.warning( - f"Boolean parsing failed for query '{query}': {e}. Falling back to simple parsing." - ) - # Fallback to simple split AND-join - words = query.split() - if not words: - return None - node = ("TERM", words[0]) - for w in words[1:]: - node = ("AND", node, ("TERM", w)) - return node - - @staticmethod - def build_sql_filter(parsed_query, session=None): - """ - Build SQLAlchemy filter from parsed query. - - Returns: - SQLAlchemy filter expression - """ - if parsed_query is None: - return None - - if session is None: - session = SessionLocal() - - def evaluate_node(node): - if node is None: - return None - - node_type = node[0] - if node_type == "AND": - left = evaluate_node(node[1]) - right = evaluate_node(node[2]) - if left is not None and right is not None: - return and_(left, right) - return left or right - - elif node_type == "OR": - left = evaluate_node(node[1]) - right = evaluate_node(node[2]) - if left is not None and right is not None: - return or_(left, right) - return left or right - - elif node_type == "NOT": - arg = evaluate_node(node[1]) - if arg is not None: - return not_(arg) - return None - - elif node_type == "FIELD_VAL": - field, val = node[1] - return build_field_filter(field, val) - - elif node_type == "PHRASE": - val = node[1] - return or_( - Item.title.ilike(f"%{val}%"), - Item.abstract.ilike(f"%{val}%"), - Item.ai_keywords.ilike(f"%{val}%"), - ) - - elif node_type == "TERM": - val = node[1] - if "*" in val: - sql_val = val.replace("*", "%") - else: - sql_val = f"%{val}%" - return or_( - Item.title.ilike(sql_val), - Item.abstract.ilike(sql_val), - Item.doi.ilike(sql_val), - Item.ai_keywords.ilike(sql_val), - ) - return None - - def build_field_filter(field, value): - if "*" in value: - sql_val = value.replace("*", "%") - else: - sql_val = f"%{value}%" - - if field == "title": - return Item.title.ilike(sql_val) - elif field == "abstract": - return Item.abstract.ilike(sql_val) - elif field == "author": - return Item.id.in_( - session.query(Item.id) - .join(Item.authors) - .filter(Author.name.ilike(sql_val)) - ) - elif field == "year": - try: - year = int(value) - return db_year(Item.publication_date) == str(year) - except ValueError: - return None - elif field == "doi": - return Item.doi.ilike(sql_val) - elif field == "faculty": - return Item.id.in_( - session.query(Item.id) - .join(Item.collections) - .join(Collection.community) - .filter(Community.name.ilike(sql_val)) - ) - elif field == "department": - return Item.id.in_( - session.query(Item.id) - .join(Item.collections) - .filter(Collection.name.ilike(sql_val)) - ) - elif field == "keyword": - return or_( - Item.ai_keywords.ilike(sql_val), Item.dc_subject.ilike(sql_val) - ) - elif field == "language": - return Item.language_code == value.lower() - elif field == "oa" or field == "openaccess": - if value.lower() in ("true", "yes", "1"): - return Item.dc_rights.like("%openAccess%") - else: - return ~Item.dc_rights.like("%openAccess%") - else: - return or_( - Item.title.ilike(sql_val), - Item.abstract.ilike(sql_val), - Item.doi.ilike(sql_val), - Item.ai_keywords.ilike(sql_val), - ) - - return evaluate_node(parsed_query) - - @staticmethod - def execute_search( - query: str, - limit: int = 50, - offset: int = 0, - sort_by: str = "relevance", - filters: Optional[Dict] = None, - ) -> Dict: - """ - Execute advanced search query. - - Args: - query: Search query string (supports Boolean operators) - limit: Maximum results to return - offset: Pagination offset - sort_by: Sort order ('relevance', 'date', 'citations', 'title') - filters: Additional filters (year_from, year_to, oa_only, etc.) - - Returns: - { - 'total': int, - 'results': [paper_dict, ...], - 'query_parsed': str, - 'took_ms': float - } - """ - import time - - start_time = time.time() - - session = SessionLocal() - try: - # Parse query - parsed = SearchQuery.parse_boolean_query(query) - - # Build base query - q = session.query(Item) - - # Apply parsed query filters - sql_filter = SearchQuery.build_sql_filter(parsed, session) - if sql_filter is not None: - q = q.filter(sql_filter) - - # Apply additional filters - if filters: - if filters.get("year_from"): - q = q.filter( - db_year(Item.publication_date) >= str(filters["year_from"]) - ) - - if filters.get("year_to"): - q = q.filter( - db_year(Item.publication_date) <= str(filters["year_to"]) - ) - - if filters.get("oa_only"): - q = q.filter(Item.dc_rights.like("%openAccess%")) - - if filters.get("faculty"): - q = ( - q.join(Item.collections) - .join(Collection.community) - .filter(Community.name.ilike(f'%{filters["faculty"]}%')) - ) - - if filters.get("has_pdf"): - from uraas.database import File - - q = q.join(File, File.item_id == Item.id) - - # Get total count - total = q.count() - - # Apply sorting - if sort_by == "date": - q = q.order_by(Item.publication_date.desc().nullslast()) - elif sort_by == "title": - q = q.order_by(Item.title) - elif sort_by == "citations": - # Join with citation metrics if available - from uraas.services.citation_tracker import CitationMetrics - - q = q.outerjoin( - CitationMetrics, CitationMetrics.item_id == Item.id - ).order_by(CitationMetrics.citation_count.desc().nullslast()) - else: # relevance (default) - # Simple relevance: prioritize title matches - q = q.order_by(Item.created_at.desc()) - - # Pagination - results = q.limit(limit).offset(offset).all() - - # Format results - formatted_results = [] - for item in results: - formatted_results.append( - { - "id": item.id, - "title": item.title, - "abstract": item.abstract[:300] if item.abstract else None, - "doi": item.doi, - "url": item.url, - "year": ( - item.publication_date.year - if item.publication_date - else None - ), - "authors": [a.name for a in item.authors[:5]], - "faculty": ( - item.collections[0].community.name - if item.collections and item.collections[0].community - else None - ), - "department": ( - item.collections[0].name if item.collections else None - ), - "is_oa": "openAccess" in (item.dc_rights or ""), - "docid": item.docid, - "language": item.language_code, - "keywords": ( - item.ai_keywords.split(",")[:5] if item.ai_keywords else [] - ), - } - ) - - took_ms = (time.time() - start_time) * 1000 - - return { - "total": total, - "results": formatted_results, - "query_parsed": SearchQuery.ast_to_string(parsed), - "took_ms": round(took_ms, 2), - "page": offset // limit + 1, - "pages": (total + limit - 1) // limit, - } - - except Exception as e: - logger.error(f"Search failed: {e}") - return {"total": 0, "results": [], "error": str(e), "took_ms": 0} - finally: - session.close() - - @staticmethod - def get_search_suggestions(partial_query: str, field: str = "all") -> List[str]: - """ - Get autocomplete suggestions for search queries. - - Args: - partial_query: Partial search term - field: Field to search ('author', 'keyword', 'faculty', 'all') - - Returns: - List of suggested completions - """ - session = SessionLocal() - try: - suggestions = [] - - if field in ("author", "all"): - authors = ( - session.query(Author.name) - .filter(Author.name.ilike(f"%{partial_query}%")) - .limit(10) - .all() - ) - suggestions.extend([f"author:{a[0]}" for a in authors]) - - if field in ("faculty", "all"): - faculties = ( - session.query(Community.name) - .filter(Community.name.ilike(f"%{partial_query}%")) - .limit(5) - .all() - ) - suggestions.extend([f"faculty:{f[0]}" for f in faculties]) - - if field in ("keyword", "all"): - # Extract keywords from papers - items = ( - session.query(Item.ai_keywords) - .filter(Item.ai_keywords.ilike(f"%{partial_query}%")) - .limit(20) - .all() - ) - - keywords = set() - for item in items: - if item[0]: - for kw in item[0].split(","): - kw = kw.strip() - if partial_query.lower() in kw.lower(): - keywords.add(kw) - - suggestions.extend([f"keyword:{k}" for k in list(keywords)[:10]]) - - return suggestions[:15] - - finally: - session.close() - - -# ── Saved Searches ──────────────────────────────────────────────────────────── - - -class SavedSearch: - """Manage saved search queries for users.""" - - @staticmethod - def save_search(name: str, query: str, filters: Optional[Dict] = None) -> int: - """Save a search query for later reuse.""" - # TODO: Implement user authentication first - # For now, store in a simple table - pass - - @staticmethod - def get_saved_searches() -> List[Dict]: - """Get all saved searches.""" - # TODO: Implement - pass - - @staticmethod - def execute_saved_search(search_id: int) -> Dict: - """Execute a previously saved search.""" - # TODO: Implement - pass +""" +Advanced Search Service +Implements Boolean operators, field-specific queries, and full-text search. +Comparable to Scopus/Web of Science search capabilities. +""" + +import logging +import re +from typing import Dict, List, Optional, Tuple + +from sqlalchemy import and_, func, not_, or_, text + +from uraas.database import Author, Collection, Community, Item, SessionLocal, db_year + +logger = logging.getLogger(__name__) + + +class SearchQuery: + """Parse and execute advanced search queries.""" + + @staticmethod + def ast_to_string(node) -> str: + if node is None: + return "" + node_type = node[0] + if node_type in ("AND", "OR"): + return f"({SearchQuery.ast_to_string(node[1])} {node_type} {SearchQuery.ast_to_string(node[2])})" + elif node_type == "NOT": + return f"(NOT {SearchQuery.ast_to_string(node[1])})" + elif node_type == "FIELD_VAL": + field, val = node[1] + return f"{field}:{val}" + elif node_type == "PHRASE": + return f'"{node[1]}"' + elif node_type == "TERM": + return node[1] + return str(node) + + @staticmethod + def parse_boolean_query(query: str): + """ + Parse Boolean search query into structured format (AST). + + Supports: + - AND, OR, NOT operators + - Parentheses for grouping + - Field-specific searches: title:machine, author:smith, year:2020 + - Phrase searches: "machine learning" + - Wildcards: machin* (suffix), *learning (prefix) + + Returns: + AST node (tuple) + """ + query = query.strip() + if not query: + return None + + try: + # Tokenize + token_specification = [ + ("LPAR", r"\("), + ("RPAR", r"\)"), + ("AND", r"\bAND\b"), + ("OR", r"\bOR\b"), + ("NOT", r"\bNOT\b"), + ( + "FIELD_VAL", + r'\b(\w+):(?:(?:"([^"]+)")|(?:\'([^\']+)\')|([^\s\)\(]+))', + ), + ("PHRASE", r'"([^"]+)"|\'([^\']+)\''), + ("TERM", r"[^\s\)\(]+"), + ("SKIP", r"\s+"), + ] + tok_regex = "|".join( + f"(?P<{name}>{pattern})" for name, pattern in token_specification + ) + tokens = [] + for mo in re.finditer(tok_regex, query): + kind = mo.lastgroup + if kind == "SKIP": + continue + val = mo.group(kind) + if kind == "FIELD_VAL": + field_match = re.match(r"^(\w+):(.*)$", val) + field = field_match.group(1).lower() + raw_val = field_match.group(2) + if (raw_val.startswith('"') and raw_val.endswith('"')) or ( + raw_val.startswith("'") and raw_val.endswith("'") + ): + raw_val = raw_val[1:-1] + tokens.append(("FIELD_VAL", (field, raw_val))) + elif kind == "PHRASE": + if (val.startswith('"') and val.endswith('"')) or ( + val.startswith("'") and val.endswith("'") + ): + val = val[1:-1] + tokens.append(("PHRASE", val)) + elif kind == "TERM": + tokens.append(("TERM", val)) + else: + tokens.append((kind, val)) + + # Insert implicit ANDs + operand_types = {"TERM", "PHRASE", "FIELD_VAL", "RPAR"} + start_operand_types = {"TERM", "PHRASE", "FIELD_VAL", "LPAR", "NOT"} + tokens_with_ands = [] + for i, tok in enumerate(tokens): + if i > 0: + prev_tok = tokens[i - 1] + if prev_tok[0] in operand_types and tok[0] in start_operand_types: + tokens_with_ands.append(("AND", "AND")) + tokens_with_ands.append(tok) + + # Parse to AST using Shunting-yard + precedence = {"NOT": 3, "AND": 2, "OR": 1} + output_stack = [] + operator_stack = [] + + for tok_type, val in tokens_with_ands: + if tok_type in {"TERM", "PHRASE", "FIELD_VAL"}: + output_stack.append((tok_type, val)) + elif tok_type == "LPAR": + operator_stack.append((tok_type, val)) + elif tok_type == "RPAR": + while operator_stack and operator_stack[-1][0] != "LPAR": + op = operator_stack.pop() + if op[0] == "NOT": + if not output_stack: + raise ValueError("Invalid NOT query") + arg = output_stack.pop() + output_stack.append(("NOT", arg)) + else: + if len(output_stack) < 2: + raise ValueError( + f"Invalid query: missing arguments for {op[0]}" + ) + right = output_stack.pop() + left = output_stack.pop() + output_stack.append((op[0], left, right)) + if not operator_stack: + raise ValueError("Mismatched parentheses") + operator_stack.pop() # pop LPAR + elif tok_type in {"AND", "OR", "NOT"}: + while ( + operator_stack + and operator_stack[-1][0] in precedence + and precedence[operator_stack[-1][0]] >= precedence[tok_type] + ): + op = operator_stack.pop() + if op[0] == "NOT": + if not output_stack: + raise ValueError("Invalid NOT query") + arg = output_stack.pop() + output_stack.append(("NOT", arg)) + else: + if len(output_stack) < 2: + raise ValueError( + f"Invalid query: missing arguments for {op[0]}" + ) + right = output_stack.pop() + left = output_stack.pop() + output_stack.append((op[0], left, right)) + operator_stack.append((tok_type, val)) + + while operator_stack: + op = operator_stack.pop() + if op[0] == "LPAR": + raise ValueError("Mismatched parentheses") + if op[0] == "NOT": + if not output_stack: + raise ValueError("Invalid NOT query") + arg = output_stack.pop() + output_stack.append(("NOT", arg)) + else: + if len(output_stack) < 2: + raise ValueError( + f"Invalid query: missing arguments for {op[0]}" + ) + right = output_stack.pop() + left = output_stack.pop() + output_stack.append((op[0], left, right)) + + if not output_stack: + return None + if len(output_stack) > 1: + raise ValueError("Invalid query expression") + return output_stack[0] + + except Exception as e: + logger.warning( + f"Boolean parsing failed for query '{query}': {e}. Falling back to simple parsing." + ) + # Fallback to simple split AND-join + words = query.split() + if not words: + return None + node = ("TERM", words[0]) + for w in words[1:]: + node = ("AND", node, ("TERM", w)) + return node + + @staticmethod + def build_sql_filter(parsed_query, session=None): + """ + Build SQLAlchemy filter from parsed query. + + Returns: + SQLAlchemy filter expression + """ + if parsed_query is None: + return None + + if session is None: + session = SessionLocal() + + def evaluate_node(node): + if node is None: + return None + + node_type = node[0] + if node_type == "AND": + left = evaluate_node(node[1]) + right = evaluate_node(node[2]) + if left is not None and right is not None: + return and_(left, right) + return left or right + + elif node_type == "OR": + left = evaluate_node(node[1]) + right = evaluate_node(node[2]) + if left is not None and right is not None: + return or_(left, right) + return left or right + + elif node_type == "NOT": + arg = evaluate_node(node[1]) + if arg is not None: + return not_(arg) + return None + + elif node_type == "FIELD_VAL": + field, val = node[1] + return build_field_filter(field, val) + + elif node_type == "PHRASE": + val = node[1] + return or_( + Item.title.ilike(f"%{val}%"), + Item.abstract.ilike(f"%{val}%"), + Item.ai_keywords.ilike(f"%{val}%"), + ) + + elif node_type == "TERM": + val = node[1] + if "*" in val: + sql_val = val.replace("*", "%") + else: + sql_val = f"%{val}%" + return or_( + Item.title.ilike(sql_val), + Item.abstract.ilike(sql_val), + Item.doi.ilike(sql_val), + Item.ai_keywords.ilike(sql_val), + ) + return None + + def build_field_filter(field, value): + if "*" in value: + sql_val = value.replace("*", "%") + else: + sql_val = f"%{value}%" + + if field == "title": + return Item.title.ilike(sql_val) + elif field == "abstract": + return Item.abstract.ilike(sql_val) + elif field == "author": + return Item.id.in_( + session.query(Item.id) + .join(Item.authors) + .filter(Author.name.ilike(sql_val)) + ) + elif field == "year": + try: + year = int(value) + return db_year(Item.publication_date) == str(year) + except ValueError: + return None + elif field == "doi": + return Item.doi.ilike(sql_val) + elif field == "faculty": + return Item.id.in_( + session.query(Item.id) + .join(Item.collections) + .join(Collection.community) + .filter(Community.name.ilike(sql_val)) + ) + elif field == "department": + return Item.id.in_( + session.query(Item.id) + .join(Item.collections) + .filter(Collection.name.ilike(sql_val)) + ) + elif field == "keyword": + return or_( + Item.ai_keywords.ilike(sql_val), Item.dc_subject.ilike(sql_val) + ) + elif field == "language": + return Item.language_code == value.lower() + elif field == "oa" or field == "openaccess": + if value.lower() in ("true", "yes", "1"): + return Item.dc_rights.like("%openAccess%") + else: + return ~Item.dc_rights.like("%openAccess%") + else: + return or_( + Item.title.ilike(sql_val), + Item.abstract.ilike(sql_val), + Item.doi.ilike(sql_val), + Item.ai_keywords.ilike(sql_val), + ) + + return evaluate_node(parsed_query) + + @staticmethod + def execute_search( + query: str, + limit: int = 50, + offset: int = 0, + sort_by: str = "relevance", + filters: Optional[Dict] = None, + ) -> Dict: + """ + Execute advanced search query. + + Args: + query: Search query string (supports Boolean operators) + limit: Maximum results to return + offset: Pagination offset + sort_by: Sort order ('relevance', 'date', 'citations', 'title') + filters: Additional filters (year_from, year_to, oa_only, etc.) + + Returns: + { + 'total': int, + 'results': [paper_dict, ...], + 'query_parsed': str, + 'took_ms': float + } + """ + import time + + start_time = time.time() + + session = SessionLocal() + try: + # Parse query + parsed = SearchQuery.parse_boolean_query(query) + + # Build base query + q = session.query(Item) + + # Apply parsed query filters + sql_filter = SearchQuery.build_sql_filter(parsed, session) + if sql_filter is not None: + q = q.filter(sql_filter) + + # Apply additional filters + if filters: + if filters.get("year_from"): + q = q.filter( + db_year(Item.publication_date) >= str(filters["year_from"]) + ) + + if filters.get("year_to"): + q = q.filter( + db_year(Item.publication_date) <= str(filters["year_to"]) + ) + + if filters.get("oa_only"): + q = q.filter(Item.dc_rights.like("%openAccess%")) + + if filters.get("faculty"): + q = ( + q.join(Item.collections) + .join(Collection.community) + .filter(Community.name.ilike(f'%{filters["faculty"]}%')) + ) + + if filters.get("has_pdf"): + from uraas.database import File + + q = q.join(File, File.item_id == Item.id) + + # Get total count + total = q.count() + + # Apply sorting + if sort_by == "date": + q = q.order_by(Item.publication_date.desc().nullslast()) + elif sort_by == "title": + q = q.order_by(Item.title) + elif sort_by == "citations": + # Join with citation metrics if available + from uraas.services.citation_tracker import CitationMetrics + + q = q.outerjoin( + CitationMetrics, CitationMetrics.item_id == Item.id + ).order_by(CitationMetrics.citation_count.desc().nullslast()) + else: # relevance (default) + # Simple relevance: prioritize title matches + q = q.order_by(Item.created_at.desc()) + + # Pagination + results = q.limit(limit).offset(offset).all() + + # Format results + formatted_results = [] + for item in results: + formatted_results.append( + { + "id": item.id, + "title": item.title, + "abstract": item.abstract[:300] if item.abstract else None, + "doi": item.doi, + "url": item.url, + "year": ( + item.publication_date.year + if item.publication_date + else None + ), + "authors": [a.name for a in item.authors[:5]], + "faculty": ( + item.collections[0].community.name + if item.collections and item.collections[0].community + else None + ), + "department": ( + item.collections[0].name if item.collections else None + ), + "is_oa": "openAccess" in (item.dc_rights or ""), + "docid": item.docid, + "language": item.language_code, + "keywords": ( + item.ai_keywords.split(",")[:5] if item.ai_keywords else [] + ), + } + ) + + took_ms = (time.time() - start_time) * 1000 + + return { + "total": total, + "results": formatted_results, + "query_parsed": SearchQuery.ast_to_string(parsed), + "took_ms": round(took_ms, 2), + "page": offset // limit + 1, + "pages": (total + limit - 1) // limit, + } + + except Exception as e: + logger.error(f"Search failed: {e}") + return {"total": 0, "results": [], "error": str(e), "took_ms": 0} + finally: + session.close() + + @staticmethod + def get_search_suggestions(partial_query: str, field: str = "all") -> List[str]: + """ + Get autocomplete suggestions for search queries. + + Args: + partial_query: Partial search term + field: Field to search ('author', 'keyword', 'faculty', 'all') + + Returns: + List of suggested completions + """ + session = SessionLocal() + try: + suggestions = [] + + if field in ("author", "all"): + authors = ( + session.query(Author.name) + .filter(Author.name.ilike(f"%{partial_query}%")) + .limit(10) + .all() + ) + suggestions.extend([f"author:{a[0]}" for a in authors]) + + if field in ("faculty", "all"): + faculties = ( + session.query(Community.name) + .filter(Community.name.ilike(f"%{partial_query}%")) + .limit(5) + .all() + ) + suggestions.extend([f"faculty:{f[0]}" for f in faculties]) + + if field in ("keyword", "all"): + # Extract keywords from papers + items = ( + session.query(Item.ai_keywords) + .filter(Item.ai_keywords.ilike(f"%{partial_query}%")) + .limit(20) + .all() + ) + + keywords = set() + for item in items: + if item[0]: + for kw in item[0].split(","): + kw = kw.strip() + if partial_query.lower() in kw.lower(): + keywords.add(kw) + + suggestions.extend([f"keyword:{k}" for k in list(keywords)[:10]]) + + return suggestions[:15] + + finally: + session.close() + + +# ── Saved Searches ──────────────────────────────────────────────────────────── + + +class SavedSearch: + """Manage saved search queries for users.""" + + @staticmethod + def save_search(name: str, query: str, filters: Optional[Dict] = None) -> int: + """Save a search query for later reuse.""" + # TODO: Implement user authentication first + # For now, store in a simple table + pass + + @staticmethod + def get_saved_searches() -> List[Dict]: + """Get all saved searches.""" + # TODO: Implement + pass + + @staticmethod + def execute_saved_search(search_id: int) -> Dict: + """Execute a previously saved search.""" + # TODO: Implement + pass diff --git a/uraas/services/alignment_engine.py b/uraas/services/alignment_engine.py index 11df71811150af3a08eea45ab668a16e2154fe32..234edd44ce1775d5dc5209cca1acc573c4e90e43 100644 --- a/uraas/services/alignment_engine.py +++ b/uraas/services/alignment_engine.py @@ -1,202 +1,202 @@ -""" -Framework Alignment Engine — single source of truth for scoring papers -against AU charters, Agenda 2063 aspirations and regional-bloc themes. - -Hybrid score per pillar (0-100): - keyword = min(1.0, matched_keyword_count / KEYWORD_SATURATION) - semantic = max(0.0, cosine(embed(title+abstract), pillar_description_emb)) - hybrid = 100 * (0.6 * semantic + 0.4 * keyword) # embedding available - = 100 * keyword # keyword-only fallback - -Scores are computed ONCE at ingest (DatabaseStoragePipeline) or by -scripts/backfill_alignment.py and stored as JSON on items.alignment_scores. -Matched keywords are kept per pillar as auditable evidence (the credibility -layer surfaces them as chips, OSDG-style). Endpoints read precomputed -AlignmentAggregate rows — never score at request time. -""" - -import json -import logging -from datetime import datetime -from typing import Dict, List, Optional, Tuple - -from uraas.config.alignment_frameworks import ( - ALIGNMENT_VERSION, - FRAMEWORKS, - GAP_THRESHOLD, -) -from uraas.utils.ai_classifier import _clean_text, _keyword_score -from uraas.utils.embedding_model import EmbeddingModel - -logger = logging.getLogger(__name__) - -# Keyword hits saturate the evidence score at this count. -KEYWORD_SATURATION = 4 -# Pillars scoring below this are omitted from the stored JSON (keeps rows small). -MIN_RECORD = 5.0 -# Evidence chips per pillar. -MAX_EVIDENCE_KEYWORDS = 6 - -SEMANTIC_WEIGHT = 0.6 -KEYWORD_WEIGHT = 0.4 -# Cosine similarity between unrelated academic texts and pillar anchors sits -# around 0.2-0.3 (embedding-space baseline). Rescale so that floor maps to 0, -# keeping topically aligned papers (>0.5 cosine) strongly separated. -SEMANTIC_FLOOR = 0.25 - -# ── Pillar embedding cache (per ALIGNMENT_VERSION, computed once) ──────────── -_pillar_cache: Dict[int, Optional[dict]] = {} - - -def _pillar_embeddings() -> Optional[dict]: - """{(framework_key, pillar_key): normalized vector} or None (no model).""" - if ALIGNMENT_VERSION in _pillar_cache: - return _pillar_cache[ALIGNMENT_VERSION] - - keys, texts = [], [] - for fkey, framework in FRAMEWORKS.items(): - for pkey, pillar in framework["pillars"].items(): - keys.append((fkey, pkey)) - # Embed name + description + keywords for a richer pillar anchor. - texts.append( - f"{pillar['name']}. {pillar['description']} " - + ", ".join(pillar["keywords"]) - ) - - vectors = EmbeddingModel.encode(texts) - if vectors is None: - _pillar_cache[ALIGNMENT_VERSION] = None - return None - - import numpy as np - - norms = np.linalg.norm(vectors, axis=1, keepdims=True) - norms[norms == 0] = 1.0 - vectors = vectors / norms - result = {k: vectors[i] for i, k in enumerate(keys)} - _pillar_cache[ALIGNMENT_VERSION] = result - return result - - -def scoring_mode() -> str: - """'hybrid' when the embedding model is loaded, else 'keyword_only'.""" - return "hybrid" if EmbeddingModel.is_available() else "keyword_only" - - -def score_item_alignment( - title: str, abstract: str, dc_subject: str = "" -) -> Tuple[Optional[str], int]: - """Score one paper against every framework pillar. - - Returns (json_string_for_items.alignment_scores, ALIGNMENT_VERSION). - The JSON omits pillars below MIN_RECORD; returns (None, version) when - nothing aligns. Never raises — callers store the result at ingest.""" - try: - text = _clean_text(f"{title or ''} {abstract or ''} {dc_subject or ''}") - if not text.strip(): - return None, ALIGNMENT_VERSION - - pillar_embs = _pillar_embeddings() - doc_vec = None - if pillar_embs is not None: - encoded = EmbeddingModel.encode([f"{title or ''}. {abstract or ''}"]) - if encoded is not None: - import numpy as np - - doc_vec = encoded[0] - norm = np.linalg.norm(doc_vec) - doc_vec = doc_vec / norm if norm else None - - result = {} - for fkey, framework in FRAMEWORKS.items(): - pillars_out = {} - for pkey, pillar in framework["pillars"].items(): - hits, matched = _keyword_score(text, pillar["keywords"]) - keyword_score = min(1.0, hits / KEYWORD_SATURATION) - - semantic = 0.0 - if doc_vec is not None and pillar_embs is not None: - cosine = float(doc_vec @ pillar_embs[(fkey, pkey)]) - # Rescale past the unrelated-text baseline so noise -> 0. - semantic = max(0.0, (cosine - SEMANTIC_FLOOR) / (1.0 - SEMANTIC_FLOOR)) - hybrid = 100.0 * ( - SEMANTIC_WEIGHT * semantic + KEYWORD_WEIGHT * keyword_score - ) - else: - hybrid = 100.0 * keyword_score - - if hybrid < MIN_RECORD: - continue - pillars_out[pkey] = { - "score": round(hybrid, 1), - "semantic": round(semantic, 3), - "keyword": round(keyword_score, 3), - "matched_keywords": matched[:MAX_EVIDENCE_KEYWORDS], - } - if pillars_out: - result[fkey] = { - "overall": round( - max(p["score"] for p in pillars_out.values()), 1 - ), - "pillars": pillars_out, - } - - return (json.dumps(result) if result else None), ALIGNMENT_VERSION - except Exception as e: - logger.error("score_item_alignment failed: %s", e) - return None, ALIGNMENT_VERSION - - -def get_alignment(item) -> dict: - """Parse an Item's stored alignment JSON ({} when absent/stale).""" - if not getattr(item, "alignment_scores", None): - return {} - try: - return json.loads(item.alignment_scores) - except Exception: - return {} - - -def recompute_aggregates(session, institution: Optional[str] = None) -> int: - """Rebuild AlignmentAggregate rows for one institution (or "" = all). - - One row per (institution, framework, pillar): mean score over items where - the pillar is present, count of items at/above GAP_THRESHOLD, and the - top-5 item ids by score (evidence chips). Returns rows written.""" - from uraas.database import AlignmentAggregate, Item - - inst_key = institution or "" - q = session.query(Item).filter( - Item.special_collection_score > 0, Item.alignment_scores.isnot(None) - ) - if institution: - q = q.filter(Item.institution == institution) - - # (framework, pillar) -> list of (score, item_id) - buckets: Dict[Tuple[str, str], List[Tuple[float, int]]] = {} - for item in q.all(): - for fkey, fdata in get_alignment(item).items(): - for pkey, pdata in fdata.get("pillars", {}).items(): - buckets.setdefault((fkey, pkey), []).append( - (pdata.get("score", 0.0), item.id) - ) - - session.query(AlignmentAggregate).filter_by(institution=inst_key).delete() - rows = 0 - for (fkey, pkey), entries in buckets.items(): - scores = [s for s, _ in entries] - top = sorted(entries, reverse=True)[:5] - session.add( - AlignmentAggregate( - institution=inst_key, - framework=fkey, - pillar=pkey, - avg_score=round(sum(scores) / len(scores), 1), - paper_count=sum(1 for s in scores if s >= GAP_THRESHOLD), - top_item_ids=",".join(str(i) for _, i in top), - computed_at=datetime.utcnow(), - ) - ) - rows += 1 - session.commit() - return rows +""" +Framework Alignment Engine — single source of truth for scoring papers +against AU charters, Agenda 2063 aspirations and regional-bloc themes. + +Hybrid score per pillar (0-100): + keyword = min(1.0, matched_keyword_count / KEYWORD_SATURATION) + semantic = max(0.0, cosine(embed(title+abstract), pillar_description_emb)) + hybrid = 100 * (0.6 * semantic + 0.4 * keyword) # embedding available + = 100 * keyword # keyword-only fallback + +Scores are computed ONCE at ingest (DatabaseStoragePipeline) or by +scripts/backfill_alignment.py and stored as JSON on items.alignment_scores. +Matched keywords are kept per pillar as auditable evidence (the credibility +layer surfaces them as chips, OSDG-style). Endpoints read precomputed +AlignmentAggregate rows — never score at request time. +""" + +import json +import logging +from datetime import datetime +from typing import Dict, List, Optional, Tuple + +from uraas.config.alignment_frameworks import ( + ALIGNMENT_VERSION, + FRAMEWORKS, + GAP_THRESHOLD, +) +from uraas.utils.ai_classifier import _clean_text, _keyword_score +from uraas.utils.embedding_model import EmbeddingModel + +logger = logging.getLogger(__name__) + +# Keyword hits saturate the evidence score at this count. +KEYWORD_SATURATION = 4 +# Pillars scoring below this are omitted from the stored JSON (keeps rows small). +MIN_RECORD = 5.0 +# Evidence chips per pillar. +MAX_EVIDENCE_KEYWORDS = 6 + +SEMANTIC_WEIGHT = 0.6 +KEYWORD_WEIGHT = 0.4 +# Cosine similarity between unrelated academic texts and pillar anchors sits +# around 0.2-0.3 (embedding-space baseline). Rescale so that floor maps to 0, +# keeping topically aligned papers (>0.5 cosine) strongly separated. +SEMANTIC_FLOOR = 0.25 + +# ── Pillar embedding cache (per ALIGNMENT_VERSION, computed once) ──────────── +_pillar_cache: Dict[int, Optional[dict]] = {} + + +def _pillar_embeddings() -> Optional[dict]: + """{(framework_key, pillar_key): normalized vector} or None (no model).""" + if ALIGNMENT_VERSION in _pillar_cache: + return _pillar_cache[ALIGNMENT_VERSION] + + keys, texts = [], [] + for fkey, framework in FRAMEWORKS.items(): + for pkey, pillar in framework["pillars"].items(): + keys.append((fkey, pkey)) + # Embed name + description + keywords for a richer pillar anchor. + texts.append( + f"{pillar['name']}. {pillar['description']} " + + ", ".join(pillar["keywords"]) + ) + + vectors = EmbeddingModel.encode(texts) + if vectors is None: + _pillar_cache[ALIGNMENT_VERSION] = None + return None + + import numpy as np + + norms = np.linalg.norm(vectors, axis=1, keepdims=True) + norms[norms == 0] = 1.0 + vectors = vectors / norms + result = {k: vectors[i] for i, k in enumerate(keys)} + _pillar_cache[ALIGNMENT_VERSION] = result + return result + + +def scoring_mode() -> str: + """'hybrid' when the embedding model is loaded, else 'keyword_only'.""" + return "hybrid" if EmbeddingModel.is_available() else "keyword_only" + + +def score_item_alignment( + title: str, abstract: str, dc_subject: str = "" +) -> Tuple[Optional[str], int]: + """Score one paper against every framework pillar. + + Returns (json_string_for_items.alignment_scores, ALIGNMENT_VERSION). + The JSON omits pillars below MIN_RECORD; returns (None, version) when + nothing aligns. Never raises — callers store the result at ingest.""" + try: + text = _clean_text(f"{title or ''} {abstract or ''} {dc_subject or ''}") + if not text.strip(): + return None, ALIGNMENT_VERSION + + pillar_embs = _pillar_embeddings() + doc_vec = None + if pillar_embs is not None: + encoded = EmbeddingModel.encode([f"{title or ''}. {abstract or ''}"]) + if encoded is not None: + import numpy as np + + doc_vec = encoded[0] + norm = np.linalg.norm(doc_vec) + doc_vec = doc_vec / norm if norm else None + + result = {} + for fkey, framework in FRAMEWORKS.items(): + pillars_out = {} + for pkey, pillar in framework["pillars"].items(): + hits, matched = _keyword_score(text, pillar["keywords"]) + keyword_score = min(1.0, hits / KEYWORD_SATURATION) + + semantic = 0.0 + if doc_vec is not None and pillar_embs is not None: + cosine = float(doc_vec @ pillar_embs[(fkey, pkey)]) + # Rescale past the unrelated-text baseline so noise -> 0. + semantic = max(0.0, (cosine - SEMANTIC_FLOOR) / (1.0 - SEMANTIC_FLOOR)) + hybrid = 100.0 * ( + SEMANTIC_WEIGHT * semantic + KEYWORD_WEIGHT * keyword_score + ) + else: + hybrid = 100.0 * keyword_score + + if hybrid < MIN_RECORD: + continue + pillars_out[pkey] = { + "score": round(hybrid, 1), + "semantic": round(semantic, 3), + "keyword": round(keyword_score, 3), + "matched_keywords": matched[:MAX_EVIDENCE_KEYWORDS], + } + if pillars_out: + result[fkey] = { + "overall": round( + max(p["score"] for p in pillars_out.values()), 1 + ), + "pillars": pillars_out, + } + + return (json.dumps(result) if result else None), ALIGNMENT_VERSION + except Exception as e: + logger.error("score_item_alignment failed: %s", e) + return None, ALIGNMENT_VERSION + + +def get_alignment(item) -> dict: + """Parse an Item's stored alignment JSON ({} when absent/stale).""" + if not getattr(item, "alignment_scores", None): + return {} + try: + return json.loads(item.alignment_scores) + except Exception: + return {} + + +def recompute_aggregates(session, institution: Optional[str] = None) -> int: + """Rebuild AlignmentAggregate rows for one institution (or "" = all). + + One row per (institution, framework, pillar): mean score over items where + the pillar is present, count of items at/above GAP_THRESHOLD, and the + top-5 item ids by score (evidence chips). Returns rows written.""" + from uraas.database import AlignmentAggregate, Item + + inst_key = institution or "" + q = session.query(Item).filter( + Item.special_collection_score > 0, Item.alignment_scores.isnot(None) + ) + if institution: + q = q.filter(Item.institution == institution) + + # (framework, pillar) -> list of (score, item_id) + buckets: Dict[Tuple[str, str], List[Tuple[float, int]]] = {} + for item in q.all(): + for fkey, fdata in get_alignment(item).items(): + for pkey, pdata in fdata.get("pillars", {}).items(): + buckets.setdefault((fkey, pkey), []).append( + (pdata.get("score", 0.0), item.id) + ) + + session.query(AlignmentAggregate).filter_by(institution=inst_key).delete() + rows = 0 + for (fkey, pkey), entries in buckets.items(): + scores = [s for s, _ in entries] + top = sorted(entries, reverse=True)[:5] + session.add( + AlignmentAggregate( + institution=inst_key, + framework=fkey, + pillar=pkey, + avg_score=round(sum(scores) / len(scores), 1), + paper_count=sum(1 for s in scores if s >= GAP_THRESHOLD), + top_item_ids=",".join(str(i) for _, i in top), + computed_at=datetime.utcnow(), + ) + ) + rows += 1 + session.commit() + return rows diff --git a/uraas/services/batch_approval.py b/uraas/services/batch_approval.py index 38d9da08d6c06523a60b265a77624a8456c437c2..ea7946a0aa80bbf21a8d1ae89a3bbe16a2698ba4 100644 --- a/uraas/services/batch_approval.py +++ b/uraas/services/batch_approval.py @@ -1,337 +1,337 @@ -"""Batch approval orchestration for IR deposits. - -Lifecycle ---------- -pending_approval ← batch created, approval email sent - │ - ├─(approve link clicked)─→ approved ─→ depositing ─→ completed - │ └─→ failed - └─(reject link clicked) ─→ rejected - -Security --------- -* Tokens are 32-byte cryptographically random values (URL-safe base64, ~43 chars). -* Tokens expire after APPROVAL_LINK_TTL_HOURS (default 48 h). -* Each token is single-use: once consumed (approved or rejected) its status - changes and subsequent requests return 410 Gone. -""" - -import json -import logging -import secrets -import threading -from datetime import datetime, timedelta - -from uraas.config import config -from uraas.database import DepositBatch, Item, File, SessionLocal -from uraas.services.email_service import send_batch_approval_request - -logger = logging.getLogger(__name__) - -APPROVAL_LINK_TTL_HOURS = 48 - - -# ── Public API ──────────────────────────────────────────────────────────────── - -def queue_batch( - *, - item_ids: list[int], - collection_uuid: str, - collection_name: str, - approval_email: str, - requested_by: str, -) -> dict: - """Persist a pending batch and fire the approval email. - - Returns a result dict with status, batch_id, and email_sent flag. - """ - if not item_ids: - return {"status": "error", "message": "No item IDs provided"} - - token = secrets.token_urlsafe(32) - expires = datetime.utcnow() + timedelta(hours=APPROVAL_LINK_TTL_HOURS) - - db = SessionLocal() - try: - batch = DepositBatch( - token=token, - status="pending_approval", - approval_email=approval_email, - collection_uuid=collection_uuid, - collection_name=collection_name, - item_ids_json=json.dumps(item_ids), - item_count=len(item_ids), - requested_by=requested_by, - expires_at=expires, - ) - db.add(batch) - db.commit() - db.refresh(batch) - batch_id = batch.id - finally: - db.close() - - approve_url = f"{config.DASHBOARD_BASE_URL}/api/ir/batch/{token}/approve" - reject_url = f"{config.DASHBOARD_BASE_URL}/api/ir/batch/{token}/reject" - - email_sent = send_batch_approval_request( - to_email=approval_email, - approve_url=approve_url, - reject_url=reject_url, - batch_id=batch_id, - item_count=len(item_ids), - collection_name=collection_name, - requested_by=requested_by, - expires_hours=APPROVAL_LINK_TTL_HOURS, - ) - - return { - "status": "queued", - "batch_id": batch_id, - "item_count": len(item_ids), - "approval_email": approval_email, - "email_sent": email_sent, - "expires_at": expires.isoformat(), - # Return the approve URL so an admin can action it directly from the - # dashboard if SMTP is not yet configured. - "approve_url": approve_url, - } - - -def approve_batch(token: str) -> dict: - """Validate token and kick off the deposit in a background thread. - - Returns a result dict suitable for rendering an HTML confirmation page. - """ - db = SessionLocal() - try: - batch = db.query(DepositBatch).filter_by(token=token).first() - if not batch: - return {"status": "not_found", "message": "Approval link not found or already used."} - if batch.status != "pending_approval": - return { - "status": "already_actioned", - "message": f"This batch has already been {batch.status}.", - "batch_status": batch.status, - "batch_id": batch.id, - } - if batch.expires_at and datetime.utcnow() > batch.expires_at: - batch.status = "rejected" - batch.notes = "Approval link expired" - batch.updated_at = datetime.utcnow() - db.commit() - return {"status": "expired", "message": "This approval link has expired."} - - batch.status = "approved" - batch.approved_at = datetime.utcnow() - batch.updated_at = datetime.utcnow() - db.commit() - batch_id = batch.id - finally: - db.close() - - # Start deposit in background so the HTTP response returns immediately - t = threading.Thread(target=_run_deposit, args=(batch_id,), daemon=True) - t.start() - - return { - "status": "approved", - "batch_id": batch_id, - "message": ( - f"Batch #{batch_id} approved. Deposit is running in the background. " - "Check the dashboard for progress." - ), - } - - -def reject_batch(token: str, reason: str = "") -> dict: - """Mark the batch as rejected. Called from the email rejection link.""" - db = SessionLocal() - try: - batch = db.query(DepositBatch).filter_by(token=token).first() - if not batch: - return {"status": "not_found", "message": "Rejection link not found or already used."} - if batch.status != "pending_approval": - return { - "status": "already_actioned", - "message": f"This batch has already been {batch.status}.", - "batch_status": batch.status, - "batch_id": batch.id, - } - batch.status = "rejected" - batch.notes = reason or "Rejected via email link" - batch.updated_at = datetime.utcnow() - db.commit() - return { - "status": "rejected", - "batch_id": batch.id, - "message": f"Batch #{batch.id} has been rejected and will not be deposited.", - } - finally: - db.close() - - -def get_batch(token: str) -> dict | None: - """Return batch detail dict for a given token (admin status polling).""" - db = SessionLocal() - try: - batch = db.query(DepositBatch).filter_by(token=token).first() - return _batch_to_dict(batch) if batch else None - finally: - db.close() - - -def get_batches(limit: int = 50) -> list[dict]: - """Return all batches most-recent-first for the admin panel.""" - db = SessionLocal() - try: - rows = ( - db.query(DepositBatch) - .order_by(DepositBatch.created_at.desc()) - .limit(limit) - .all() - ) - return [_batch_to_dict(b) for b in rows] - finally: - db.close() - - -# ── Internal deposit runner ─────────────────────────────────────────────────── - -def _run_deposit(batch_id: int): - """Background thread: iterate over the batch items and deposit each one.""" - from uraas.services.ir_client import DSpaceClient, IRConnectionError - - db = SessionLocal() - try: - batch = db.query(DepositBatch).filter_by(id=batch_id).first() - if not batch or batch.status != "approved": - return - - batch.status = "depositing" - batch.updated_at = datetime.utcnow() - db.commit() - - item_ids: list[int] = json.loads(batch.item_ids_json or "[]") - collection_uuid: str = batch.collection_uuid or "" - deposit_log: list[dict] = [] - ok_count = 0 - fail_count = 0 - - client = DSpaceClient() - try: - client.login() - except IRConnectionError as exc: - _mark_failed(db, batch, f"IR login failed: {exc}") - return - - UNILAG_ROR = "https://ror.org/05rk03822" - - for item_id in item_ids: - item = db.query(Item).filter_by(id=item_id).first() - if not item: - deposit_log.append({"item_id": item_id, "status": "error", "message": "Not found in local DB"}) - fail_count += 1 - continue - - # Only deposit papers affiliated with UNILAG — other institutions - # will get their own IR integrations later. - is_unilag = (item.ror == UNILAG_ROR) or ( - "university of lagos" in (item.institution or "").lower() - ) - if not is_unilag: - deposit_log.append({ - "item_id": item_id, - "status": "skipped", - "message": f"Not a UNILAG paper (institution: {item.institution!r})", - }) - continue - - # Resolve local PDF path if one exists - pdf_path: str | None = None - file_rec = db.query(File).filter_by(item_id=item_id).first() - if file_rec and file_rec.file_path: - import os - candidate = file_rec.file_path - if not os.path.isabs(candidate): - candidate = os.path.join(config.STORAGE_PATH, candidate) - if os.path.exists(candidate): - pdf_path = candidate - - try: - result = client.deposit_item(collection_uuid, item, pdf_path=pdf_path) - except Exception as exc: - result = {"status": "error", "message": str(exc)[:500]} - - result["item_id"] = item_id - deposit_log.append(result) - - if result["status"] == "ok": - ok_count += 1 - elif result["status"] == "duplicate": - ok_count += 1 # not a failure — item is already in IR - else: - fail_count += 1 - - # Persist progress periodically - if len(deposit_log) % 10 == 0: - batch.deposited_count = ok_count - batch.failed_count = fail_count - batch.deposit_log = json.dumps(deposit_log) - batch.updated_at = datetime.utcnow() - db.commit() - - # Final state - batch.deposited_count = ok_count - batch.failed_count = fail_count - batch.deposit_log = json.dumps(deposit_log) - batch.status = "completed" if fail_count == 0 else "failed" - batch.completed_at = datetime.utcnow() - batch.updated_at = datetime.utcnow() - if fail_count: - batch.notes = f"{fail_count} item(s) failed to deposit — see deposit_log for details" - db.commit() - logger.info( - "Batch %s deposit complete: %d ok, %d failed", batch_id, ok_count, fail_count - ) - - except Exception as exc: - logger.exception("Unexpected error in deposit batch %s: %s", batch_id, exc) - try: - _mark_failed(db, db.query(DepositBatch).filter_by(id=batch_id).first(), str(exc)) - except Exception: - pass - finally: - db.close() - - -def _mark_failed(db, batch, reason: str): - if batch: - batch.status = "failed" - batch.notes = reason - batch.updated_at = datetime.utcnow() - db.commit() - logger.error("Deposit batch failed: %s", reason) - - -# ── Serialisation helper ────────────────────────────────────────────────────── - -def _batch_to_dict(batch: DepositBatch) -> dict: - return { - "id": batch.id, - "status": batch.status, - "approval_email": batch.approval_email, - "collection_uuid": batch.collection_uuid, - "collection_name": batch.collection_name, - "item_count": batch.item_count, - "deposited_count": batch.deposited_count, - "failed_count": batch.failed_count, - "requested_by": batch.requested_by, - "notes": batch.notes, - "created_at": batch.created_at.isoformat() if batch.created_at else None, - "updated_at": batch.updated_at.isoformat() if batch.updated_at else None, - "expires_at": batch.expires_at.isoformat() if batch.expires_at else None, - "approved_at": batch.approved_at.isoformat() if batch.approved_at else None, - "completed_at": batch.completed_at.isoformat() if batch.completed_at else None, - "deposit_log": json.loads(batch.deposit_log or "[]"), - } +"""Batch approval orchestration for IR deposits. + +Lifecycle +--------- +pending_approval ← batch created, approval email sent + │ + ├─(approve link clicked)─→ approved ─→ depositing ─→ completed + │ └─→ failed + └─(reject link clicked) ─→ rejected + +Security +-------- +* Tokens are 32-byte cryptographically random values (URL-safe base64, ~43 chars). +* Tokens expire after APPROVAL_LINK_TTL_HOURS (default 48 h). +* Each token is single-use: once consumed (approved or rejected) its status + changes and subsequent requests return 410 Gone. +""" + +import json +import logging +import secrets +import threading +from datetime import datetime, timedelta + +from uraas.config import config +from uraas.database import DepositBatch, Item, File, SessionLocal +from uraas.services.email_service import send_batch_approval_request + +logger = logging.getLogger(__name__) + +APPROVAL_LINK_TTL_HOURS = 48 + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def queue_batch( + *, + item_ids: list[int], + collection_uuid: str, + collection_name: str, + approval_email: str, + requested_by: str, +) -> dict: + """Persist a pending batch and fire the approval email. + + Returns a result dict with status, batch_id, and email_sent flag. + """ + if not item_ids: + return {"status": "error", "message": "No item IDs provided"} + + token = secrets.token_urlsafe(32) + expires = datetime.utcnow() + timedelta(hours=APPROVAL_LINK_TTL_HOURS) + + db = SessionLocal() + try: + batch = DepositBatch( + token=token, + status="pending_approval", + approval_email=approval_email, + collection_uuid=collection_uuid, + collection_name=collection_name, + item_ids_json=json.dumps(item_ids), + item_count=len(item_ids), + requested_by=requested_by, + expires_at=expires, + ) + db.add(batch) + db.commit() + db.refresh(batch) + batch_id = batch.id + finally: + db.close() + + approve_url = f"{config.DASHBOARD_BASE_URL}/api/ir/batch/{token}/approve" + reject_url = f"{config.DASHBOARD_BASE_URL}/api/ir/batch/{token}/reject" + + email_sent = send_batch_approval_request( + to_email=approval_email, + approve_url=approve_url, + reject_url=reject_url, + batch_id=batch_id, + item_count=len(item_ids), + collection_name=collection_name, + requested_by=requested_by, + expires_hours=APPROVAL_LINK_TTL_HOURS, + ) + + return { + "status": "queued", + "batch_id": batch_id, + "item_count": len(item_ids), + "approval_email": approval_email, + "email_sent": email_sent, + "expires_at": expires.isoformat(), + # Return the approve URL so an admin can action it directly from the + # dashboard if SMTP is not yet configured. + "approve_url": approve_url, + } + + +def approve_batch(token: str) -> dict: + """Validate token and kick off the deposit in a background thread. + + Returns a result dict suitable for rendering an HTML confirmation page. + """ + db = SessionLocal() + try: + batch = db.query(DepositBatch).filter_by(token=token).first() + if not batch: + return {"status": "not_found", "message": "Approval link not found or already used."} + if batch.status != "pending_approval": + return { + "status": "already_actioned", + "message": f"This batch has already been {batch.status}.", + "batch_status": batch.status, + "batch_id": batch.id, + } + if batch.expires_at and datetime.utcnow() > batch.expires_at: + batch.status = "rejected" + batch.notes = "Approval link expired" + batch.updated_at = datetime.utcnow() + db.commit() + return {"status": "expired", "message": "This approval link has expired."} + + batch.status = "approved" + batch.approved_at = datetime.utcnow() + batch.updated_at = datetime.utcnow() + db.commit() + batch_id = batch.id + finally: + db.close() + + # Start deposit in background so the HTTP response returns immediately + t = threading.Thread(target=_run_deposit, args=(batch_id,), daemon=True) + t.start() + + return { + "status": "approved", + "batch_id": batch_id, + "message": ( + f"Batch #{batch_id} approved. Deposit is running in the background. " + "Check the dashboard for progress." + ), + } + + +def reject_batch(token: str, reason: str = "") -> dict: + """Mark the batch as rejected. Called from the email rejection link.""" + db = SessionLocal() + try: + batch = db.query(DepositBatch).filter_by(token=token).first() + if not batch: + return {"status": "not_found", "message": "Rejection link not found or already used."} + if batch.status != "pending_approval": + return { + "status": "already_actioned", + "message": f"This batch has already been {batch.status}.", + "batch_status": batch.status, + "batch_id": batch.id, + } + batch.status = "rejected" + batch.notes = reason or "Rejected via email link" + batch.updated_at = datetime.utcnow() + db.commit() + return { + "status": "rejected", + "batch_id": batch.id, + "message": f"Batch #{batch.id} has been rejected and will not be deposited.", + } + finally: + db.close() + + +def get_batch(token: str) -> dict | None: + """Return batch detail dict for a given token (admin status polling).""" + db = SessionLocal() + try: + batch = db.query(DepositBatch).filter_by(token=token).first() + return _batch_to_dict(batch) if batch else None + finally: + db.close() + + +def get_batches(limit: int = 50) -> list[dict]: + """Return all batches most-recent-first for the admin panel.""" + db = SessionLocal() + try: + rows = ( + db.query(DepositBatch) + .order_by(DepositBatch.created_at.desc()) + .limit(limit) + .all() + ) + return [_batch_to_dict(b) for b in rows] + finally: + db.close() + + +# ── Internal deposit runner ─────────────────────────────────────────────────── + +def _run_deposit(batch_id: int): + """Background thread: iterate over the batch items and deposit each one.""" + from uraas.services.ir_client import DSpaceClient, IRConnectionError + + db = SessionLocal() + try: + batch = db.query(DepositBatch).filter_by(id=batch_id).first() + if not batch or batch.status != "approved": + return + + batch.status = "depositing" + batch.updated_at = datetime.utcnow() + db.commit() + + item_ids: list[int] = json.loads(batch.item_ids_json or "[]") + collection_uuid: str = batch.collection_uuid or "" + deposit_log: list[dict] = [] + ok_count = 0 + fail_count = 0 + + client = DSpaceClient() + try: + client.login() + except IRConnectionError as exc: + _mark_failed(db, batch, f"IR login failed: {exc}") + return + + UNILAG_ROR = "https://ror.org/05rk03822" + + for item_id in item_ids: + item = db.query(Item).filter_by(id=item_id).first() + if not item: + deposit_log.append({"item_id": item_id, "status": "error", "message": "Not found in local DB"}) + fail_count += 1 + continue + + # Only deposit papers affiliated with UNILAG — other institutions + # will get their own IR integrations later. + is_unilag = (item.ror == UNILAG_ROR) or ( + "university of lagos" in (item.institution or "").lower() + ) + if not is_unilag: + deposit_log.append({ + "item_id": item_id, + "status": "skipped", + "message": f"Not a UNILAG paper (institution: {item.institution!r})", + }) + continue + + # Resolve local PDF path if one exists + pdf_path: str | None = None + file_rec = db.query(File).filter_by(item_id=item_id).first() + if file_rec and file_rec.file_path: + import os + candidate = file_rec.file_path + if not os.path.isabs(candidate): + candidate = os.path.join(config.STORAGE_PATH, candidate) + if os.path.exists(candidate): + pdf_path = candidate + + try: + result = client.deposit_item(collection_uuid, item, pdf_path=pdf_path) + except Exception as exc: + result = {"status": "error", "message": str(exc)[:500]} + + result["item_id"] = item_id + deposit_log.append(result) + + if result["status"] == "ok": + ok_count += 1 + elif result["status"] == "duplicate": + ok_count += 1 # not a failure — item is already in IR + else: + fail_count += 1 + + # Persist progress periodically + if len(deposit_log) % 10 == 0: + batch.deposited_count = ok_count + batch.failed_count = fail_count + batch.deposit_log = json.dumps(deposit_log) + batch.updated_at = datetime.utcnow() + db.commit() + + # Final state + batch.deposited_count = ok_count + batch.failed_count = fail_count + batch.deposit_log = json.dumps(deposit_log) + batch.status = "completed" if fail_count == 0 else "failed" + batch.completed_at = datetime.utcnow() + batch.updated_at = datetime.utcnow() + if fail_count: + batch.notes = f"{fail_count} item(s) failed to deposit — see deposit_log for details" + db.commit() + logger.info( + "Batch %s deposit complete: %d ok, %d failed", batch_id, ok_count, fail_count + ) + + except Exception as exc: + logger.exception("Unexpected error in deposit batch %s: %s", batch_id, exc) + try: + _mark_failed(db, db.query(DepositBatch).filter_by(id=batch_id).first(), str(exc)) + except Exception: + pass + finally: + db.close() + + +def _mark_failed(db, batch, reason: str): + if batch: + batch.status = "failed" + batch.notes = reason + batch.updated_at = datetime.utcnow() + db.commit() + logger.error("Deposit batch failed: %s", reason) + + +# ── Serialisation helper ────────────────────────────────────────────────────── + +def _batch_to_dict(batch: DepositBatch) -> dict: + return { + "id": batch.id, + "status": batch.status, + "approval_email": batch.approval_email, + "collection_uuid": batch.collection_uuid, + "collection_name": batch.collection_name, + "item_count": batch.item_count, + "deposited_count": batch.deposited_count, + "failed_count": batch.failed_count, + "requested_by": batch.requested_by, + "notes": batch.notes, + "created_at": batch.created_at.isoformat() if batch.created_at else None, + "updated_at": batch.updated_at.isoformat() if batch.updated_at else None, + "expires_at": batch.expires_at.isoformat() if batch.expires_at else None, + "approved_at": batch.approved_at.isoformat() if batch.approved_at else None, + "completed_at": batch.completed_at.isoformat() if batch.completed_at else None, + "deposit_log": json.loads(batch.deposit_log or "[]"), + } diff --git a/uraas/services/citation_tracker.py b/uraas/services/citation_tracker.py index 91f12fba2a877c5fb1a8ca8aa66eadf0f6775b5d..a7e4aa4d35809879c94ebb817e629c20504fe073 100644 --- a/uraas/services/citation_tracker.py +++ b/uraas/services/citation_tracker.py @@ -1,422 +1,422 @@ -""" -Citation Tracking Service -Fetches citation counts and citation graphs from OpenAlex and Crossref. -Calculates h-index and other bibliometric indicators. -""" - -import logging -from datetime import datetime, timedelta -from typing import Dict, List, Optional, Tuple - -import requests -from sqlalchemy import Column, DateTime, Float, ForeignKey, Integer, String, Text -from sqlalchemy.orm import relationship - -from uraas.database import Author, Base, Item, SessionLocal - -logger = logging.getLogger(__name__) - - -# ── New Database Models for Citations ──────────────────────────────────────── - - -class Citation(Base): - """Citation relationship between papers.""" - - __tablename__ = "citations" - - id = Column(Integer, primary_key=True) - citing_item_id = Column(Integer, ForeignKey("items.id", ondelete="CASCADE")) - cited_item_id = Column(Integer, ForeignKey("items.id", ondelete="CASCADE")) - citation_date = Column(DateTime) - source = Column(String(50)) # 'openalex', 'crossref', 'manual' - - citing_item = relationship("Item", foreign_keys=[citing_item_id]) - cited_item = relationship("Item", foreign_keys=[cited_item_id]) - - -class CitationMetrics(Base): - """Cached citation metrics for papers.""" - - __tablename__ = "citation_metrics" - - id = Column(Integer, primary_key=True) - item_id = Column(Integer, ForeignKey("items.id", ondelete="CASCADE"), unique=True) - citation_count = Column(Integer, default=0) - h_index = Column(Integer, default=0) - i10_index = Column(Integer, default=0) # papers with 10+ citations - last_updated = Column(DateTime, default=datetime.utcnow) - - item = relationship("Item") - - -class AuthorMetrics(Base): - """Cached bibliometric indicators for authors.""" - - __tablename__ = "author_metrics" - - id = Column(Integer, primary_key=True) - author_id = Column( - Integer, ForeignKey("authors.id", ondelete="CASCADE"), unique=True - ) - total_citations = Column(Integer, default=0) - h_index = Column(Integer, default=0) - i10_index = Column(Integer, default=0) - total_papers = Column(Integer, default=0) - last_updated = Column(DateTime, default=datetime.utcnow) - - author = relationship("Author") - - -# ── Citation Fetching Service ───────────────────────────────────────────────── - - -class CitationTracker: - """Fetches and tracks citations from external APIs.""" - - OPENALEX_API = "https://api.openalex.org/works" - CROSSREF_API = "https://api.crossref.org/works" - - @staticmethod - def fetch_citations_openalex(doi: str) -> Optional[Dict]: - """ - Fetch citation data from OpenAlex. - - Returns: - { - 'citation_count': int, - 'cited_by_api_url': str, - 'citations': [{'doi': str, 'title': str, 'year': int}, ...] - } - """ - try: - url = f"{CitationTracker.OPENALEX_API}/doi:{doi}" - headers = {"User-Agent": "URAAS/1.0 (mailto:library@unilag.edu.ng)"} - response = requests.get(url, headers=headers, timeout=10) - - if response.status_code != 200: - return None - - data = response.json() - citation_count = data.get("cited_by_count", 0) - cited_by_url = data.get("cited_by_api_url") - - # Fetch citing papers - citations = [] - if cited_by_url and citation_count > 0: - cite_response = requests.get(cited_by_url, headers=headers, timeout=10) - if cite_response.status_code == 200: - cite_data = cite_response.json() - for result in cite_data.get("results", [])[:100]: # Limit to 100 - citations.append( - { - "doi": result.get("doi", "").replace( - "https://doi.org/", "" - ), - "title": result.get("title", ""), - "year": result.get("publication_year"), - "authors": [ - a.get("author", {}).get("display_name") - for a in result.get("authorships", [])[:3] - ], - } - ) - - return { - "citation_count": citation_count, - "cited_by_api_url": cited_by_url, - "citations": citations, - } - - except Exception as e: - logger.error(f"OpenAlex citation fetch failed for {doi}: {e}") - return None - - @staticmethod - def fetch_citations_crossref(doi: str) -> Optional[int]: - """Fetch citation count from Crossref (simpler, just count).""" - try: - url = f"{CitationTracker.CROSSREF_API}/{doi}" - headers = {"User-Agent": "URAAS/1.0 (mailto:library@unilag.edu.ng)"} - response = requests.get(url, headers=headers, timeout=10) - - if response.status_code != 200: - return None - - data = response.json() - return data.get("message", {}).get("is-referenced-by-count", 0) - - except Exception as e: - logger.error(f"Crossref citation fetch failed for {doi}: {e}") - return None - - @staticmethod - def update_paper_citations(item_id: int) -> bool: - """Update citation metrics for a single paper.""" - session = SessionLocal() - try: - item = session.query(Item).filter_by(id=item_id).first() - if not item or not item.doi: - return False - - # Try OpenAlex first (more detailed) - oa_data = CitationTracker.fetch_citations_openalex(item.doi) - - if oa_data: - citation_count = oa_data["citation_count"] - - # Update or create metrics - metrics = ( - session.query(CitationMetrics).filter_by(item_id=item_id).first() - ) - if not metrics: - metrics = CitationMetrics(item_id=item_id) - session.add(metrics) - - metrics.citation_count = citation_count - metrics.last_updated = datetime.utcnow() - - # Store citation relationships - for cite in oa_data["citations"]: - if cite["doi"]: - # Check if citing paper exists in our DB - citing_item = ( - session.query(Item).filter_by(doi=cite["doi"]).first() - ) - if citing_item: - # Create citation link - existing = ( - session.query(Citation) - .filter_by( - citing_item_id=citing_item.id, cited_item_id=item_id - ) - .first() - ) - - if not existing: - citation = Citation( - citing_item_id=citing_item.id, - cited_item_id=item_id, - citation_date=( - datetime(cite["year"], 1, 1) - if cite["year"] - else None - ), - source="openalex", - ) - session.add(citation) - - session.commit() - logger.info( - f"Updated citations for item {item_id}: {citation_count} citations" - ) - return True - - # Fallback to Crossref - cr_count = CitationTracker.fetch_citations_crossref(item.doi) - if cr_count is not None: - metrics = ( - session.query(CitationMetrics).filter_by(item_id=item_id).first() - ) - if not metrics: - metrics = CitationMetrics(item_id=item_id) - session.add(metrics) - - metrics.citation_count = cr_count - metrics.last_updated = datetime.utcnow() - session.commit() - return True - - return False - - except Exception as e: - session.rollback() - logger.error(f"Failed to update citations for item {item_id}: {e}") - return False - finally: - session.close() - - @staticmethod - def calculate_h_index(citation_counts: List[int]) -> int: - """ - Calculate h-index from list of citation counts. - h-index = largest number h such that h papers have at least h citations each. - - Example: [100, 50, 30, 20, 15, 10, 8, 5, 3, 2, 1, 1, 0, 0] - - Paper 1: 100 citations ≥ 1 ✓ - - Paper 2: 50 citations ≥ 2 ✓ - - ... - - Paper 10: 2 citations ≥ 10 ✗ - Result: h-index = 9 - """ - if not citation_counts: - return 0 - - sorted_counts = sorted(citation_counts, reverse=True) - h = 0 - for i, count in enumerate(sorted_counts, start=1): - if count >= i: - h = i - else: - break - return h - - @staticmethod - def update_author_metrics(author_id: int) -> bool: - """Calculate and update bibliometric indicators for an author.""" - session = SessionLocal() - try: - author = session.query(Author).filter_by(id=author_id).first() - if not author: - return False - - # Get all papers by this author with citation metrics - papers = ( - session.query(Item, CitationMetrics) - .join(Item.authors) - .outerjoin(CitationMetrics, CitationMetrics.item_id == Item.id) - .filter(Author.id == author_id) - .all() - ) - - if not papers: - return False - - citation_counts = [m.citation_count if m else 0 for _, m in papers] - total_citations = sum(citation_counts) - h_index = CitationTracker.calculate_h_index(citation_counts) - i10_index = sum(1 for c in citation_counts if c >= 10) - - # Update or create author metrics - metrics = ( - session.query(AuthorMetrics).filter_by(author_id=author_id).first() - ) - if not metrics: - metrics = AuthorMetrics(author_id=author_id) - session.add(metrics) - - metrics.total_citations = total_citations - metrics.h_index = h_index - metrics.i10_index = i10_index - metrics.total_papers = len(papers) - metrics.last_updated = datetime.utcnow() - - session.commit() - logger.info( - f"Updated metrics for author {author.name}: h-index={h_index}, citations={total_citations}" - ) - return True - - except Exception as e: - session.rollback() - logger.error(f"Failed to update author metrics for {author_id}: {e}") - return False - finally: - session.close() - - @staticmethod - def bulk_update_citations(limit: int = 100, force: bool = False) -> Dict: - """ - Update citations for papers that haven't been updated recently. - - Args: - limit: Maximum number of papers to update - force: Update all papers regardless of last update time - - Returns: - {'updated': int, 'failed': int, 'skipped': int} - """ - session = SessionLocal() - stats = {"updated": 0, "failed": 0, "skipped": 0} - - try: - # Find papers with DOIs that need updating - cutoff_date = datetime.utcnow() - timedelta(days=7) # Update weekly - - query = session.query(Item).filter(Item.doi.isnot(None)) - - if not force: - # Only update papers not updated in last 7 days - query = query.outerjoin(CitationMetrics).filter( - (CitationMetrics.last_updated.is_(None)) - | (CitationMetrics.last_updated < cutoff_date) - ) - - papers = query.limit(limit).all() - - for paper in papers: - success = CitationTracker.update_paper_citations(paper.id) - if success: - stats["updated"] += 1 - else: - stats["failed"] += 1 - - logger.info(f"Bulk citation update: {stats}") - return stats - - finally: - session.close() - - -# ── API Helper Functions ────────────────────────────────────────────────────── - - -def get_paper_citations(item_id: int) -> Dict: - """Get citation data for a paper.""" - session = SessionLocal() - try: - metrics = session.query(CitationMetrics).filter_by(item_id=item_id).first() - - # Get citing papers - citations = ( - session.query(Citation, Item) - .join(Item, Citation.citing_item_id == Item.id) - .filter(Citation.cited_item_id == item_id) - .all() - ) - - return { - "citation_count": metrics.citation_count if metrics else 0, - "last_updated": metrics.last_updated.isoformat() if metrics else None, - "citing_papers": [ - { - "id": item.id, - "title": item.title, - "doi": item.doi, - "year": ( - item.publication_date.year if item.publication_date else None - ), - "authors": [a.name for a in item.authors[:3]], - } - for _, item in citations - ], - } - finally: - session.close() - - -def get_author_bibliometrics(author_id: int) -> Dict: - """Get bibliometric indicators for an author.""" - session = SessionLocal() - try: - metrics = session.query(AuthorMetrics).filter_by(author_id=author_id).first() - author = session.query(Author).filter_by(id=author_id).first() - - if not metrics or not author: - return {} - - return { - "author_name": author.name, - "total_papers": metrics.total_papers, - "total_citations": metrics.total_citations, - "h_index": metrics.h_index, - "i10_index": metrics.i10_index, - "last_updated": metrics.last_updated.isoformat(), - "citations_per_paper": ( - round(metrics.total_citations / metrics.total_papers, 1) - if metrics.total_papers - else 0 - ), - } - finally: - session.close() +""" +Citation Tracking Service +Fetches citation counts and citation graphs from OpenAlex and Crossref. +Calculates h-index and other bibliometric indicators. +""" + +import logging +from datetime import datetime, timedelta +from typing import Dict, List, Optional, Tuple + +import requests +from sqlalchemy import Column, DateTime, Float, ForeignKey, Integer, String, Text +from sqlalchemy.orm import relationship + +from uraas.database import Author, Base, Item, SessionLocal + +logger = logging.getLogger(__name__) + + +# ── New Database Models for Citations ──────────────────────────────────────── + + +class Citation(Base): + """Citation relationship between papers.""" + + __tablename__ = "citations" + + id = Column(Integer, primary_key=True) + citing_item_id = Column(Integer, ForeignKey("items.id", ondelete="CASCADE")) + cited_item_id = Column(Integer, ForeignKey("items.id", ondelete="CASCADE")) + citation_date = Column(DateTime) + source = Column(String(50)) # 'openalex', 'crossref', 'manual' + + citing_item = relationship("Item", foreign_keys=[citing_item_id]) + cited_item = relationship("Item", foreign_keys=[cited_item_id]) + + +class CitationMetrics(Base): + """Cached citation metrics for papers.""" + + __tablename__ = "citation_metrics" + + id = Column(Integer, primary_key=True) + item_id = Column(Integer, ForeignKey("items.id", ondelete="CASCADE"), unique=True) + citation_count = Column(Integer, default=0) + h_index = Column(Integer, default=0) + i10_index = Column(Integer, default=0) # papers with 10+ citations + last_updated = Column(DateTime, default=datetime.utcnow) + + item = relationship("Item") + + +class AuthorMetrics(Base): + """Cached bibliometric indicators for authors.""" + + __tablename__ = "author_metrics" + + id = Column(Integer, primary_key=True) + author_id = Column( + Integer, ForeignKey("authors.id", ondelete="CASCADE"), unique=True + ) + total_citations = Column(Integer, default=0) + h_index = Column(Integer, default=0) + i10_index = Column(Integer, default=0) + total_papers = Column(Integer, default=0) + last_updated = Column(DateTime, default=datetime.utcnow) + + author = relationship("Author") + + +# ── Citation Fetching Service ───────────────────────────────────────────────── + + +class CitationTracker: + """Fetches and tracks citations from external APIs.""" + + OPENALEX_API = "https://api.openalex.org/works" + CROSSREF_API = "https://api.crossref.org/works" + + @staticmethod + def fetch_citations_openalex(doi: str) -> Optional[Dict]: + """ + Fetch citation data from OpenAlex. + + Returns: + { + 'citation_count': int, + 'cited_by_api_url': str, + 'citations': [{'doi': str, 'title': str, 'year': int}, ...] + } + """ + try: + url = f"{CitationTracker.OPENALEX_API}/doi:{doi}" + headers = {"User-Agent": "URAAS/1.0 (mailto:library@unilag.edu.ng)"} + response = requests.get(url, headers=headers, timeout=10) + + if response.status_code != 200: + return None + + data = response.json() + citation_count = data.get("cited_by_count", 0) + cited_by_url = data.get("cited_by_api_url") + + # Fetch citing papers + citations = [] + if cited_by_url and citation_count > 0: + cite_response = requests.get(cited_by_url, headers=headers, timeout=10) + if cite_response.status_code == 200: + cite_data = cite_response.json() + for result in cite_data.get("results", [])[:100]: # Limit to 100 + citations.append( + { + "doi": result.get("doi", "").replace( + "https://doi.org/", "" + ), + "title": result.get("title", ""), + "year": result.get("publication_year"), + "authors": [ + a.get("author", {}).get("display_name") + for a in result.get("authorships", [])[:3] + ], + } + ) + + return { + "citation_count": citation_count, + "cited_by_api_url": cited_by_url, + "citations": citations, + } + + except Exception as e: + logger.error(f"OpenAlex citation fetch failed for {doi}: {e}") + return None + + @staticmethod + def fetch_citations_crossref(doi: str) -> Optional[int]: + """Fetch citation count from Crossref (simpler, just count).""" + try: + url = f"{CitationTracker.CROSSREF_API}/{doi}" + headers = {"User-Agent": "URAAS/1.0 (mailto:library@unilag.edu.ng)"} + response = requests.get(url, headers=headers, timeout=10) + + if response.status_code != 200: + return None + + data = response.json() + return data.get("message", {}).get("is-referenced-by-count", 0) + + except Exception as e: + logger.error(f"Crossref citation fetch failed for {doi}: {e}") + return None + + @staticmethod + def update_paper_citations(item_id: int) -> bool: + """Update citation metrics for a single paper.""" + session = SessionLocal() + try: + item = session.query(Item).filter_by(id=item_id).first() + if not item or not item.doi: + return False + + # Try OpenAlex first (more detailed) + oa_data = CitationTracker.fetch_citations_openalex(item.doi) + + if oa_data: + citation_count = oa_data["citation_count"] + + # Update or create metrics + metrics = ( + session.query(CitationMetrics).filter_by(item_id=item_id).first() + ) + if not metrics: + metrics = CitationMetrics(item_id=item_id) + session.add(metrics) + + metrics.citation_count = citation_count + metrics.last_updated = datetime.utcnow() + + # Store citation relationships + for cite in oa_data["citations"]: + if cite["doi"]: + # Check if citing paper exists in our DB + citing_item = ( + session.query(Item).filter_by(doi=cite["doi"]).first() + ) + if citing_item: + # Create citation link + existing = ( + session.query(Citation) + .filter_by( + citing_item_id=citing_item.id, cited_item_id=item_id + ) + .first() + ) + + if not existing: + citation = Citation( + citing_item_id=citing_item.id, + cited_item_id=item_id, + citation_date=( + datetime(cite["year"], 1, 1) + if cite["year"] + else None + ), + source="openalex", + ) + session.add(citation) + + session.commit() + logger.info( + f"Updated citations for item {item_id}: {citation_count} citations" + ) + return True + + # Fallback to Crossref + cr_count = CitationTracker.fetch_citations_crossref(item.doi) + if cr_count is not None: + metrics = ( + session.query(CitationMetrics).filter_by(item_id=item_id).first() + ) + if not metrics: + metrics = CitationMetrics(item_id=item_id) + session.add(metrics) + + metrics.citation_count = cr_count + metrics.last_updated = datetime.utcnow() + session.commit() + return True + + return False + + except Exception as e: + session.rollback() + logger.error(f"Failed to update citations for item {item_id}: {e}") + return False + finally: + session.close() + + @staticmethod + def calculate_h_index(citation_counts: List[int]) -> int: + """ + Calculate h-index from list of citation counts. + h-index = largest number h such that h papers have at least h citations each. + + Example: [100, 50, 30, 20, 15, 10, 8, 5, 3, 2, 1, 1, 0, 0] + - Paper 1: 100 citations ≥ 1 ✓ + - Paper 2: 50 citations ≥ 2 ✓ + - ... + - Paper 10: 2 citations ≥ 10 ✗ + Result: h-index = 9 + """ + if not citation_counts: + return 0 + + sorted_counts = sorted(citation_counts, reverse=True) + h = 0 + for i, count in enumerate(sorted_counts, start=1): + if count >= i: + h = i + else: + break + return h + + @staticmethod + def update_author_metrics(author_id: int) -> bool: + """Calculate and update bibliometric indicators for an author.""" + session = SessionLocal() + try: + author = session.query(Author).filter_by(id=author_id).first() + if not author: + return False + + # Get all papers by this author with citation metrics + papers = ( + session.query(Item, CitationMetrics) + .join(Item.authors) + .outerjoin(CitationMetrics, CitationMetrics.item_id == Item.id) + .filter(Author.id == author_id) + .all() + ) + + if not papers: + return False + + citation_counts = [m.citation_count if m else 0 for _, m in papers] + total_citations = sum(citation_counts) + h_index = CitationTracker.calculate_h_index(citation_counts) + i10_index = sum(1 for c in citation_counts if c >= 10) + + # Update or create author metrics + metrics = ( + session.query(AuthorMetrics).filter_by(author_id=author_id).first() + ) + if not metrics: + metrics = AuthorMetrics(author_id=author_id) + session.add(metrics) + + metrics.total_citations = total_citations + metrics.h_index = h_index + metrics.i10_index = i10_index + metrics.total_papers = len(papers) + metrics.last_updated = datetime.utcnow() + + session.commit() + logger.info( + f"Updated metrics for author {author.name}: h-index={h_index}, citations={total_citations}" + ) + return True + + except Exception as e: + session.rollback() + logger.error(f"Failed to update author metrics for {author_id}: {e}") + return False + finally: + session.close() + + @staticmethod + def bulk_update_citations(limit: int = 100, force: bool = False) -> Dict: + """ + Update citations for papers that haven't been updated recently. + + Args: + limit: Maximum number of papers to update + force: Update all papers regardless of last update time + + Returns: + {'updated': int, 'failed': int, 'skipped': int} + """ + session = SessionLocal() + stats = {"updated": 0, "failed": 0, "skipped": 0} + + try: + # Find papers with DOIs that need updating + cutoff_date = datetime.utcnow() - timedelta(days=7) # Update weekly + + query = session.query(Item).filter(Item.doi.isnot(None)) + + if not force: + # Only update papers not updated in last 7 days + query = query.outerjoin(CitationMetrics).filter( + (CitationMetrics.last_updated.is_(None)) + | (CitationMetrics.last_updated < cutoff_date) + ) + + papers = query.limit(limit).all() + + for paper in papers: + success = CitationTracker.update_paper_citations(paper.id) + if success: + stats["updated"] += 1 + else: + stats["failed"] += 1 + + logger.info(f"Bulk citation update: {stats}") + return stats + + finally: + session.close() + + +# ── API Helper Functions ────────────────────────────────────────────────────── + + +def get_paper_citations(item_id: int) -> Dict: + """Get citation data for a paper.""" + session = SessionLocal() + try: + metrics = session.query(CitationMetrics).filter_by(item_id=item_id).first() + + # Get citing papers + citations = ( + session.query(Citation, Item) + .join(Item, Citation.citing_item_id == Item.id) + .filter(Citation.cited_item_id == item_id) + .all() + ) + + return { + "citation_count": metrics.citation_count if metrics else 0, + "last_updated": metrics.last_updated.isoformat() if metrics else None, + "citing_papers": [ + { + "id": item.id, + "title": item.title, + "doi": item.doi, + "year": ( + item.publication_date.year if item.publication_date else None + ), + "authors": [a.name for a in item.authors[:3]], + } + for _, item in citations + ], + } + finally: + session.close() + + +def get_author_bibliometrics(author_id: int) -> Dict: + """Get bibliometric indicators for an author.""" + session = SessionLocal() + try: + metrics = session.query(AuthorMetrics).filter_by(author_id=author_id).first() + author = session.query(Author).filter_by(id=author_id).first() + + if not metrics or not author: + return {} + + return { + "author_name": author.name, + "total_papers": metrics.total_papers, + "total_citations": metrics.total_citations, + "h_index": metrics.h_index, + "i10_index": metrics.i10_index, + "last_updated": metrics.last_updated.isoformat(), + "citations_per_paper": ( + round(metrics.total_citations / metrics.total_papers, 1) + if metrics.total_papers + else 0 + ), + } + finally: + session.close() diff --git a/uraas/services/comparator_engine.py b/uraas/services/comparator_engine.py index 80cb3a18ecf6ddaedf8e792f7fad53151e14a21a..0744eb48d8fc8eabbf6d44cefaef3369a08f88af 100644 --- a/uraas/services/comparator_engine.py +++ b/uraas/services/comparator_engine.py @@ -1,360 +1,360 @@ -""" -Multi-Institution Comparator Engine -Core feature of APA Intelligence Platform - allows comparing multiple African universities -""" - -from datetime import datetime, timedelta -from typing import Dict, List, Optional - -from sqlalchemy import distinct, func -from sqlalchemy.orm import aliased - -from uraas.database import ( - Author, - Collection, - Community, - File, - Item, - SessionLocal, - item_authors, -) - - -class InstitutionProfile: - """Profile data for a single institution""" - - def __init__(self, ror_id: str, name: str): - self.ror_id = ror_id - self.name = name - self.metrics = {} - - def calculate_metrics(self, session): - """Calculate all metrics for this institution""" - - # Basic counts - items = session.query(Item).filter(Item.ror == self.ror_id).all() - self.metrics["total_papers"] = len(items) - self.metrics["total_authors"] = ( - session.query(func.count(distinct(Author.id))) - .select_from(Author) - .join(Author.items) - .filter(Item.ror == self.ror_id) - .scalar() - ) or 0 - - # Open Access - oa_count = sum(1 for i in items if "openAccess" in (i.dc_rights or "")) - self.metrics["open_access_count"] = oa_count - self.metrics["oa_rate"] = round(oa_count / len(items) * 100, 1) if items else 0 - - # Indigenous Knowledge - tk_count = sum( - 1 for i in items if i.tk_label or i.content_type == "indigenous_knowledge" - ) - self.metrics["tk_papers"] = tk_count - self.metrics["tk_rate"] = round(tk_count / len(items) * 100, 1) if items else 0 - - # Patents - patent_count = sum(1 for i in items if i.patent_id) - self.metrics["patents"] = patent_count - self.metrics["patent_rate"] = ( - round(patent_count / len(items) * 100, 1) if items else 0 - ) - - # African Languages - african_lang_count = sum(1 for i in items if i.is_african_language) - self.metrics["african_language_papers"] = african_lang_count - self.metrics["african_lang_rate"] = ( - round(african_lang_count / len(items) * 100, 1) if items else 0 - ) - - # Temporal analysis - years = [i.publication_date.year for i in items if i.publication_date] - if years: - self.metrics["year_range"] = [min(years), max(years)] - self.metrics["years_active"] = max(years) - min(years) + 1 - - # Growth rate (last 3 years vs previous 3 years) - current_year = datetime.now().year - recent = sum(1 for y in years if y >= current_year - 3) - previous = sum(1 for y in years if current_year - 6 <= y < current_year - 3) - self.metrics["growth_rate"] = ( - round((recent - previous) / previous * 100, 1) if previous else 0 - ) - else: - self.metrics["year_range"] = [] - self.metrics["years_active"] = 0 - self.metrics["growth_rate"] = 0 - - # Efficiency ratios - self.metrics["papers_per_author"] = ( - round(len(items) / self.metrics["total_authors"], 2) - if self.metrics["total_authors"] - else 0 - ) - self.metrics["patents_per_100_papers"] = ( - round(patent_count / len(items) * 100, 1) if items else 0 - ) - - # DocID coverage - docid_count = sum(1 for i in items if i.docid) - self.metrics["docid_coverage"] = ( - round(docid_count / len(items) * 100, 1) if items else 0 - ) - - # Sub-region lookup from registry - from uraas.config.institutions import get_registry - - reg = get_registry() - inst_cfg = reg.get(self.ror_id) - self.metrics["sub_region"] = inst_cfg.sub_region if inst_cfg else "Unknown" - - return self.metrics - - -class ComparatorEngine: - """ - Multi-Institution Comparison Engine - Allows comparing 2-15 institutions simultaneously across all African universities - """ - - @staticmethod - def compare_institutions(ror_ids: List[str]) -> Dict: - """ - Compare multiple institutions across all metrics - - Args: - ror_ids: List of ROR identifiers for institutions - - Returns: - Comprehensive comparison data structure - """ - session = SessionLocal() - try: - profiles = [] - - for ror_id in ror_ids: - # Get institution name from first paper or use ROR - item = session.query(Item).filter(Item.ror == ror_id).first() - name = item.institution if item else ror_id - - profile = InstitutionProfile(ror_id, name) - profile.calculate_metrics(session) - profiles.append(profile) - - # Build comparison matrix - comparison = { - "institutions": [ - {"ror_id": p.ror_id, "name": p.name, "metrics": p.metrics} - for p in profiles - ], - "rankings": ComparatorEngine._calculate_rankings(profiles), - "insights": ComparatorEngine._generate_insights(profiles), - } - - return comparison - - finally: - session.close() - - @staticmethod - def _calculate_rankings(profiles: List[InstitutionProfile]) -> Dict: - """Calculate rankings across key metrics""" - - rankings = {} - - metrics_to_rank = [ - "total_papers", - "oa_rate", - "tk_rate", - "patent_rate", - "african_lang_rate", - "growth_rate", - "papers_per_author", - "patents_per_100_papers", - ] - - for metric in metrics_to_rank: - sorted_profiles = sorted( - profiles, key=lambda p: p.metrics.get(metric, 0), reverse=True - ) - rankings[metric] = [ - { - "rank": i + 1, - "institution": p.name, - "value": p.metrics.get(metric, 0), - } - for i, p in enumerate(sorted_profiles) - ] - - return rankings - - @staticmethod - def _generate_insights(profiles: List[InstitutionProfile]) -> List[Dict]: - """Generate strategic insights from comparison""" - - insights = [] - - # Find leader in each category - categories = { - "total_papers": "Research Volume Leader", - "oa_rate": "Open Access Champion", - "tk_rate": "Indigenous Knowledge Preservation Leader", - "patent_rate": "Innovation Commercialization Leader", - "african_lang_rate": "Linguistic Diversity Champion", - "growth_rate": "Fastest Growing Institution", - } - - for metric, title in categories.items(): - leader = max(profiles, key=lambda p: p.metrics.get(metric, 0)) - if leader.metrics.get(metric, 0) > 0: - insights.append( - { - "category": title, - "institution": leader.name, - "value": leader.metrics.get(metric, 0), - "metric": metric, - } - ) - - # Identify gaps - for profile in profiles: - if profile.metrics.get("tk_rate", 0) < 5: - insights.append( - { - "category": "Opportunity", - "institution": profile.name, - "message": "Low indigenous knowledge digitization - opportunity for cultural preservation initiatives", - "metric": "tk_rate", - } - ) - - if profile.metrics.get("patent_rate", 0) < 2: - insights.append( - { - "category": "Opportunity", - "institution": profile.name, - "message": "Low patent-to-paper ratio - opportunity to strengthen innovation commercialization", - "metric": "patent_rate", - } - ) - - return insights - - @staticmethod - def get_collaboration_matrix(ror_ids: List[str]) -> Dict: - """ - Calculate collaboration patterns between institutions - Returns data for Collaboration Mesh visualization - """ - session = SessionLocal() - try: - # Find papers with authors from multiple institutions - collaborations = {} - - for ror1 in ror_ids: - for ror2 in ror_ids: - if ror1 >= ror2: # Avoid duplicates - continue - - # Count co-authored papers: distinct papers at ror1 that share at least one - # author with a paper at ror2. Self-join Item via the item_authors table. - i1 = aliased(Item, name="i1") - i2 = aliased(Item, name="i2") - ia1 = item_authors.alias("ia1") - ia2 = item_authors.alias("ia2") - count = ( - session.query(func.count(distinct(i1.id))) - .select_from(i1) - .join(ia1, ia1.c.item_id == i1.id) - .join(ia2, ia2.c.author_id == ia1.c.author_id) - .join(i2, i2.id == ia2.c.item_id) - .filter(i1.ror == ror1, i2.ror == ror2) - .scalar() - ) or 0 - - if count > 0: - key = f"{ror1}_{ror2}" - collaborations[key] = { - "source": ror1, - "target": ror2, - "weight": count, - } - - return { - "nodes": [{"id": ror, "label": ror} for ror in ror_ids], - "edges": list(collaborations.values()), - } - - finally: - session.close() - - @staticmethod - def generate_senate_report(ror_ids: List[str], format: str = "json") -> Dict: - """ - Generate comprehensive report for university senate - - Args: - ror_ids: Institutions to include - format: 'json', 'csv', or 'pdf' - - Returns: - Structured report data - """ - comparison = ComparatorEngine.compare_institutions(ror_ids) - collaboration = ComparatorEngine.get_collaboration_matrix(ror_ids) - - report = { - "title": "APA Intelligence Platform - Institutional Comparison Report", - "generated_at": datetime.utcnow().isoformat(), - "institutions_analyzed": len(ror_ids), - "executive_summary": { - "total_papers": sum( - i["metrics"]["total_papers"] for i in comparison["institutions"] - ), - "total_authors": sum( - i["metrics"]["total_authors"] for i in comparison["institutions"] - ), - "average_oa_rate": round( - sum(i["metrics"]["oa_rate"] for i in comparison["institutions"]) - / len(ror_ids), - 1, - ), - "total_collaborations": len(collaboration["edges"]), - }, - "detailed_comparison": comparison, - "collaboration_network": collaboration, - "recommendations": ComparatorEngine._generate_recommendations(comparison), - } - - return report - - @staticmethod - def _generate_recommendations(comparison: Dict) -> List[str]: - """Generate strategic recommendations based on comparison""" - - recommendations = [] - - # Analyze patterns - institutions = comparison["institutions"] - avg_oa = sum(i["metrics"]["oa_rate"] for i in institutions) / len(institutions) - avg_tk = sum(i["metrics"]["tk_rate"] for i in institutions) / len(institutions) - - if avg_oa < 50: - recommendations.append( - "Regional open access rates below 50 percent - recommend coordinated OA policy development" - ) - - if avg_tk < 10: - recommendations.append( - "Low indigenous knowledge digitization across region - opportunity for APA-led cultural preservation initiative" - ) - - # Find best practices - tk_leader = max(institutions, key=lambda i: i["metrics"]["tk_rate"]) - if tk_leader["metrics"]["tk_rate"] > 20: - recommendations.append( - f"{tk_leader['name']} demonstrates strong indigenous knowledge preservation ({tk_leader['metrics']['tk_rate']}%) - recommend knowledge sharing workshop" - ) - - return recommendations +""" +Multi-Institution Comparator Engine +Core feature of APA Intelligence Platform - allows comparing multiple African universities +""" + +from datetime import datetime, timedelta +from typing import Dict, List, Optional + +from sqlalchemy import distinct, func +from sqlalchemy.orm import aliased + +from uraas.database import ( + Author, + Collection, + Community, + File, + Item, + SessionLocal, + item_authors, +) + + +class InstitutionProfile: + """Profile data for a single institution""" + + def __init__(self, ror_id: str, name: str): + self.ror_id = ror_id + self.name = name + self.metrics = {} + + def calculate_metrics(self, session): + """Calculate all metrics for this institution""" + + # Basic counts + items = session.query(Item).filter(Item.ror == self.ror_id).all() + self.metrics["total_papers"] = len(items) + self.metrics["total_authors"] = ( + session.query(func.count(distinct(Author.id))) + .select_from(Author) + .join(Author.items) + .filter(Item.ror == self.ror_id) + .scalar() + ) or 0 + + # Open Access + oa_count = sum(1 for i in items if "openAccess" in (i.dc_rights or "")) + self.metrics["open_access_count"] = oa_count + self.metrics["oa_rate"] = round(oa_count / len(items) * 100, 1) if items else 0 + + # Indigenous Knowledge + tk_count = sum( + 1 for i in items if i.tk_label or i.content_type == "indigenous_knowledge" + ) + self.metrics["tk_papers"] = tk_count + self.metrics["tk_rate"] = round(tk_count / len(items) * 100, 1) if items else 0 + + # Patents + patent_count = sum(1 for i in items if i.patent_id) + self.metrics["patents"] = patent_count + self.metrics["patent_rate"] = ( + round(patent_count / len(items) * 100, 1) if items else 0 + ) + + # African Languages + african_lang_count = sum(1 for i in items if i.is_african_language) + self.metrics["african_language_papers"] = african_lang_count + self.metrics["african_lang_rate"] = ( + round(african_lang_count / len(items) * 100, 1) if items else 0 + ) + + # Temporal analysis + years = [i.publication_date.year for i in items if i.publication_date] + if years: + self.metrics["year_range"] = [min(years), max(years)] + self.metrics["years_active"] = max(years) - min(years) + 1 + + # Growth rate (last 3 years vs previous 3 years) + current_year = datetime.now().year + recent = sum(1 for y in years if y >= current_year - 3) + previous = sum(1 for y in years if current_year - 6 <= y < current_year - 3) + self.metrics["growth_rate"] = ( + round((recent - previous) / previous * 100, 1) if previous else 0 + ) + else: + self.metrics["year_range"] = [] + self.metrics["years_active"] = 0 + self.metrics["growth_rate"] = 0 + + # Efficiency ratios + self.metrics["papers_per_author"] = ( + round(len(items) / self.metrics["total_authors"], 2) + if self.metrics["total_authors"] + else 0 + ) + self.metrics["patents_per_100_papers"] = ( + round(patent_count / len(items) * 100, 1) if items else 0 + ) + + # DocID coverage + docid_count = sum(1 for i in items if i.docid) + self.metrics["docid_coverage"] = ( + round(docid_count / len(items) * 100, 1) if items else 0 + ) + + # Sub-region lookup from registry + from uraas.config.institutions import get_registry + + reg = get_registry() + inst_cfg = reg.get(self.ror_id) + self.metrics["sub_region"] = inst_cfg.sub_region if inst_cfg else "Unknown" + + return self.metrics + + +class ComparatorEngine: + """ + Multi-Institution Comparison Engine + Allows comparing 2-15 institutions simultaneously across all African universities + """ + + @staticmethod + def compare_institutions(ror_ids: List[str]) -> Dict: + """ + Compare multiple institutions across all metrics + + Args: + ror_ids: List of ROR identifiers for institutions + + Returns: + Comprehensive comparison data structure + """ + session = SessionLocal() + try: + profiles = [] + + for ror_id in ror_ids: + # Get institution name from first paper or use ROR + item = session.query(Item).filter(Item.ror == ror_id).first() + name = item.institution if item else ror_id + + profile = InstitutionProfile(ror_id, name) + profile.calculate_metrics(session) + profiles.append(profile) + + # Build comparison matrix + comparison = { + "institutions": [ + {"ror_id": p.ror_id, "name": p.name, "metrics": p.metrics} + for p in profiles + ], + "rankings": ComparatorEngine._calculate_rankings(profiles), + "insights": ComparatorEngine._generate_insights(profiles), + } + + return comparison + + finally: + session.close() + + @staticmethod + def _calculate_rankings(profiles: List[InstitutionProfile]) -> Dict: + """Calculate rankings across key metrics""" + + rankings = {} + + metrics_to_rank = [ + "total_papers", + "oa_rate", + "tk_rate", + "patent_rate", + "african_lang_rate", + "growth_rate", + "papers_per_author", + "patents_per_100_papers", + ] + + for metric in metrics_to_rank: + sorted_profiles = sorted( + profiles, key=lambda p: p.metrics.get(metric, 0), reverse=True + ) + rankings[metric] = [ + { + "rank": i + 1, + "institution": p.name, + "value": p.metrics.get(metric, 0), + } + for i, p in enumerate(sorted_profiles) + ] + + return rankings + + @staticmethod + def _generate_insights(profiles: List[InstitutionProfile]) -> List[Dict]: + """Generate strategic insights from comparison""" + + insights = [] + + # Find leader in each category + categories = { + "total_papers": "Research Volume Leader", + "oa_rate": "Open Access Champion", + "tk_rate": "Indigenous Knowledge Preservation Leader", + "patent_rate": "Innovation Commercialization Leader", + "african_lang_rate": "Linguistic Diversity Champion", + "growth_rate": "Fastest Growing Institution", + } + + for metric, title in categories.items(): + leader = max(profiles, key=lambda p: p.metrics.get(metric, 0)) + if leader.metrics.get(metric, 0) > 0: + insights.append( + { + "category": title, + "institution": leader.name, + "value": leader.metrics.get(metric, 0), + "metric": metric, + } + ) + + # Identify gaps + for profile in profiles: + if profile.metrics.get("tk_rate", 0) < 5: + insights.append( + { + "category": "Opportunity", + "institution": profile.name, + "message": "Low indigenous knowledge digitization - opportunity for cultural preservation initiatives", + "metric": "tk_rate", + } + ) + + if profile.metrics.get("patent_rate", 0) < 2: + insights.append( + { + "category": "Opportunity", + "institution": profile.name, + "message": "Low patent-to-paper ratio - opportunity to strengthen innovation commercialization", + "metric": "patent_rate", + } + ) + + return insights + + @staticmethod + def get_collaboration_matrix(ror_ids: List[str]) -> Dict: + """ + Calculate collaboration patterns between institutions + Returns data for Collaboration Mesh visualization + """ + session = SessionLocal() + try: + # Find papers with authors from multiple institutions + collaborations = {} + + for ror1 in ror_ids: + for ror2 in ror_ids: + if ror1 >= ror2: # Avoid duplicates + continue + + # Count co-authored papers: distinct papers at ror1 that share at least one + # author with a paper at ror2. Self-join Item via the item_authors table. + i1 = aliased(Item, name="i1") + i2 = aliased(Item, name="i2") + ia1 = item_authors.alias("ia1") + ia2 = item_authors.alias("ia2") + count = ( + session.query(func.count(distinct(i1.id))) + .select_from(i1) + .join(ia1, ia1.c.item_id == i1.id) + .join(ia2, ia2.c.author_id == ia1.c.author_id) + .join(i2, i2.id == ia2.c.item_id) + .filter(i1.ror == ror1, i2.ror == ror2) + .scalar() + ) or 0 + + if count > 0: + key = f"{ror1}_{ror2}" + collaborations[key] = { + "source": ror1, + "target": ror2, + "weight": count, + } + + return { + "nodes": [{"id": ror, "label": ror} for ror in ror_ids], + "edges": list(collaborations.values()), + } + + finally: + session.close() + + @staticmethod + def generate_senate_report(ror_ids: List[str], format: str = "json") -> Dict: + """ + Generate comprehensive report for university senate + + Args: + ror_ids: Institutions to include + format: 'json', 'csv', or 'pdf' + + Returns: + Structured report data + """ + comparison = ComparatorEngine.compare_institutions(ror_ids) + collaboration = ComparatorEngine.get_collaboration_matrix(ror_ids) + + report = { + "title": "APA Intelligence Platform - Institutional Comparison Report", + "generated_at": datetime.utcnow().isoformat(), + "institutions_analyzed": len(ror_ids), + "executive_summary": { + "total_papers": sum( + i["metrics"]["total_papers"] for i in comparison["institutions"] + ), + "total_authors": sum( + i["metrics"]["total_authors"] for i in comparison["institutions"] + ), + "average_oa_rate": round( + sum(i["metrics"]["oa_rate"] for i in comparison["institutions"]) + / len(ror_ids), + 1, + ), + "total_collaborations": len(collaboration["edges"]), + }, + "detailed_comparison": comparison, + "collaboration_network": collaboration, + "recommendations": ComparatorEngine._generate_recommendations(comparison), + } + + return report + + @staticmethod + def _generate_recommendations(comparison: Dict) -> List[str]: + """Generate strategic recommendations based on comparison""" + + recommendations = [] + + # Analyze patterns + institutions = comparison["institutions"] + avg_oa = sum(i["metrics"]["oa_rate"] for i in institutions) / len(institutions) + avg_tk = sum(i["metrics"]["tk_rate"] for i in institutions) / len(institutions) + + if avg_oa < 50: + recommendations.append( + "Regional open access rates below 50 percent - recommend coordinated OA policy development" + ) + + if avg_tk < 10: + recommendations.append( + "Low indigenous knowledge digitization across region - opportunity for APA-led cultural preservation initiative" + ) + + # Find best practices + tk_leader = max(institutions, key=lambda i: i["metrics"]["tk_rate"]) + if tk_leader["metrics"]["tk_rate"] > 20: + recommendations.append( + f"{tk_leader['name']} demonstrates strong indigenous knowledge preservation ({tk_leader['metrics']['tk_rate']}%) - recommend knowledge sharing workshop" + ) + + return recommendations diff --git a/uraas/services/email_service.py b/uraas/services/email_service.py index 313479b07d3dad116533855d78ab269468b3f3ad..784d9dd7e0f02e7a4d0d1967c262440589969b25 100644 --- a/uraas/services/email_service.py +++ b/uraas/services/email_service.py @@ -1,215 +1,215 @@ -"""SMTP email service for batch approval notifications. - -Sends a styled HTML email to the approver with one-click Approve / Reject -links. Requires SMTP_HOST, SMTP_USER, SMTP_PASSWORD in the environment. -If SMTP is not configured the call logs a warning and returns False so the -rest of the deposit flow degrades gracefully (the admin can still approve -via the dashboard). -""" - -import logging -import smtplib -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -from email.utils import parseaddr - -from uraas.config import config - -logger = logging.getLogger(__name__) - - -def _is_smtp_configured() -> bool: - return bool(config.SMTP_HOST and config.SMTP_USER and config.SMTP_PASSWORD) - - -def send_batch_approval_request( - *, - to_email: str, - approve_url: str, - reject_url: str, - batch_id: int, - item_count: int, - collection_name: str, - requested_by: str, - expires_hours: int = 48, -) -> bool: - """Send the approval-request email and return True on success.""" - if not _is_smtp_configured(): - logger.warning( - "SMTP not configured — skipping approval email for batch %s. " - "Admin can approve at: %s", - batch_id, - approve_url, - ) - return False - - subject = f"[URAAS] Approve IR deposit — Batch #{batch_id} ({item_count} papers)" - html = _build_html( - batch_id=batch_id, - item_count=item_count, - collection_name=collection_name, - requested_by=requested_by, - approve_url=approve_url, - reject_url=reject_url, - expires_hours=expires_hours, - ) - plain = _build_plain( - batch_id=batch_id, - item_count=item_count, - collection_name=collection_name, - requested_by=requested_by, - approve_url=approve_url, - reject_url=reject_url, - expires_hours=expires_hours, - ) - - msg = MIMEMultipart("alternative") - msg["Subject"] = subject - msg["From"] = config.SMTP_FROM - msg["To"] = to_email - msg.attach(MIMEText(plain, "plain", "utf-8")) - msg.attach(MIMEText(html, "html", "utf-8")) - - try: - if config.SMTP_USE_TLS: - server = smtplib.SMTP(config.SMTP_HOST, config.SMTP_PORT, timeout=30) - server.ehlo() - server.starttls() - server.ehlo() - else: - server = smtplib.SMTP_SSL(config.SMTP_HOST, config.SMTP_PORT, timeout=30) - - server.login(config.SMTP_USER, config.SMTP_PASSWORD) - # envelope MAIL FROM must be bare address, not "Name " - envelope_from = parseaddr(config.SMTP_FROM)[1] or config.SMTP_USER - server.sendmail(envelope_from, [to_email], msg.as_bytes()) - server.quit() - logger.info("Approval email sent for batch %s → %s", batch_id, to_email) - return True - except Exception as exc: - logger.error("Failed to send approval email for batch %s: %s", batch_id, exc) - return False - - -# ── Email templates ─────────────────────────────────────────────────────────── - -def _build_html( - batch_id, item_count, collection_name, requested_by, - approve_url, reject_url, expires_hours, -) -> str: - return f""" - - - - - -
- - - - - - - -
-

University of Lagos

-

URAAS — IR Deposit Approval

-
-

- {requested_by} has queued a batch of research papers - for deposit into the UNILAG Institutional Repository. - Your approval is required before any data is sent to the live IR. -

- - - - - - - - - - - - - - - - - - - - - - -
Batch ID#{batch_id}
Papers to deposit{item_count}
Target collection{collection_name or 'Not specified'}
Requested by{requested_by}
Link expiresin {expires_hours} hours
-

- Clicking Approve will immediately begin depositing all - {item_count} paper(s) to the live DSpace IR at - api-ir.unilag.edu.ng. - This action cannot be undone for items that are already deposited. -

- - - - - - -
- - ✓  Approve Deposit - - - - ✗  Reject Batch - -
-

- If the buttons do not work, copy these URLs into your browser:
- Approve: {approve_url}
- Reject: {reject_url} -

-
- URAAS — APA Intelligence & Analytics Platform  |  University of Lagos
- This is an automated message. Do not reply. -
-
- -""" - - -def _build_plain( - batch_id, item_count, collection_name, requested_by, - approve_url, reject_url, expires_hours, -) -> str: - return f"""URAAS — IR Deposit Approval Request -University of Lagos Institutional Repository -============================================= - -{requested_by} has queued a deposit batch. - - Batch ID : #{batch_id} - Papers : {item_count} - Target collection: {collection_name or 'Not specified'} - Requested by : {requested_by} - Link expires : in {expires_hours} hours - -ACTION REQUIRED ---------------- - -APPROVE (sends papers to the live IR): -{approve_url} - -REJECT (cancels the batch): -{reject_url} - --- -URAAS — APA Intelligence & Analytics Platform -University of Lagos -This is an automated message. Do not reply. -""" +"""SMTP email service for batch approval notifications. + +Sends a styled HTML email to the approver with one-click Approve / Reject +links. Requires SMTP_HOST, SMTP_USER, SMTP_PASSWORD in the environment. +If SMTP is not configured the call logs a warning and returns False so the +rest of the deposit flow degrades gracefully (the admin can still approve +via the dashboard). +""" + +import logging +import smtplib +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.utils import parseaddr + +from uraas.config import config + +logger = logging.getLogger(__name__) + + +def _is_smtp_configured() -> bool: + return bool(config.SMTP_HOST and config.SMTP_USER and config.SMTP_PASSWORD) + + +def send_batch_approval_request( + *, + to_email: str, + approve_url: str, + reject_url: str, + batch_id: int, + item_count: int, + collection_name: str, + requested_by: str, + expires_hours: int = 48, +) -> bool: + """Send the approval-request email and return True on success.""" + if not _is_smtp_configured(): + logger.warning( + "SMTP not configured — skipping approval email for batch %s. " + "Admin can approve at: %s", + batch_id, + approve_url, + ) + return False + + subject = f"[URAAS] Approve IR deposit — Batch #{batch_id} ({item_count} papers)" + html = _build_html( + batch_id=batch_id, + item_count=item_count, + collection_name=collection_name, + requested_by=requested_by, + approve_url=approve_url, + reject_url=reject_url, + expires_hours=expires_hours, + ) + plain = _build_plain( + batch_id=batch_id, + item_count=item_count, + collection_name=collection_name, + requested_by=requested_by, + approve_url=approve_url, + reject_url=reject_url, + expires_hours=expires_hours, + ) + + msg = MIMEMultipart("alternative") + msg["Subject"] = subject + msg["From"] = config.SMTP_FROM + msg["To"] = to_email + msg.attach(MIMEText(plain, "plain", "utf-8")) + msg.attach(MIMEText(html, "html", "utf-8")) + + try: + if config.SMTP_USE_TLS: + server = smtplib.SMTP(config.SMTP_HOST, config.SMTP_PORT, timeout=30) + server.ehlo() + server.starttls() + server.ehlo() + else: + server = smtplib.SMTP_SSL(config.SMTP_HOST, config.SMTP_PORT, timeout=30) + + server.login(config.SMTP_USER, config.SMTP_PASSWORD) + # envelope MAIL FROM must be bare address, not "Name " + envelope_from = parseaddr(config.SMTP_FROM)[1] or config.SMTP_USER + server.sendmail(envelope_from, [to_email], msg.as_bytes()) + server.quit() + logger.info("Approval email sent for batch %s → %s", batch_id, to_email) + return True + except Exception as exc: + logger.error("Failed to send approval email for batch %s: %s", batch_id, exc) + return False + + +# ── Email templates ─────────────────────────────────────────────────────────── + +def _build_html( + batch_id, item_count, collection_name, requested_by, + approve_url, reject_url, expires_hours, +) -> str: + return f""" + + + + + +
+ + + + + + + +
+

University of Lagos

+

URAAS — IR Deposit Approval

+
+

+ {requested_by} has queued a batch of research papers + for deposit into the UNILAG Institutional Repository. + Your approval is required before any data is sent to the live IR. +

+ + + + + + + + + + + + + + + + + + + + + + +
Batch ID#{batch_id}
Papers to deposit{item_count}
Target collection{collection_name or 'Not specified'}
Requested by{requested_by}
Link expiresin {expires_hours} hours
+

+ Clicking Approve will immediately begin depositing all + {item_count} paper(s) to the live DSpace IR at + api-ir.unilag.edu.ng. + This action cannot be undone for items that are already deposited. +

+ + + + + + +
+ + ✓  Approve Deposit + + + + ✗  Reject Batch + +
+

+ If the buttons do not work, copy these URLs into your browser:
+ Approve: {approve_url}
+ Reject: {reject_url} +

+
+ URAAS — APA Intelligence & Analytics Platform  |  University of Lagos
+ This is an automated message. Do not reply. +
+
+ +""" + + +def _build_plain( + batch_id, item_count, collection_name, requested_by, + approve_url, reject_url, expires_hours, +) -> str: + return f"""URAAS — IR Deposit Approval Request +University of Lagos Institutional Repository +============================================= + +{requested_by} has queued a deposit batch. + + Batch ID : #{batch_id} + Papers : {item_count} + Target collection: {collection_name or 'Not specified'} + Requested by : {requested_by} + Link expires : in {expires_hours} hours + +ACTION REQUIRED +--------------- + +APPROVE (sends papers to the live IR): +{approve_url} + +REJECT (cancels the batch): +{reject_url} + +-- +URAAS — APA Intelligence & Analytics Platform +University of Lagos +This is an automated message. Do not reply. +""" diff --git a/uraas/services/ir_client.py b/uraas/services/ir_client.py index 5a4f8da2d5256f2f245dd767142d6787a878c797..67f0ccac534ec563687520d324aa0f5c5d7bd3aa 100644 --- a/uraas/services/ir_client.py +++ b/uraas/services/ir_client.py @@ -1,494 +1,494 @@ -"""DSpace 7.x REST API client for the live UNILAG IR. - -Spec reference: UNILAG_IR_Build_Spec.docx §4 & §7 -Backend base: https://api-ir.unilag.edu.ng/server - -Auth pattern (§4.1): prime CSRF token → POST login → JWT + refreshed CSRF. -Every write sends both headers; CSRF rotates on each response and must be -tracked. Reads on public objects need no auth. -""" - -import logging -import os - -import requests - -from uraas.config import config - -logger = logging.getLogger(__name__) - -_TIMEOUT = 15 # seconds for IR requests - - -class IRConnectionError(Exception): - pass - - -class DSpaceClient: - """Thin session-scoped DSpace 7.x REST client. - - Instantiate once per task; do not share across threads. - """ - - def __init__(self): - self.base = config.DSPACE_API_URL.rstrip("/") - self._s = requests.Session() - self._s.headers.update({ - "User-Agent": "URAAS/1.0 (APA Intelligence Platform; uraas-bot@unilag.edu.ng)", - "Accept": "application/json", - }) - self.jwt: str | None = None - self.csrf: str | None = None - - # ── Authentication ──────────────────────────────────────────────────────── - - def _prime_csrf(self) -> str: - """GET /api/authn/status to seed the CSRF cookie/header. - - DSpace 9.1 does not return a CSRF token on GET requests — the token is - only issued on the first failed write (403). This method returns - whatever it finds; the login() method handles the missing-token case - by retrying after the first 403. - """ - r = self._s.get(f"{self.base}/api/authn/status", timeout=_TIMEOUT) - r.raise_for_status() - token = ( - r.headers.get("DSPACE-XSRF-TOKEN") - or self._s.cookies.get("DSPACE-XSRF-TOKEN", "") - or self._s.cookies.get("DSPACE-XSRF-COOKIE", "") - ) - self.csrf = token - return token - - def _refresh_csrf(self, response: requests.Response): - new = ( - response.headers.get("DSPACE-XSRF-TOKEN") - or self._s.cookies.get("DSPACE-XSRF-COOKIE", "") - ) - if new: - self.csrf = new - - def login(self): - """Authenticate and store JWT + CSRF token for subsequent writes. - - DSpace 9.1 CSRF dance: the first POST to /api/authn/login returns 403 - and seeds the CSRF token in the response header + cookie. We catch - that specific 403, extract the token, and retry once. - """ - if not config.DSPACE_USERNAME or not config.DSPACE_PASSWORD: - raise IRConnectionError( - "DSPACE_USERNAME / DSPACE_PASSWORD not configured in .env" - ) - self._prime_csrf() - r = self._s.post( - f"{self.base}/api/authn/login", - headers={"X-XSRF-TOKEN": self.csrf} if self.csrf else {}, - data={"user": config.DSPACE_USERNAME, "password": config.DSPACE_PASSWORD}, - timeout=_TIMEOUT, - ) - self._refresh_csrf(r) - - # DSpace 9.1: first write with no/stale CSRF returns 403 + new token. - if r.status_code == 403 and self.csrf: - r = self._s.post( - f"{self.base}/api/authn/login", - headers={"X-XSRF-TOKEN": self.csrf}, - data={"user": config.DSPACE_USERNAME, "password": config.DSPACE_PASSWORD}, - timeout=_TIMEOUT, - ) - self._refresh_csrf(r) - - if r.status_code == 401: - raise IRConnectionError("DSpace login failed — check DSPACE_USERNAME/PASSWORD") - r.raise_for_status() - auth = r.headers.get("Authorization", "") - if auth.startswith("Bearer "): - self.jwt = auth[7:] - else: - raise IRConnectionError("DSpace login did not return a JWT") - - def _write_headers(self) -> dict: - return { - "Authorization": f"Bearer {self.jwt}", - "X-XSRF-TOKEN": self.csrf, - } - - # ── Read-only probes (no auth required) ─────────────────────────────────── - - def probe(self) -> dict: - """Check connectivity and return DSpace version. Safe to call without creds.""" - try: - r = self._s.get(f"{self.base}/api", timeout=_TIMEOUT) - r.raise_for_status() - data = r.json() - return {"ok": True, "version": data.get("dspaceVersion", "unknown")} - except Exception as exc: - return {"ok": False, "error": str(exc)} - - def get_total_items(self) -> int: - """Total archived items via discovery endpoint (§7.2).""" - try: - r = self._s.get( - f"{self.base}/api/discover/search/objects", - params={"dsoType": "item", "size": 1}, - timeout=_TIMEOUT, - ) - r.raise_for_status() - return ( - r.json() - .get("_embedded", {}) - .get("searchResult", {}) - .get("page", {}) - .get("totalElements", 0) - ) - except Exception: - return 0 - - def get_facet(self, facet: str, size: int = 20) -> list[dict]: - """Return facet buckets from the discovery layer (§7.2). - - facet: one of dateIssued, author, subject, itemtype … - Returns list of {"label": str, "count": int}. - """ - try: - r = self._s.get( - f"{self.base}/api/discover/facets/{facet}", - params={"dsoType": "item", "size": size}, - timeout=_TIMEOUT, - ) - r.raise_for_status() - raw = ( - r.json() - .get("_embedded", {}) - .get("values", []) - ) - return [{"label": v.get("label", ""), "count": v.get("count", 0)} for v in raw] - except Exception as exc: - logger.warning("get_facet %s: %s", facet, exc) - return [] - - def get_collections(self, size: int = 200) -> list[dict]: - """List all DSpace collections for the deposit UI dropdown.""" - try: - r = self._s.get( - f"{self.base}/api/core/collections", - params={"size": size}, - timeout=_TIMEOUT, - ) - r.raise_for_status() - raw = r.json().get("_embedded", {}).get("collections", []) - return [ - {"uuid": c.get("uuid", ""), "name": c.get("name", "Unnamed")} - for c in raw - ] - except Exception as exc: - logger.warning("get_collections: %s", exc) - return [] - - def get_submittable_collections(self) -> list[dict]: - """Return only collections the logged-in user has submission rights to. - - Reads the eperson's group memberships, extracts collection UUIDs from - COLLECTION_{uuid}_SUBMIT group names, then resolves names via the - collections endpoint. Requires prior login(). - """ - if not self.jwt: - self.login() - import re as _re - try: - # 1. Get authn/status to find eperson link - r = self._s.get( - f"{self.base}/api/authn/status", - headers=self._write_headers(), - timeout=_TIMEOUT, - ) - r.raise_for_status() - ep_href = r.json().get("_links", {}).get("eperson", {}).get("href", "") - if not ep_href: - return [] - - # 2. Fetch eperson → groups link - r2 = self._s.get(ep_href, headers=self._write_headers(), timeout=_TIMEOUT) - r2.raise_for_status() - groups_href = r2.json().get("_links", {}).get("groups", {}).get("href", "") - if not groups_href: - return [] - - # 3. Extract collection UUIDs from COLLECTION_{uuid}_SUBMIT group names - r3 = self._s.get(groups_href, headers=self._write_headers(), timeout=_TIMEOUT) - r3.raise_for_status() - groups = r3.json().get("_embedded", {}).get("groups", []) - submit_uuids = set() - for g in groups: - m = _re.match(r"COLLECTION_([0-9a-f\-]{36})_SUBMIT", g.get("name", "")) - if m: - submit_uuids.add(m.group(1)) - - if not submit_uuids: - return [] - - # 4. Fetch all collections and filter to submittable ones - all_cols = self.get_collections(size=300) - return [c for c in all_cols if c["uuid"] in submit_uuids] - except Exception as exc: - logger.warning("get_submittable_collections: %s", exc) - return [] - - def get_live_stats(self) -> dict: - """Composite live stats tile for the dashboard (§7.3).""" - total = self.get_total_items() - by_year = self.get_facet("dateIssued", size=30) - by_type = self.get_facet("itemtype", size=20) - return { - "total_items": total, - "by_year": by_year, - "by_type": by_type, - } - - # ── Deposit (Path B — REST submission flow, §6.2) ───────────────────────── - - def _check_duplicate(self, doi: str | None, title: str, year: str | None) -> bool: - """Return True if an item with this DOI or (title+year) already exists in IR.""" - if doi: - r = self._s.get( - f"{self.base}/api/discover/search/objects", - params={"query": f"dc.identifier.uri:{doi}", "dsoType": "item", "size": 1}, - timeout=_TIMEOUT, - ) - try: - if r.json().get("_embedded", {}).get("searchResult", {}).get("page", {}).get("totalElements", 0) > 0: - return True - except Exception: - pass - # Normalised title check - safe_title = title.replace('"', '\\"')[:100] - r = self._s.get( - f"{self.base}/api/discover/search/objects", - params={"query": f'dc.title:"{safe_title}"', "dsoType": "item", "size": 1}, - timeout=_TIMEOUT, - ) - try: - return ( - r.json() - .get("_embedded", {}) - .get("searchResult", {}) - .get("page", {}) - .get("totalElements", 0) - ) > 0 - except Exception: - return False - - def deposit_item( - self, - collection_uuid: str, - item, # uraas.database.Item ORM object - pdf_path: str | None = None, - ) -> dict: - """ - Deposit one item to the IR and return a result dict. - - Steps (§6.2): - 1. Create workspace item in the target collection (optionally with PDF) - 2. PATCH Dublin Core metadata - 3. POST to workflow → archived - - Returns {"status": "ok"|"duplicate"|"error", "dspace_id": ..., "message": ...} - """ - if not self.jwt: - self.login() - - # Idempotency guard (§8 operational concerns) - if self._check_duplicate(item.doi, item.title or "", item.dc_date_issued): - return {"status": "duplicate", "message": "Already exists in IR"} - - # 1. Create workspace item ────────────────────────────────────────────── - headers = self._write_headers() - params = {"owningCollection": collection_uuid} - - if pdf_path and os.path.exists(pdf_path): - with open(pdf_path, "rb") as fh: - r = self._s.post( - f"{self.base}/api/submission/workspaceitems", - headers=headers, - params=params, - files={"file": (os.path.basename(pdf_path), fh, "application/pdf")}, - timeout=60, - ) - else: - # DSpace 9 requires Content-Type: application/json even for an empty body. - r = self._s.post( - f"{self.base}/api/submission/workspaceitems", - headers={**headers, "Content-Type": "application/json"}, - params=params, - data="{}", - timeout=_TIMEOUT, - ) - - self._refresh_csrf(r) - if r.status_code == 401: - # JWT expired mid-batch — re-auth once and retry - self.login() - r = self._s.post( - f"{self.base}/api/submission/workspaceitems", - headers={**self._write_headers(), "Content-Type": "application/json"}, - params=params, - data="{}", - timeout=_TIMEOUT, - ) - self._refresh_csrf(r) - - r.raise_for_status() - data = r.json() - # File uploads return {_embedded: {workspaceitems: [{id: ...}]}} - # Empty-body creates return a flat {id: ...} object. - ws_id = data.get("id") or ( - data.get("_embedded", {}).get("workspaceitems", [{}])[0].get("id") - ) - if not ws_id: - return {"status": "error", "message": "No workspace item ID returned"} - - # 2. PATCH Dublin Core metadata ──────────────────────────────────────── - patch_ops = _build_metadata_patch(item) - r2 = self._s.patch( - f"{self.base}/api/submission/workspaceitems/{ws_id}", - headers={**self._write_headers(), "Content-Type": "application/json"}, - json=patch_ops, - timeout=_TIMEOUT, - ) - self._refresh_csrf(r2) - if not r2.ok: - logger.warning("metadata patch failed for ws %s: %s %s", ws_id, r2.status_code, r2.text[:200]) - - # 3. Grant the submission license (required by UNILAG DSpace 9 form) ── - license_patch = [{"op": "replace", "path": "/sections/license/granted", "value": True}] - rl = self._s.patch( - f"{self.base}/api/submission/workspaceitems/{ws_id}", - headers={**self._write_headers(), "Content-Type": "application/json"}, - json=license_patch, - timeout=_TIMEOUT, - ) - self._refresh_csrf(rl) - if not rl.ok: - logger.warning("license grant failed for ws %s: %s", ws_id, rl.status_code) - - # 4. Check for blocking validation errors before submitting ─────────── - rv = self._s.get( - f"{self.base}/api/submission/workspaceitems/{ws_id}", - headers=self._write_headers(), - timeout=_TIMEOUT, - ) - self._refresh_csrf(rv) - if rv.ok: - errors = rv.json().get("errors", []) - blocking = [e for e in errors if "filerequired" in e.get("message", "")] - if blocking: - # This collection requires a file; we have no PDF → abort cleanly. - self._s.delete( - f"{self.base}/api/submission/workspaceitems/{ws_id}", - headers=self._write_headers(), - timeout=_TIMEOUT, - ) - return { - "status": "error", - "message": "Collection requires a PDF file; no local file available for this item", - } - - # 5. Submit to workflow → archived ──────────────────────────────────── - r3 = self._s.post( - f"{self.base}/api/workflow/workflowitems", - headers={**self._write_headers(), "Content-Type": "text/uri-list"}, - data=f"{self.base}/api/submission/workspaceitems/{ws_id}", - timeout=_TIMEOUT, - ) - self._refresh_csrf(r3) - r3.raise_for_status() - - dspace_id = r3.json().get("id") or r3.json().get("uuid", "") - return {"status": "ok", "dspace_id": str(dspace_id), "message": "Deposited"} - - -# ── Dublin Core field mapping (§6.4) ───────────────────────────────────────── - -def _mv(value: str, place: int = 0) -> dict: - """Build a DSpace 9 metadata value object (language/authority/confidence required).""" - return { - "value": value, - "language": None, - "authority": None, - "confidence": -1, - "place": place, - } - - -def _build_metadata_patch(item) -> list[dict]: - """Build JSON-Patch ops to set DC fields on a DSpace 9 workspace item. - - DSpace 9 requires full value objects with language/authority/confidence/place. - Multi-value fields are batched into a single op (repeated ops on the same - path would overwrite instead of append). - """ - ops = [] - - def _add(dc_path: str, value: str, section: str = "traditionalpageone"): - if value and value.strip(): - ops.append({ - "op": "add", - "path": f"/sections/{section}/{dc_path}", - "value": [_mv(value.strip())], - }) - - _add("dc.title", item.title or "") - _add("dc.date.issued", item.dc_date_issued or "") - _add("dc.type", item.dc_type or "") - _add("dc.language.iso", item.dc_language or "en") - if item.institution: - _add("dc.publisher", item.institution) - - # URI: prefer explicit dc.identifier.uri, else build from DOI, else fallback to url. - # dc.rights and dc.identifier.doi are not in the UNILAG submission form. - doi = item.doi or "" - uri = ( - item.dc_identifier_uri - or (f"https://doi.org/{doi}" if doi else "") - or item.url - or "" - ) - _add("dc.identifier.uri", uri) - - # Authors — all in one op so every author is preserved - author_names = [ - a.name for a in getattr(item, "authors", []) if getattr(a, "name", "") - ] - if author_names: - ops.append({ - "op": "add", - "path": "/sections/traditionalpageone/dc.contributor.author", - "value": [_mv(n, i) for i, n in enumerate(author_names)], - }) - - # Abstract and subjects go in traditionalpagetwo (DSpace 9 default layout) - if item.abstract and item.abstract.strip(): - ops.append({ - "op": "add", - "path": "/sections/traditionalpagetwo/dc.description.abstract", - "value": [_mv(item.abstract.strip())], - }) - - subject_tags = [t.strip() for t in (item.dc_subject or "").split(",") if t.strip()] - if subject_tags: - ops.append({ - "op": "add", - "path": "/sections/traditionalpagetwo/dc.subject", - "value": [_mv(t, i) for i, t in enumerate(subject_tags)], - }) - - # ARK + DocID as identifiers - other_ids = [v for v in [item.ark, item.docid] if v] - if other_ids: - ops.append({ - "op": "add", - "path": "/sections/traditionalpageone/dc.identifier.other", - "value": [_mv(v, i) for i, v in enumerate(other_ids)], - }) - - return ops +"""DSpace 7.x REST API client for the live UNILAG IR. + +Spec reference: UNILAG_IR_Build_Spec.docx §4 & §7 +Backend base: https://api-ir.unilag.edu.ng/server + +Auth pattern (§4.1): prime CSRF token → POST login → JWT + refreshed CSRF. +Every write sends both headers; CSRF rotates on each response and must be +tracked. Reads on public objects need no auth. +""" + +import logging +import os + +import requests + +from uraas.config import config + +logger = logging.getLogger(__name__) + +_TIMEOUT = 15 # seconds for IR requests + + +class IRConnectionError(Exception): + pass + + +class DSpaceClient: + """Thin session-scoped DSpace 7.x REST client. + + Instantiate once per task; do not share across threads. + """ + + def __init__(self): + self.base = config.DSPACE_API_URL.rstrip("/") + self._s = requests.Session() + self._s.headers.update({ + "User-Agent": "URAAS/1.0 (APA Intelligence Platform; uraas-bot@unilag.edu.ng)", + "Accept": "application/json", + }) + self.jwt: str | None = None + self.csrf: str | None = None + + # ── Authentication ──────────────────────────────────────────────────────── + + def _prime_csrf(self) -> str: + """GET /api/authn/status to seed the CSRF cookie/header. + + DSpace 9.1 does not return a CSRF token on GET requests — the token is + only issued on the first failed write (403). This method returns + whatever it finds; the login() method handles the missing-token case + by retrying after the first 403. + """ + r = self._s.get(f"{self.base}/api/authn/status", timeout=_TIMEOUT) + r.raise_for_status() + token = ( + r.headers.get("DSPACE-XSRF-TOKEN") + or self._s.cookies.get("DSPACE-XSRF-TOKEN", "") + or self._s.cookies.get("DSPACE-XSRF-COOKIE", "") + ) + self.csrf = token + return token + + def _refresh_csrf(self, response: requests.Response): + new = ( + response.headers.get("DSPACE-XSRF-TOKEN") + or self._s.cookies.get("DSPACE-XSRF-COOKIE", "") + ) + if new: + self.csrf = new + + def login(self): + """Authenticate and store JWT + CSRF token for subsequent writes. + + DSpace 9.1 CSRF dance: the first POST to /api/authn/login returns 403 + and seeds the CSRF token in the response header + cookie. We catch + that specific 403, extract the token, and retry once. + """ + if not config.DSPACE_USERNAME or not config.DSPACE_PASSWORD: + raise IRConnectionError( + "DSPACE_USERNAME / DSPACE_PASSWORD not configured in .env" + ) + self._prime_csrf() + r = self._s.post( + f"{self.base}/api/authn/login", + headers={"X-XSRF-TOKEN": self.csrf} if self.csrf else {}, + data={"user": config.DSPACE_USERNAME, "password": config.DSPACE_PASSWORD}, + timeout=_TIMEOUT, + ) + self._refresh_csrf(r) + + # DSpace 9.1: first write with no/stale CSRF returns 403 + new token. + if r.status_code == 403 and self.csrf: + r = self._s.post( + f"{self.base}/api/authn/login", + headers={"X-XSRF-TOKEN": self.csrf}, + data={"user": config.DSPACE_USERNAME, "password": config.DSPACE_PASSWORD}, + timeout=_TIMEOUT, + ) + self._refresh_csrf(r) + + if r.status_code == 401: + raise IRConnectionError("DSpace login failed — check DSPACE_USERNAME/PASSWORD") + r.raise_for_status() + auth = r.headers.get("Authorization", "") + if auth.startswith("Bearer "): + self.jwt = auth[7:] + else: + raise IRConnectionError("DSpace login did not return a JWT") + + def _write_headers(self) -> dict: + return { + "Authorization": f"Bearer {self.jwt}", + "X-XSRF-TOKEN": self.csrf, + } + + # ── Read-only probes (no auth required) ─────────────────────────────────── + + def probe(self) -> dict: + """Check connectivity and return DSpace version. Safe to call without creds.""" + try: + r = self._s.get(f"{self.base}/api", timeout=_TIMEOUT) + r.raise_for_status() + data = r.json() + return {"ok": True, "version": data.get("dspaceVersion", "unknown")} + except Exception as exc: + return {"ok": False, "error": str(exc)} + + def get_total_items(self) -> int: + """Total archived items via discovery endpoint (§7.2).""" + try: + r = self._s.get( + f"{self.base}/api/discover/search/objects", + params={"dsoType": "item", "size": 1}, + timeout=_TIMEOUT, + ) + r.raise_for_status() + return ( + r.json() + .get("_embedded", {}) + .get("searchResult", {}) + .get("page", {}) + .get("totalElements", 0) + ) + except Exception: + return 0 + + def get_facet(self, facet: str, size: int = 20) -> list[dict]: + """Return facet buckets from the discovery layer (§7.2). + + facet: one of dateIssued, author, subject, itemtype … + Returns list of {"label": str, "count": int}. + """ + try: + r = self._s.get( + f"{self.base}/api/discover/facets/{facet}", + params={"dsoType": "item", "size": size}, + timeout=_TIMEOUT, + ) + r.raise_for_status() + raw = ( + r.json() + .get("_embedded", {}) + .get("values", []) + ) + return [{"label": v.get("label", ""), "count": v.get("count", 0)} for v in raw] + except Exception as exc: + logger.warning("get_facet %s: %s", facet, exc) + return [] + + def get_collections(self, size: int = 200) -> list[dict]: + """List all DSpace collections for the deposit UI dropdown.""" + try: + r = self._s.get( + f"{self.base}/api/core/collections", + params={"size": size}, + timeout=_TIMEOUT, + ) + r.raise_for_status() + raw = r.json().get("_embedded", {}).get("collections", []) + return [ + {"uuid": c.get("uuid", ""), "name": c.get("name", "Unnamed")} + for c in raw + ] + except Exception as exc: + logger.warning("get_collections: %s", exc) + return [] + + def get_submittable_collections(self) -> list[dict]: + """Return only collections the logged-in user has submission rights to. + + Reads the eperson's group memberships, extracts collection UUIDs from + COLLECTION_{uuid}_SUBMIT group names, then resolves names via the + collections endpoint. Requires prior login(). + """ + if not self.jwt: + self.login() + import re as _re + try: + # 1. Get authn/status to find eperson link + r = self._s.get( + f"{self.base}/api/authn/status", + headers=self._write_headers(), + timeout=_TIMEOUT, + ) + r.raise_for_status() + ep_href = r.json().get("_links", {}).get("eperson", {}).get("href", "") + if not ep_href: + return [] + + # 2. Fetch eperson → groups link + r2 = self._s.get(ep_href, headers=self._write_headers(), timeout=_TIMEOUT) + r2.raise_for_status() + groups_href = r2.json().get("_links", {}).get("groups", {}).get("href", "") + if not groups_href: + return [] + + # 3. Extract collection UUIDs from COLLECTION_{uuid}_SUBMIT group names + r3 = self._s.get(groups_href, headers=self._write_headers(), timeout=_TIMEOUT) + r3.raise_for_status() + groups = r3.json().get("_embedded", {}).get("groups", []) + submit_uuids = set() + for g in groups: + m = _re.match(r"COLLECTION_([0-9a-f\-]{36})_SUBMIT", g.get("name", "")) + if m: + submit_uuids.add(m.group(1)) + + if not submit_uuids: + return [] + + # 4. Fetch all collections and filter to submittable ones + all_cols = self.get_collections(size=300) + return [c for c in all_cols if c["uuid"] in submit_uuids] + except Exception as exc: + logger.warning("get_submittable_collections: %s", exc) + return [] + + def get_live_stats(self) -> dict: + """Composite live stats tile for the dashboard (§7.3).""" + total = self.get_total_items() + by_year = self.get_facet("dateIssued", size=30) + by_type = self.get_facet("itemtype", size=20) + return { + "total_items": total, + "by_year": by_year, + "by_type": by_type, + } + + # ── Deposit (Path B — REST submission flow, §6.2) ───────────────────────── + + def _check_duplicate(self, doi: str | None, title: str, year: str | None) -> bool: + """Return True if an item with this DOI or (title+year) already exists in IR.""" + if doi: + r = self._s.get( + f"{self.base}/api/discover/search/objects", + params={"query": f"dc.identifier.uri:{doi}", "dsoType": "item", "size": 1}, + timeout=_TIMEOUT, + ) + try: + if r.json().get("_embedded", {}).get("searchResult", {}).get("page", {}).get("totalElements", 0) > 0: + return True + except Exception: + pass + # Normalised title check + safe_title = title.replace('"', '\\"')[:100] + r = self._s.get( + f"{self.base}/api/discover/search/objects", + params={"query": f'dc.title:"{safe_title}"', "dsoType": "item", "size": 1}, + timeout=_TIMEOUT, + ) + try: + return ( + r.json() + .get("_embedded", {}) + .get("searchResult", {}) + .get("page", {}) + .get("totalElements", 0) + ) > 0 + except Exception: + return False + + def deposit_item( + self, + collection_uuid: str, + item, # uraas.database.Item ORM object + pdf_path: str | None = None, + ) -> dict: + """ + Deposit one item to the IR and return a result dict. + + Steps (§6.2): + 1. Create workspace item in the target collection (optionally with PDF) + 2. PATCH Dublin Core metadata + 3. POST to workflow → archived + + Returns {"status": "ok"|"duplicate"|"error", "dspace_id": ..., "message": ...} + """ + if not self.jwt: + self.login() + + # Idempotency guard (§8 operational concerns) + if self._check_duplicate(item.doi, item.title or "", item.dc_date_issued): + return {"status": "duplicate", "message": "Already exists in IR"} + + # 1. Create workspace item ────────────────────────────────────────────── + headers = self._write_headers() + params = {"owningCollection": collection_uuid} + + if pdf_path and os.path.exists(pdf_path): + with open(pdf_path, "rb") as fh: + r = self._s.post( + f"{self.base}/api/submission/workspaceitems", + headers=headers, + params=params, + files={"file": (os.path.basename(pdf_path), fh, "application/pdf")}, + timeout=60, + ) + else: + # DSpace 9 requires Content-Type: application/json even for an empty body. + r = self._s.post( + f"{self.base}/api/submission/workspaceitems", + headers={**headers, "Content-Type": "application/json"}, + params=params, + data="{}", + timeout=_TIMEOUT, + ) + + self._refresh_csrf(r) + if r.status_code == 401: + # JWT expired mid-batch — re-auth once and retry + self.login() + r = self._s.post( + f"{self.base}/api/submission/workspaceitems", + headers={**self._write_headers(), "Content-Type": "application/json"}, + params=params, + data="{}", + timeout=_TIMEOUT, + ) + self._refresh_csrf(r) + + r.raise_for_status() + data = r.json() + # File uploads return {_embedded: {workspaceitems: [{id: ...}]}} + # Empty-body creates return a flat {id: ...} object. + ws_id = data.get("id") or ( + data.get("_embedded", {}).get("workspaceitems", [{}])[0].get("id") + ) + if not ws_id: + return {"status": "error", "message": "No workspace item ID returned"} + + # 2. PATCH Dublin Core metadata ──────────────────────────────────────── + patch_ops = _build_metadata_patch(item) + r2 = self._s.patch( + f"{self.base}/api/submission/workspaceitems/{ws_id}", + headers={**self._write_headers(), "Content-Type": "application/json"}, + json=patch_ops, + timeout=_TIMEOUT, + ) + self._refresh_csrf(r2) + if not r2.ok: + logger.warning("metadata patch failed for ws %s: %s %s", ws_id, r2.status_code, r2.text[:200]) + + # 3. Grant the submission license (required by UNILAG DSpace 9 form) ── + license_patch = [{"op": "replace", "path": "/sections/license/granted", "value": True}] + rl = self._s.patch( + f"{self.base}/api/submission/workspaceitems/{ws_id}", + headers={**self._write_headers(), "Content-Type": "application/json"}, + json=license_patch, + timeout=_TIMEOUT, + ) + self._refresh_csrf(rl) + if not rl.ok: + logger.warning("license grant failed for ws %s: %s", ws_id, rl.status_code) + + # 4. Check for blocking validation errors before submitting ─────────── + rv = self._s.get( + f"{self.base}/api/submission/workspaceitems/{ws_id}", + headers=self._write_headers(), + timeout=_TIMEOUT, + ) + self._refresh_csrf(rv) + if rv.ok: + errors = rv.json().get("errors", []) + blocking = [e for e in errors if "filerequired" in e.get("message", "")] + if blocking: + # This collection requires a file; we have no PDF → abort cleanly. + self._s.delete( + f"{self.base}/api/submission/workspaceitems/{ws_id}", + headers=self._write_headers(), + timeout=_TIMEOUT, + ) + return { + "status": "error", + "message": "Collection requires a PDF file; no local file available for this item", + } + + # 5. Submit to workflow → archived ──────────────────────────────────── + r3 = self._s.post( + f"{self.base}/api/workflow/workflowitems", + headers={**self._write_headers(), "Content-Type": "text/uri-list"}, + data=f"{self.base}/api/submission/workspaceitems/{ws_id}", + timeout=_TIMEOUT, + ) + self._refresh_csrf(r3) + r3.raise_for_status() + + dspace_id = r3.json().get("id") or r3.json().get("uuid", "") + return {"status": "ok", "dspace_id": str(dspace_id), "message": "Deposited"} + + +# ── Dublin Core field mapping (§6.4) ───────────────────────────────────────── + +def _mv(value: str, place: int = 0) -> dict: + """Build a DSpace 9 metadata value object (language/authority/confidence required).""" + return { + "value": value, + "language": None, + "authority": None, + "confidence": -1, + "place": place, + } + + +def _build_metadata_patch(item) -> list[dict]: + """Build JSON-Patch ops to set DC fields on a DSpace 9 workspace item. + + DSpace 9 requires full value objects with language/authority/confidence/place. + Multi-value fields are batched into a single op (repeated ops on the same + path would overwrite instead of append). + """ + ops = [] + + def _add(dc_path: str, value: str, section: str = "traditionalpageone"): + if value and value.strip(): + ops.append({ + "op": "add", + "path": f"/sections/{section}/{dc_path}", + "value": [_mv(value.strip())], + }) + + _add("dc.title", item.title or "") + _add("dc.date.issued", item.dc_date_issued or "") + _add("dc.type", item.dc_type or "") + _add("dc.language.iso", item.dc_language or "en") + if item.institution: + _add("dc.publisher", item.institution) + + # URI: prefer explicit dc.identifier.uri, else build from DOI, else fallback to url. + # dc.rights and dc.identifier.doi are not in the UNILAG submission form. + doi = item.doi or "" + uri = ( + item.dc_identifier_uri + or (f"https://doi.org/{doi}" if doi else "") + or item.url + or "" + ) + _add("dc.identifier.uri", uri) + + # Authors — all in one op so every author is preserved + author_names = [ + a.name for a in getattr(item, "authors", []) if getattr(a, "name", "") + ] + if author_names: + ops.append({ + "op": "add", + "path": "/sections/traditionalpageone/dc.contributor.author", + "value": [_mv(n, i) for i, n in enumerate(author_names)], + }) + + # Abstract and subjects go in traditionalpagetwo (DSpace 9 default layout) + if item.abstract and item.abstract.strip(): + ops.append({ + "op": "add", + "path": "/sections/traditionalpagetwo/dc.description.abstract", + "value": [_mv(item.abstract.strip())], + }) + + subject_tags = [t.strip() for t in (item.dc_subject or "").split(",") if t.strip()] + if subject_tags: + ops.append({ + "op": "add", + "path": "/sections/traditionalpagetwo/dc.subject", + "value": [_mv(t, i) for i, t in enumerate(subject_tags)], + }) + + # ARK + DocID as identifiers + other_ids = [v for v in [item.ark, item.docid] if v] + if other_ids: + ops.append({ + "op": "add", + "path": "/sections/traditionalpageone/dc.identifier.other", + "value": [_mv(v, i) for i, v in enumerate(other_ids)], + }) + + return ops diff --git a/uraas/services/narratives.py b/uraas/services/narratives.py index e10f821e060472d634e5061e5953ca55242b5a1c..3dd509cc0c239bf7cbb3549493b9b564bac78067 100644 --- a/uraas/services/narratives.py +++ b/uraas/services/narratives.py @@ -1,54 +1,54 @@ -""" -Deterministic narrative templates — one human-readable insight sentence per -chart, generated server-side from the precomputed metrics and returned in the -API envelope's `narrative` field. No LLM dependency; failures return "" so a -narrative can never break an endpoint. -""" - -import logging - -logger = logging.getLogger(__name__) - -NARRATIVE_TEMPLATES = { - "alignment_profile": ( - "{institution} shows strongest alignment with “{top_pillar}” " - "({top_score}/100 across {top_count} papers); {gap_count} of " - "{pillar_count} pillars fall below the gap threshold of {threshold}." - ), - "alignment_gaps": ( - "{gap_count} research gaps identified across {framework_count} frameworks — " - "the weakest area is “{worst_pillar}” ({worst_framework}) at {worst_score}/100." - ), - "intra_african": ( - "Intra-African collaboration is {pct}% — {ratio}× the continental " - "average of {baseline}% (Research Policy, 2022). Top partner: {top_partner}." - ), - "intra_african_no_partner": ( - "Intra-African collaboration is {pct}% against a continental average " - "of {baseline}% (Research Policy, 2022)." - ), - "country_pairs": ( - "{pair_count} active country pairs; the strongest link is " - "{country_a}–{country_b} with {count} co-publications." - ), - "citation_velocity": ( - "Citations are accruing at {recent_rate} per year; on average papers " - "gather {avg_first2y} citations in their first two years." - ), - "pan_african_share": ( - "{share}% of citations to this collection come from African " - "institutions (computed for the {covered} most-cited works)." - ), -} - - -def narrate(template_key: str, **values) -> str: - """Render a narrative template; returns "" on any error.""" - template = NARRATIVE_TEMPLATES.get(template_key) - if not template: - return "" - try: - return template.format(**values) - except Exception as e: - logger.debug("narrate(%s) failed: %s", template_key, e) - return "" +""" +Deterministic narrative templates — one human-readable insight sentence per +chart, generated server-side from the precomputed metrics and returned in the +API envelope's `narrative` field. No LLM dependency; failures return "" so a +narrative can never break an endpoint. +""" + +import logging + +logger = logging.getLogger(__name__) + +NARRATIVE_TEMPLATES = { + "alignment_profile": ( + "{institution} shows strongest alignment with “{top_pillar}” " + "({top_score}/100 across {top_count} papers); {gap_count} of " + "{pillar_count} pillars fall below the gap threshold of {threshold}." + ), + "alignment_gaps": ( + "{gap_count} research gaps identified across {framework_count} frameworks — " + "the weakest area is “{worst_pillar}” ({worst_framework}) at {worst_score}/100." + ), + "intra_african": ( + "Intra-African collaboration is {pct}% — {ratio}× the continental " + "average of {baseline}% (Research Policy, 2022). Top partner: {top_partner}." + ), + "intra_african_no_partner": ( + "Intra-African collaboration is {pct}% against a continental average " + "of {baseline}% (Research Policy, 2022)." + ), + "country_pairs": ( + "{pair_count} active country pairs; the strongest link is " + "{country_a}–{country_b} with {count} co-publications." + ), + "citation_velocity": ( + "Citations are accruing at {recent_rate} per year; on average papers " + "gather {avg_first2y} citations in their first two years." + ), + "pan_african_share": ( + "{share}% of citations to this collection come from African " + "institutions (computed for the {covered} most-cited works)." + ), +} + + +def narrate(template_key: str, **values) -> str: + """Render a narrative template; returns "" on any error.""" + template = NARRATIVE_TEMPLATES.get(template_key) + if not template: + return "" + try: + return template.format(**values) + except Exception as e: + logger.debug("narrate(%s) failed: %s", template_key, e) + return "" diff --git a/uraas/services/sc_engine.py b/uraas/services/sc_engine.py index b0e5e55f8f9f6768d314cf4cfd14ea5e31882461..b9db4721efb8ff87fa0e9ca666632e25384b6525 100644 --- a/uraas/services/sc_engine.py +++ b/uraas/services/sc_engine.py @@ -1,302 +1,302 @@ -""" -Special Collections Decision Engine — the single source of truth for deciding -whether a paper belongs to URAAS's Special Collections (SC). - -This is an in-process decision engine (no Elasticsearch/Celery): it layers -negative filters on top of the positive keyword taxonomy so that grey -literature, jargon, and STEM/medical noise are rejected, while genuine -indigenous-knowledge / African-literature / cultural-heritage work is kept. - -It is consumed by: - - the crawl pipeline (uraas/pipelines/database.py) at save time - - the analytics SC gate (uraas/analytics/engine.py:_get_sc_item_ids) - - the comparator (uraas/services/comparator_engine.py) via SC_FILTER - - the re-classify/prune script (scripts/reclassify_and_prune_sc.py) - -The keyword taxonomy itself lives in uraas.utils.ai_classifier.SPECIAL_COLLECTIONS -— we import it rather than redefine it. -""" - -import re -from typing import List, Set, Tuple - -from uraas.database import Item -from uraas.utils.ai_classifier import ( - SPECIAL_COLLECTIONS, - _clean_text, - _keyword_score, -) - -# ── SQLAlchemy predicate: after re-scoring, score>0 ≡ Special Collection ────── -# Defined once here so every query (analytics, comparator) filters identically. -SC_FILTER = Item.special_collection_score > 0 - -# ── Category weighting ──────────────────────────────────────────────────────── -# "Strong" categories can qualify a paper on their own. "Ethnic Languages & -# Groups" is support-only: it corroborates but cannot solely qualify a paper, -# because it contains bare ethnonym/language tokens that are false-positive -# magnets (e.g. "ss" → stainless steel, "ewe" → sheep). -STRONG_CATEGORIES: Set[str] = { - "Indigenous Knowledge", - "African Literature", - "Cultural Heritage", - "Postcolonial Studies", - "Pan-African Studies", - "African Philosophy", - "Ethnomusicology", -} -SUPPORT_CATEGORIES: Set[str] = {"Ethnic Languages & Groups"} - -# Bare tokens that must NOT qualify a paper unless they appear as a multi-word -# phrase or a second independent category also matches. These collide heavily -# with non-SC vocabulary. -AMBIGUOUS_TOKENS: Set[str] = { - "ss", - "ewe", - "luo", - "twi", - "akan", - "sotho", - "venda", - "oromo", - "igbo", - "hausa", - "yoruba", - "zulu", - "xhosa", - "shona", - "somali", - "wolof", - "ganda", - "fante", - "chewa", - "tsonga", - "ndebele", - "maasai", - "fulani", - "kikuyu", - "swahili", - "amharic", - "tigrinya", - "kinyarwanda", - "lingala", - "bambara", - "setswana", - "sesotho", -} - -# Strong-category keywords that are themselves ambiguous: they appear as research -# *methods* or *regions* in otherwise non-SC papers (e.g. "ethnography" as a method, -# "ecowas"/"african union" as a study region, the homonym "african literature" = -# academic literature). A lone match from one of these needs SC context or a second -# corroborating category — same treatment as ambiguous ethnonyms. -AMBIGUOUS_STRONG: Set[str] = { - "ethnography", - "ecowas", - "sadc", - "african union", - "african continental", - "african development", - "african literature", # collides with "African [academic] literature review" - "east african community", -} - -# Cultural / linguistic context terms. When a bare ethnonym (AMBIGUOUS_TOKENS) -# co-occurs with one of these, the paper is genuinely about that people/language -# (e.g. "Personal name in Igbo Culture") rather than an incidental token collision -# (e.g. "Bacillus thuringiensis SS2"). Lets ethnonym-only papers qualify when the -# surrounding text confirms cultural/linguistic/historical intent. -SC_CONTEXT = re.compile( - r"\b(culture|cultural|language|languages|linguistic|linguistics|" - r"oral tradition|oral literature|folklore|folktale|folk tale|" - r"traditional knowledge|indigenous knowledge|heritage|naming|" - r"proverb|proverbs|ethnic group|ethnic groups|kingdom|" - r"colonial rule|postcolonial|precolonial|indigenous|ancestral|" - r"ritual|cosmology|worldview|mythology|customs|" - r"griot|drumming|ethnomusicolog)\b", - re.IGNORECASE, -) - -# STEM / medical / engineering exclusion. Reused from the tiered logic that -# already exists in the dashboard's language endpoint (app.py). A paper that -# matches EXCLUDE is dropped unless it carries a *strong* SC signal. -EXCLUDE = re.compile( - r"\b(machine learning|deep learning|neural network|artificial intelligence|" - r"clinical trial|randomized|randomised|patient|hospital|surgery|cancer|tumor|tumour|" - r"cardiovascular|hypertension|diabetes|preeclampsia|concrete|cement|" - r"compressive strength|tensile|alloy|composite|carbon emission|" - r"ecological footprint|gdp|economic growth|galaxy|astrophysic|ionosphere|" - r"plasma|quantum|semiconductor|mpox|covid|sars|influenza|malaria|hiv|" - r"antibiotic|cybersecurity|blockchain|iot|cloud computing|petroleum|" - r"crude oil|refinery|corrosion|nanoparticle|photovoltaic|wastewater|" - r"groundwater|finite element|stainless steel|catalyst|polymer)\b", - re.IGNORECASE, -) - - -def _category_hits(text: str) -> List[Tuple[str, int, List[str]]]: - """Return [(category, raw_match_count, matched_keywords)] for matched categories.""" - hits = [] - for category, keywords in SPECIAL_COLLECTIONS.items(): - count, matched = _keyword_score(text, keywords) - if count >= 1: - hits.append((category, count, matched)) - return hits - - -def _is_multiword(phrase: str) -> bool: - return " " in phrase.strip() - - -def is_special_collection( - title: str, abstract: str, dc_subject: str = "" -) -> Tuple[bool, float, List[str]]: - """ - Decide whether a paper is a Special Collection. - - Returns (is_sc, score, categories). - - score = sum(strong_matches * 3 + support_matches * 1); 0 when not SC. - - categories = list of matched SC category names (only meaningful when is_sc). - - Decision gates (in order): - 1. Empty-text reject — require real title/abstract text. - 2. Ambiguous-token guard — bare ethnonym tokens only count via a multi-word - phrase or a second independent category. - 3. STEM exclusion — EXCLUDE hit drops the paper unless a strong signal exists. - 4. Confidence — keep iff (>=1 strong category) OR (>=1 strong multi-word - phrase and not excluded) OR (>=2 distinct categories). - """ - # Gate 1: require title/abstract text (concept tags alone don't qualify). - primary = _clean_text(f"{title or ''} {abstract or ''}").lower() - if not primary.strip(): - return (False, 0.0, []) - - full = _clean_text(f"{title or ''} {abstract or ''} {dc_subject or ''}").lower() - - all_hits = _category_hits(full) - if not all_hits: - return (False, 0.0, []) - - has_context = bool(SC_CONTEXT.search(full)) - - # A second-category corroboration check: count distinct categories with any - # raw (pre-filter) hit, used to decide whether an ambiguous lone keyword counts. - raw_categories = {h[0] for h in all_hits} - has_second_category = len(raw_categories) >= 2 - - def _filter_ambiguous(matched, ambiguous_set): - """Keep keywords that aren't ambiguous, or that are corroborated by SC - context / a second category. An ambiguous keyword (whether single- or - multi-word, e.g. bare "igbo" or the homonym phrase "african literature") - only counts when context or a second category corroborates it. A - non-ambiguous multi-word phrase always counts.""" - kept = [] - for kw in matched: - if kw.lower() in ambiguous_set: - if has_context or has_second_category: - kept.append(kw) - else: - kept.append(kw) - return kept - - # Gate 2: filter ambiguous matches in BOTH strong and support categories. - # Bare ethnonyms ("igbo") and homonym method/region terms ("ethnography", - # "ecowas", "african literature") only count alone when context corroborates — - # this separates real ethnic/cultural studies from incidental token collisions. - strong_hits = [] - for cat, count, matched in all_hits: - if cat not in STRONG_CATEGORIES: - continue - kept = _filter_ambiguous(matched, AMBIGUOUS_STRONG) - if kept: - strong_hits.append((cat, len(kept), kept)) - - support_hits = [] - for cat, count, matched in all_hits: - if cat not in SUPPORT_CATEGORIES: - continue - kept = _filter_ambiguous(matched, AMBIGUOUS_TOKENS) - if kept: - support_hits.append((cat, len(kept), kept)) - - qualifying = strong_hits + support_hits - if not qualifying: - return (False, 0.0, []) - - categories = [c for c, _, _ in qualifying] - # A "qualifying phrase" is any multi-word matched keyword from a strong OR a - # support category — multi-word ethnonym phrases (e.g. "yoruba cosmology", - # "swahili coast") are specific enough to stand on their own. - has_qualifying_phrase = any( - _is_multiword(kw) for _, _, matched in qualifying for kw in matched - ) - # A bare ethnonym that survived Gate 2 (i.e. context present) qualifies the - # paper on its own — distinguishes "Igbo Culture" from "SS2". - has_context_support = bool(support_hits) and has_context - - # Gate 3: STEM exclusion — needs a strong signal to survive. - excluded = bool(EXCLUDE.search(full)) - if excluded and not strong_hits: - return (False, 0.0, []) - - # Gate 4: confidence. - keep = ( - bool(strong_hits) - or (has_qualifying_phrase and not excluded) - or (len(set(categories)) >= 2) - or (has_context_support and not excluded) - ) - if not keep: - return (False, 0.0, []) - - score = float(sum(c * 3 for _, c, _ in strong_hits) + sum(c for _, c, _ in support_hits)) - if score <= 0: - return (False, 0.0, []) - - return (True, score, categories) - - -def category_breakdown(title: str, abstract: str, dc_subject: str = "") -> List[dict]: - """ - For a paper already known to be SC, return per-category detail for display: - [{category, score, matched_keywords}]. Only the categories that actually - qualified the paper (after the ambiguity/context gates) are returned, so the - analytics breakdown matches the keep/drop decision exactly. - """ - is_sc, _, categories = is_special_collection(title, abstract, dc_subject) - if not is_sc: - return [] - full = _clean_text(f"{title or ''} {abstract or ''} {dc_subject or ''}").lower() - out = [] - for category in categories: - count, matched = _keyword_score(full, SPECIAL_COLLECTIONS.get(category, [])) - weight = 3 if category in STRONG_CATEGORIES else 1 - out.append( - { - "category": category, - "score": count * weight, - "matched_keywords": matched[:6], - } - ) - out.sort(key=lambda x: -x["score"]) - return out - - -def score_text(title: str, abstract: str, dc_subject: str = "") -> Tuple[float, str]: - """Convenience wrapper returning (score, comma-joined categories) for storage.""" - is_sc, score, categories = is_special_collection(title, abstract, dc_subject) - return (score, ",".join(categories) if is_sc else "") - - -if __name__ == "__main__": - # Tiny smoke check - samples = [ - ("Yoruba cosmology and oral tradition in Ifa divination", ""), - ("Compressive strength of recycled concrete aggregate", ""), - ("Indigenous medicine for malaria among the Igbo", "ethnobotany traditional healing"), - ("Deep learning for tumour segmentation", ""), - ("A study of SS 304 stainless steel corrosion", ""), - ("Ubuntu philosophy and African communalism", ""), - ] - for t, a in samples: - print(is_special_collection(t, a), "::", t) +""" +Special Collections Decision Engine — the single source of truth for deciding +whether a paper belongs to URAAS's Special Collections (SC). + +This is an in-process decision engine (no Elasticsearch/Celery): it layers +negative filters on top of the positive keyword taxonomy so that grey +literature, jargon, and STEM/medical noise are rejected, while genuine +indigenous-knowledge / African-literature / cultural-heritage work is kept. + +It is consumed by: + - the crawl pipeline (uraas/pipelines/database.py) at save time + - the analytics SC gate (uraas/analytics/engine.py:_get_sc_item_ids) + - the comparator (uraas/services/comparator_engine.py) via SC_FILTER + - the re-classify/prune script (scripts/reclassify_and_prune_sc.py) + +The keyword taxonomy itself lives in uraas.utils.ai_classifier.SPECIAL_COLLECTIONS +— we import it rather than redefine it. +""" + +import re +from typing import List, Set, Tuple + +from uraas.database import Item +from uraas.utils.ai_classifier import ( + SPECIAL_COLLECTIONS, + _clean_text, + _keyword_score, +) + +# ── SQLAlchemy predicate: after re-scoring, score>0 ≡ Special Collection ────── +# Defined once here so every query (analytics, comparator) filters identically. +SC_FILTER = Item.special_collection_score > 0 + +# ── Category weighting ──────────────────────────────────────────────────────── +# "Strong" categories can qualify a paper on their own. "Ethnic Languages & +# Groups" is support-only: it corroborates but cannot solely qualify a paper, +# because it contains bare ethnonym/language tokens that are false-positive +# magnets (e.g. "ss" → stainless steel, "ewe" → sheep). +STRONG_CATEGORIES: Set[str] = { + "Indigenous Knowledge", + "African Literature", + "Cultural Heritage", + "Postcolonial Studies", + "Pan-African Studies", + "African Philosophy", + "Ethnomusicology", +} +SUPPORT_CATEGORIES: Set[str] = {"Ethnic Languages & Groups"} + +# Bare tokens that must NOT qualify a paper unless they appear as a multi-word +# phrase or a second independent category also matches. These collide heavily +# with non-SC vocabulary. +AMBIGUOUS_TOKENS: Set[str] = { + "ss", + "ewe", + "luo", + "twi", + "akan", + "sotho", + "venda", + "oromo", + "igbo", + "hausa", + "yoruba", + "zulu", + "xhosa", + "shona", + "somali", + "wolof", + "ganda", + "fante", + "chewa", + "tsonga", + "ndebele", + "maasai", + "fulani", + "kikuyu", + "swahili", + "amharic", + "tigrinya", + "kinyarwanda", + "lingala", + "bambara", + "setswana", + "sesotho", +} + +# Strong-category keywords that are themselves ambiguous: they appear as research +# *methods* or *regions* in otherwise non-SC papers (e.g. "ethnography" as a method, +# "ecowas"/"african union" as a study region, the homonym "african literature" = +# academic literature). A lone match from one of these needs SC context or a second +# corroborating category — same treatment as ambiguous ethnonyms. +AMBIGUOUS_STRONG: Set[str] = { + "ethnography", + "ecowas", + "sadc", + "african union", + "african continental", + "african development", + "african literature", # collides with "African [academic] literature review" + "east african community", +} + +# Cultural / linguistic context terms. When a bare ethnonym (AMBIGUOUS_TOKENS) +# co-occurs with one of these, the paper is genuinely about that people/language +# (e.g. "Personal name in Igbo Culture") rather than an incidental token collision +# (e.g. "Bacillus thuringiensis SS2"). Lets ethnonym-only papers qualify when the +# surrounding text confirms cultural/linguistic/historical intent. +SC_CONTEXT = re.compile( + r"\b(culture|cultural|language|languages|linguistic|linguistics|" + r"oral tradition|oral literature|folklore|folktale|folk tale|" + r"traditional knowledge|indigenous knowledge|heritage|naming|" + r"proverb|proverbs|ethnic group|ethnic groups|kingdom|" + r"colonial rule|postcolonial|precolonial|indigenous|ancestral|" + r"ritual|cosmology|worldview|mythology|customs|" + r"griot|drumming|ethnomusicolog)\b", + re.IGNORECASE, +) + +# STEM / medical / engineering exclusion. Reused from the tiered logic that +# already exists in the dashboard's language endpoint (app.py). A paper that +# matches EXCLUDE is dropped unless it carries a *strong* SC signal. +EXCLUDE = re.compile( + r"\b(machine learning|deep learning|neural network|artificial intelligence|" + r"clinical trial|randomized|randomised|patient|hospital|surgery|cancer|tumor|tumour|" + r"cardiovascular|hypertension|diabetes|preeclampsia|concrete|cement|" + r"compressive strength|tensile|alloy|composite|carbon emission|" + r"ecological footprint|gdp|economic growth|galaxy|astrophysic|ionosphere|" + r"plasma|quantum|semiconductor|mpox|covid|sars|influenza|malaria|hiv|" + r"antibiotic|cybersecurity|blockchain|iot|cloud computing|petroleum|" + r"crude oil|refinery|corrosion|nanoparticle|photovoltaic|wastewater|" + r"groundwater|finite element|stainless steel|catalyst|polymer)\b", + re.IGNORECASE, +) + + +def _category_hits(text: str) -> List[Tuple[str, int, List[str]]]: + """Return [(category, raw_match_count, matched_keywords)] for matched categories.""" + hits = [] + for category, keywords in SPECIAL_COLLECTIONS.items(): + count, matched = _keyword_score(text, keywords) + if count >= 1: + hits.append((category, count, matched)) + return hits + + +def _is_multiword(phrase: str) -> bool: + return " " in phrase.strip() + + +def is_special_collection( + title: str, abstract: str, dc_subject: str = "" +) -> Tuple[bool, float, List[str]]: + """ + Decide whether a paper is a Special Collection. + + Returns (is_sc, score, categories). + - score = sum(strong_matches * 3 + support_matches * 1); 0 when not SC. + - categories = list of matched SC category names (only meaningful when is_sc). + + Decision gates (in order): + 1. Empty-text reject — require real title/abstract text. + 2. Ambiguous-token guard — bare ethnonym tokens only count via a multi-word + phrase or a second independent category. + 3. STEM exclusion — EXCLUDE hit drops the paper unless a strong signal exists. + 4. Confidence — keep iff (>=1 strong category) OR (>=1 strong multi-word + phrase and not excluded) OR (>=2 distinct categories). + """ + # Gate 1: require title/abstract text (concept tags alone don't qualify). + primary = _clean_text(f"{title or ''} {abstract or ''}").lower() + if not primary.strip(): + return (False, 0.0, []) + + full = _clean_text(f"{title or ''} {abstract or ''} {dc_subject or ''}").lower() + + all_hits = _category_hits(full) + if not all_hits: + return (False, 0.0, []) + + has_context = bool(SC_CONTEXT.search(full)) + + # A second-category corroboration check: count distinct categories with any + # raw (pre-filter) hit, used to decide whether an ambiguous lone keyword counts. + raw_categories = {h[0] for h in all_hits} + has_second_category = len(raw_categories) >= 2 + + def _filter_ambiguous(matched, ambiguous_set): + """Keep keywords that aren't ambiguous, or that are corroborated by SC + context / a second category. An ambiguous keyword (whether single- or + multi-word, e.g. bare "igbo" or the homonym phrase "african literature") + only counts when context or a second category corroborates it. A + non-ambiguous multi-word phrase always counts.""" + kept = [] + for kw in matched: + if kw.lower() in ambiguous_set: + if has_context or has_second_category: + kept.append(kw) + else: + kept.append(kw) + return kept + + # Gate 2: filter ambiguous matches in BOTH strong and support categories. + # Bare ethnonyms ("igbo") and homonym method/region terms ("ethnography", + # "ecowas", "african literature") only count alone when context corroborates — + # this separates real ethnic/cultural studies from incidental token collisions. + strong_hits = [] + for cat, count, matched in all_hits: + if cat not in STRONG_CATEGORIES: + continue + kept = _filter_ambiguous(matched, AMBIGUOUS_STRONG) + if kept: + strong_hits.append((cat, len(kept), kept)) + + support_hits = [] + for cat, count, matched in all_hits: + if cat not in SUPPORT_CATEGORIES: + continue + kept = _filter_ambiguous(matched, AMBIGUOUS_TOKENS) + if kept: + support_hits.append((cat, len(kept), kept)) + + qualifying = strong_hits + support_hits + if not qualifying: + return (False, 0.0, []) + + categories = [c for c, _, _ in qualifying] + # A "qualifying phrase" is any multi-word matched keyword from a strong OR a + # support category — multi-word ethnonym phrases (e.g. "yoruba cosmology", + # "swahili coast") are specific enough to stand on their own. + has_qualifying_phrase = any( + _is_multiword(kw) for _, _, matched in qualifying for kw in matched + ) + # A bare ethnonym that survived Gate 2 (i.e. context present) qualifies the + # paper on its own — distinguishes "Igbo Culture" from "SS2". + has_context_support = bool(support_hits) and has_context + + # Gate 3: STEM exclusion — needs a strong signal to survive. + excluded = bool(EXCLUDE.search(full)) + if excluded and not strong_hits: + return (False, 0.0, []) + + # Gate 4: confidence. + keep = ( + bool(strong_hits) + or (has_qualifying_phrase and not excluded) + or (len(set(categories)) >= 2) + or (has_context_support and not excluded) + ) + if not keep: + return (False, 0.0, []) + + score = float(sum(c * 3 for _, c, _ in strong_hits) + sum(c for _, c, _ in support_hits)) + if score <= 0: + return (False, 0.0, []) + + return (True, score, categories) + + +def category_breakdown(title: str, abstract: str, dc_subject: str = "") -> List[dict]: + """ + For a paper already known to be SC, return per-category detail for display: + [{category, score, matched_keywords}]. Only the categories that actually + qualified the paper (after the ambiguity/context gates) are returned, so the + analytics breakdown matches the keep/drop decision exactly. + """ + is_sc, _, categories = is_special_collection(title, abstract, dc_subject) + if not is_sc: + return [] + full = _clean_text(f"{title or ''} {abstract or ''} {dc_subject or ''}").lower() + out = [] + for category in categories: + count, matched = _keyword_score(full, SPECIAL_COLLECTIONS.get(category, [])) + weight = 3 if category in STRONG_CATEGORIES else 1 + out.append( + { + "category": category, + "score": count * weight, + "matched_keywords": matched[:6], + } + ) + out.sort(key=lambda x: -x["score"]) + return out + + +def score_text(title: str, abstract: str, dc_subject: str = "") -> Tuple[float, str]: + """Convenience wrapper returning (score, comma-joined categories) for storage.""" + is_sc, score, categories = is_special_collection(title, abstract, dc_subject) + return (score, ",".join(categories) if is_sc else "") + + +if __name__ == "__main__": + # Tiny smoke check + samples = [ + ("Yoruba cosmology and oral tradition in Ifa divination", ""), + ("Compressive strength of recycled concrete aggregate", ""), + ("Indigenous medicine for malaria among the Igbo", "ethnobotany traditional healing"), + ("Deep learning for tumour segmentation", ""), + ("A study of SS 304 stainless steel corrosion", ""), + ("Ubuntu philosophy and African communalism", ""), + ] + for t, a in samples: + print(is_special_collection(t, a), "::", t) diff --git a/uraas/spiders/sources/ajol_spider.py b/uraas/spiders/sources/ajol_spider.py index 8f290dbd3179bf6db4ddd879cf49163263e853bc..3ad43d44ecac391ffbf011f75432d02df6d4e224 100644 --- a/uraas/spiders/sources/ajol_spider.py +++ b/uraas/spiders/sources/ajol_spider.py @@ -1,248 +1,248 @@ -""" -AJOL spider — African Journals Online (ajol.info). - -AJOL is the most important African-specific journal aggregator, hosting -2,000+ peer-reviewed journals from 40+ African countries. It is the -primary source for Nigerian humanities, social sciences, law, and -indigenous knowledge research not indexed in OpenAlex or Crossref. - -Uses AJOL's search endpoint and parses the HTML results. The site -structure is stable (OJS-based); falling back to OAI-PMH harvest is -possible but much slower without affiliation filtering. - -No API key needed. Rate limit: polite 2s delay. -""" - -import os -import sys -import re -from urllib.parse import urlencode, urljoin - -import scrapy -from scrapy.http import Request - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) - -from uraas.config import config -from uraas.config.institutions import get_registry -from uraas.config.special_collections import SC_SEED_KEYWORDS - -_SEARCH_BASE = "https://www.ajol.info/index.php/ajol/search/results" -_DOI_RE = re.compile(r"10\.\d{4,}/\S+") -_SC_SEEDS = [ - "indigenous knowledge", "cultural heritage", "african literature", - "oral tradition", "ethnobotany", "traditional medicine", "postcolonial", - "yoruba", "igbo", "hausa", "nigeria", "west africa", -] - - -class AJOLSpider(scrapy.Spider): - name = "ajol" - allowed_domains = ["www.ajol.info"] - custom_settings = { - "DOWNLOAD_DELAY": 2.5, - "CONCURRENT_REQUESTS": 1, - # AJOL blocks non-browser UAs; mimic a real browser to avoid 403. - "USER_AGENT": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/124.0.0.0 Safari/537.36" - ), - "DEFAULT_REQUEST_HEADERS": { - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - "Accept-Encoding": "gzip, deflate, br", - "Referer": "https://www.ajol.info/", - }, - "HTTPERROR_ALLOWED_CODES": [403, 429], - } - - def __init__( - self, institution="unilag", target=50, boost_special=True, sc_only=False, - *args, **kwargs, - ): - super().__init__(*args, **kwargs) - self.target_limit = int(target) - _truthy = {"1", "true", "yes", "on"} - self.boost_special = ( - boost_special.lower() in _truthy - if isinstance(boost_special, str) - else bool(boost_special) - ) - self.sc_only = ( - sc_only.lower() in _truthy - if isinstance(sc_only, str) - else bool(sc_only) - ) - registry = get_registry() - self.institution_config = registry.get(institution) - if not self.institution_config: - raise ValueError(f"Institution '{institution}' not found") - self.institution_name = self.institution_config.name - self.ror_id = self.institution_config.ror - self._accepted = 0 - self._seen_urls: set = set() - - # Build short affiliation terms for matching - name_parts = self.institution_name.lower().split() - self._affil_terms = [self.institution_name.lower()] - if "university" in name_parts: - idx = name_parts.index("university") - if idx > 0: - self._affil_terms.append(name_parts[idx - 1]) # e.g. "lagos" - - def _build_url(self, query: str, page: int = 1) -> str: - params = { - "query": query, - "searchField": "query", - "orderBy": "score", - "sort": "", - "limit": 20, - "page": page, - } - return f"{_SEARCH_BASE}?{urlencode(params)}" - - async def start(self): - seen_queries: set = set() - - if not self.sc_only: - q = f'"{self.institution_name}"' - seen_queries.add(q) - yield Request( - self._build_url(q), callback=self.parse_list, - meta={"query": q, "page": 1}, - ) - - if self.boost_special: - for seed in _SC_SEEDS[:8]: - q = f'"{seed}"' - if q not in seen_queries: - seen_queries.add(q) - yield Request( - self._build_url(q), callback=self.parse_list, - meta={"query": q, "page": 1}, priority=5, - ) - - def parse_list(self, response): - if response.status in (403, 429): - self.logger.warning(f"AJOL blocked ({response.status}) — skipping: {response.url[:80]}") - return - if self._accepted >= self.target_limit: - return - - # Each result is a div with class "result" or "article_summary" - # AJOL uses OJS — the search results page lists article titles with links - results = response.css("div.article_summary, li.article, .search-result") - if not results: - # Fallback: any heading-linked article - results = response.css("h4 a, h3 a, .result a") - - for item in results: - if self._accepted >= self.target_limit: - return - link = item.css("a::attr(href)").get() or item.attrib.get("href", "") - if not link: - continue - full_url = urljoin(response.url, link) - if "/article/view/" not in full_url and "/article/download/" not in full_url: - continue - if full_url in self._seen_urls: - continue - self._seen_urls.add(full_url) - title = item.css("a::text").get("").strip() - yield Request( - full_url, callback=self.parse_article, - meta={"title": title, "url": full_url}, - ) - - # Pagination - page = response.meta.get("page", 1) - query = response.meta.get("query", "") - next_link = response.css("a.next::attr(href), a[rel=next]::attr(href)").get() - if next_link and self._accepted < self.target_limit: - yield Request( - urljoin(response.url, next_link), callback=self.parse_list, - meta={"query": query, "page": page + 1}, - ) - elif page < 5 and self._accepted < self.target_limit and results: - # Fallback numeric pagination - yield Request( - self._build_url(query, page + 1), callback=self.parse_list, - meta={"query": query, "page": page + 1}, - ) - - def parse_article(self, response): - """Extract metadata from an AJOL article detail page (OJS-based).""" - if response.status in (403, 429): - self.logger.warning(f"AJOL blocked ({response.status}) on article — skipping") - return - title = ( - response.css("h1.page-title::text, h1.article-title::text, h3.title::text").get("") - or response.meta.get("title", "") - ).strip() - if not title: - return - - abstract = " ".join( - response.css( - "div.abstract p::text, section.abstract p::text, " - "#articleAbstract p::text, .abstractSection p::text" - ).getall() - ).strip() - - authors = response.css( - "div.authors span::text, ul.authors li span.name::text, " - ".author-string-href::text" - ).getall() - authors = [a.strip() for a in authors if a.strip()] - - # DOI — look for the canonical DOI link or meta tag - doi = ( - response.css("meta[name='DC.Identifier.DOI']::attr(content)").get() - or response.css("a[href*='doi.org']::text").re_first(r"10\.\d{4,}/\S+") - or "" - ).strip() - - pdf_url = ( - response.css("a.pdf::attr(href), a[href*='download']::attr(href)").get() - ) - if pdf_url: - pdf_url = urljoin(response.url, pdf_url) - - pub_date = ( - response.css("meta[name='DC.Date.issued']::attr(content)").get() - or response.css(".published::text, .date::text").re_first(r"\d{4}") - or "" - ) - - affil_text = " ".join( - response.css( - ".affiliations::text, .author-affiliation::text, meta[name='citation_author_institution']::attr(content)" - ).getall() - ).lower() - - # Only yield if the affiliation matches our institution - matches = any(term in affil_text for term in self._affil_terms) - if not matches: - # If abstract/body text mentions the institution, still accept - page_text = response.text.lower() - matches = any(term in page_text for term in self._affil_terms) - - if not matches: - return - - self._accepted += 1 - yield { - "title": title, - "abstract": abstract, - "authors": authors, - "doi": doi or None, - "url": response.meta.get("url", response.url), - "pdf_url": pdf_url, - "publication_date": pub_date[:10] if pub_date else "", - "source_repository": "AJOL", - "is_unilag_author": True, - "raw_affiliation": affil_text[:500] or self.institution_name, - "institution": self.institution_name, - "institution_ror": self.ror_id, - } +""" +AJOL spider — African Journals Online (ajol.info). + +AJOL is the most important African-specific journal aggregator, hosting +2,000+ peer-reviewed journals from 40+ African countries. It is the +primary source for Nigerian humanities, social sciences, law, and +indigenous knowledge research not indexed in OpenAlex or Crossref. + +Uses AJOL's search endpoint and parses the HTML results. The site +structure is stable (OJS-based); falling back to OAI-PMH harvest is +possible but much slower without affiliation filtering. + +No API key needed. Rate limit: polite 2s delay. +""" + +import os +import sys +import re +from urllib.parse import urlencode, urljoin + +import scrapy +from scrapy.http import Request + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) + +from uraas.config import config +from uraas.config.institutions import get_registry +from uraas.config.special_collections import SC_SEED_KEYWORDS + +_SEARCH_BASE = "https://www.ajol.info/index.php/ajol/search/results" +_DOI_RE = re.compile(r"10\.\d{4,}/\S+") +_SC_SEEDS = [ + "indigenous knowledge", "cultural heritage", "african literature", + "oral tradition", "ethnobotany", "traditional medicine", "postcolonial", + "yoruba", "igbo", "hausa", "nigeria", "west africa", +] + + +class AJOLSpider(scrapy.Spider): + name = "ajol" + allowed_domains = ["www.ajol.info"] + custom_settings = { + "DOWNLOAD_DELAY": 2.5, + "CONCURRENT_REQUESTS": 1, + # AJOL blocks non-browser UAs; mimic a real browser to avoid 403. + "USER_AGENT": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/124.0.0.0 Safari/537.36" + ), + "DEFAULT_REQUEST_HEADERS": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + "Accept-Encoding": "gzip, deflate, br", + "Referer": "https://www.ajol.info/", + }, + "HTTPERROR_ALLOWED_CODES": [403, 429], + } + + def __init__( + self, institution="unilag", target=50, boost_special=True, sc_only=False, + *args, **kwargs, + ): + super().__init__(*args, **kwargs) + self.target_limit = int(target) + _truthy = {"1", "true", "yes", "on"} + self.boost_special = ( + boost_special.lower() in _truthy + if isinstance(boost_special, str) + else bool(boost_special) + ) + self.sc_only = ( + sc_only.lower() in _truthy + if isinstance(sc_only, str) + else bool(sc_only) + ) + registry = get_registry() + self.institution_config = registry.get(institution) + if not self.institution_config: + raise ValueError(f"Institution '{institution}' not found") + self.institution_name = self.institution_config.name + self.ror_id = self.institution_config.ror + self._accepted = 0 + self._seen_urls: set = set() + + # Build short affiliation terms for matching + name_parts = self.institution_name.lower().split() + self._affil_terms = [self.institution_name.lower()] + if "university" in name_parts: + idx = name_parts.index("university") + if idx > 0: + self._affil_terms.append(name_parts[idx - 1]) # e.g. "lagos" + + def _build_url(self, query: str, page: int = 1) -> str: + params = { + "query": query, + "searchField": "query", + "orderBy": "score", + "sort": "", + "limit": 20, + "page": page, + } + return f"{_SEARCH_BASE}?{urlencode(params)}" + + async def start(self): + seen_queries: set = set() + + if not self.sc_only: + q = f'"{self.institution_name}"' + seen_queries.add(q) + yield Request( + self._build_url(q), callback=self.parse_list, + meta={"query": q, "page": 1}, + ) + + if self.boost_special: + for seed in _SC_SEEDS[:8]: + q = f'"{seed}"' + if q not in seen_queries: + seen_queries.add(q) + yield Request( + self._build_url(q), callback=self.parse_list, + meta={"query": q, "page": 1}, priority=5, + ) + + def parse_list(self, response): + if response.status in (403, 429): + self.logger.warning(f"AJOL blocked ({response.status}) — skipping: {response.url[:80]}") + return + if self._accepted >= self.target_limit: + return + + # Each result is a div with class "result" or "article_summary" + # AJOL uses OJS — the search results page lists article titles with links + results = response.css("div.article_summary, li.article, .search-result") + if not results: + # Fallback: any heading-linked article + results = response.css("h4 a, h3 a, .result a") + + for item in results: + if self._accepted >= self.target_limit: + return + link = item.css("a::attr(href)").get() or item.attrib.get("href", "") + if not link: + continue + full_url = urljoin(response.url, link) + if "/article/view/" not in full_url and "/article/download/" not in full_url: + continue + if full_url in self._seen_urls: + continue + self._seen_urls.add(full_url) + title = item.css("a::text").get("").strip() + yield Request( + full_url, callback=self.parse_article, + meta={"title": title, "url": full_url}, + ) + + # Pagination + page = response.meta.get("page", 1) + query = response.meta.get("query", "") + next_link = response.css("a.next::attr(href), a[rel=next]::attr(href)").get() + if next_link and self._accepted < self.target_limit: + yield Request( + urljoin(response.url, next_link), callback=self.parse_list, + meta={"query": query, "page": page + 1}, + ) + elif page < 5 and self._accepted < self.target_limit and results: + # Fallback numeric pagination + yield Request( + self._build_url(query, page + 1), callback=self.parse_list, + meta={"query": query, "page": page + 1}, + ) + + def parse_article(self, response): + """Extract metadata from an AJOL article detail page (OJS-based).""" + if response.status in (403, 429): + self.logger.warning(f"AJOL blocked ({response.status}) on article — skipping") + return + title = ( + response.css("h1.page-title::text, h1.article-title::text, h3.title::text").get("") + or response.meta.get("title", "") + ).strip() + if not title: + return + + abstract = " ".join( + response.css( + "div.abstract p::text, section.abstract p::text, " + "#articleAbstract p::text, .abstractSection p::text" + ).getall() + ).strip() + + authors = response.css( + "div.authors span::text, ul.authors li span.name::text, " + ".author-string-href::text" + ).getall() + authors = [a.strip() for a in authors if a.strip()] + + # DOI — look for the canonical DOI link or meta tag + doi = ( + response.css("meta[name='DC.Identifier.DOI']::attr(content)").get() + or response.css("a[href*='doi.org']::text").re_first(r"10\.\d{4,}/\S+") + or "" + ).strip() + + pdf_url = ( + response.css("a.pdf::attr(href), a[href*='download']::attr(href)").get() + ) + if pdf_url: + pdf_url = urljoin(response.url, pdf_url) + + pub_date = ( + response.css("meta[name='DC.Date.issued']::attr(content)").get() + or response.css(".published::text, .date::text").re_first(r"\d{4}") + or "" + ) + + affil_text = " ".join( + response.css( + ".affiliations::text, .author-affiliation::text, meta[name='citation_author_institution']::attr(content)" + ).getall() + ).lower() + + # Only yield if the affiliation matches our institution + matches = any(term in affil_text for term in self._affil_terms) + if not matches: + # If abstract/body text mentions the institution, still accept + page_text = response.text.lower() + matches = any(term in page_text for term in self._affil_terms) + + if not matches: + return + + self._accepted += 1 + yield { + "title": title, + "abstract": abstract, + "authors": authors, + "doi": doi or None, + "url": response.meta.get("url", response.url), + "pdf_url": pdf_url, + "publication_date": pub_date[:10] if pub_date else "", + "source_repository": "AJOL", + "is_unilag_author": True, + "raw_affiliation": affil_text[:500] or self.institution_name, + "institution": self.institution_name, + "institution_ror": self.ror_id, + } diff --git a/uraas/spiders/sources/arxiv_spider.py b/uraas/spiders/sources/arxiv_spider.py index c38526a75fa8cdbf46d35bd1f49596a6ef2756d3..62007d8f152b39c8e98342f0374bf5428af7f1ed 100644 --- a/uraas/spiders/sources/arxiv_spider.py +++ b/uraas/spiders/sources/arxiv_spider.py @@ -1,197 +1,197 @@ -""" -arXiv spider — uses the official Atom API (export.arxiv.org/api/query). - -The HTML search page at arxiv.org/search was fragile and layout-dependent. -The Atom API is stable, rate-limit-friendly (1 req/3s recommended), and -returns clean structured XML metadata. - -Docs: info.arxiv.org/help/api/basics.html -Rate limit: 3 req/s; we use 1.5s download delay to stay polite. -""" - -import os -import sys -from urllib.parse import urlencode -from xml.etree import ElementTree - -import scrapy - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) - -from uraas.config import config -from uraas.config.institutions import get_registry -from uraas.config.special_collections import SC_SEED_KEYWORDS - -_BASE = "http://export.arxiv.org/api/query" -_NS = { - "atom": "http://www.w3.org/2005/Atom", - "arxiv": "http://arxiv.org/schemas/atom", - "opensearch": "http://a9.com/-/spec/opensearch/1.1/", -} -_BATCH = 50 -# arXiv is primarily CS/STEM — only a few SC seed terms will yield results. -_SC_RELEVANT = { - "indigenous", "cultural heritage", "traditional knowledge", "oral tradition", - "ethnobotany", "decolonial", "african literature", "postcolonial", -} - - -class ArxivSpider(scrapy.Spider): - name = "arxiv_multi" - custom_settings = { - "DOWNLOAD_DELAY": 1.5, - "CONCURRENT_REQUESTS": 1, - "USER_AGENT": f"URAAS/1.0 (+SC discovery; mailto:{config.OPENALEX_MAILTO})", - } - - def __init__( - self, - institution="unilag", - target=50, - boost_special=True, - sc_only=False, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - self.target_limit = int(target) - _truthy = {"1", "true", "yes", "on"} - self.boost_special = ( - boost_special.lower() in _truthy - if isinstance(boost_special, str) - else bool(boost_special) - ) - self.sc_only = ( - sc_only.lower() in _truthy - if isinstance(sc_only, str) - else bool(sc_only) - ) - registry = get_registry() - self.institution_config = registry.get(institution) - if not self.institution_config: - raise ValueError(f"Institution '{institution}' not found in registry") - self.institution_name = self.institution_config.name - self.ror_id = self.institution_config.ror - self._accepted = 0 - self._seen_ids: set = set() - - def _build_url(self, query: str, start: int = 0) -> str: - params = {"search_query": query, "start": start, "max_results": _BATCH} - return f"{_BASE}?{urlencode(params)}" - - async def start(self): - seen_queries: set = set() - - if not self.sc_only: - # Primary institution wave — exact phrase in all fields - q = f'all:"{self.institution_name}"' - seen_queries.add(q) - yield scrapy.Request( - self._build_url(q, 0), - callback=self.parse, - meta={"query": q, "start": 0}, - ) - - if self.boost_special: - # Combine institution with SC seeds that are relevant to arXiv - inst_q = f'all:"{self.institution_name}"' - priority_seeds = [ - s for s in SC_SEED_KEYWORDS - if any(k in s.lower() for k in _SC_RELEVANT) - ][:8] - for seed in priority_seeds: - q = f'{inst_q} AND all:"{seed}"' - if q not in seen_queries: - seen_queries.add(q) - yield scrapy.Request( - self._build_url(q, 0), - callback=self.parse, - meta={"query": q, "start": 0}, - priority=5, - ) - - def parse(self, response): - if self._accepted >= self.target_limit: - return - - try: - root = ElementTree.fromstring(response.text) - except ElementTree.ParseError: - self.logger.warning(f"arXiv XML parse error: {response.url[:120]}") - return - - entries = root.findall("atom:entry", _NS) - for entry in entries: - if self._accepted >= self.target_limit: - return - - arxiv_id = (entry.findtext("atom:id", "", _NS) or "").strip() - if arxiv_id in self._seen_ids: - continue - self._seen_ids.add(arxiv_id) - - title = ( - (entry.findtext("atom:title", "", _NS) or "") - .strip() - .replace("\n", " ") - ) - abstract = ( - (entry.findtext("atom:summary", "", _NS) or "") - .strip() - .replace("\n", " ") - ) - if not title: - continue - - authors = [] - for a in entry.findall("atom:author", _NS): - name = (a.findtext("atom:name", "", _NS) or "").strip() - if name: - authors.append(name) - - doi = (entry.findtext("arxiv:doi", "", _NS) or "").strip() or None - pub_date = (entry.findtext("atom:published", "", _NS) or "")[:10] - - pdf_url = None - for link in entry.findall("atom:link", _NS): - if link.get("title") == "pdf": - pdf_url = link.get("href") - break - - # Extract affiliations from arxiv:affiliation elements - affiliations = [ - aff.text.strip() - for a in entry.findall("atom:author", _NS) - for aff in a.findall("arxiv:affiliation", _NS) - if aff.text - ] - raw_affiliation = "; ".join(affiliations) or self.institution_name - - self._accepted += 1 - yield { - "title": title, - "abstract": abstract, - "authors": authors, - "doi": doi, - "url": arxiv_id, - "pdf_url": pdf_url, - "publication_date": pub_date, - "source_repository": "arXiv", - "is_unilag_author": True, - "raw_affiliation": raw_affiliation, - "institution": self.institution_name, - "institution_ror": self.ror_id, - } - - # Pagination - total_text = root.findtext("opensearch:totalResults", "0", _NS) or "0" - total = int(total_text) if total_text.isdigit() else 0 - query = response.meta.get("query", "") - start = response.meta.get("start", 0) - next_start = start + _BATCH - if next_start < min(total, 500) and self._accepted < self.target_limit: - yield scrapy.Request( - self._build_url(query, next_start), - callback=self.parse, - meta={"query": query, "start": next_start}, - ) +""" +arXiv spider — uses the official Atom API (export.arxiv.org/api/query). + +The HTML search page at arxiv.org/search was fragile and layout-dependent. +The Atom API is stable, rate-limit-friendly (1 req/3s recommended), and +returns clean structured XML metadata. + +Docs: info.arxiv.org/help/api/basics.html +Rate limit: 3 req/s; we use 1.5s download delay to stay polite. +""" + +import os +import sys +from urllib.parse import urlencode +from xml.etree import ElementTree + +import scrapy + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) + +from uraas.config import config +from uraas.config.institutions import get_registry +from uraas.config.special_collections import SC_SEED_KEYWORDS + +_BASE = "http://export.arxiv.org/api/query" +_NS = { + "atom": "http://www.w3.org/2005/Atom", + "arxiv": "http://arxiv.org/schemas/atom", + "opensearch": "http://a9.com/-/spec/opensearch/1.1/", +} +_BATCH = 50 +# arXiv is primarily CS/STEM — only a few SC seed terms will yield results. +_SC_RELEVANT = { + "indigenous", "cultural heritage", "traditional knowledge", "oral tradition", + "ethnobotany", "decolonial", "african literature", "postcolonial", +} + + +class ArxivSpider(scrapy.Spider): + name = "arxiv_multi" + custom_settings = { + "DOWNLOAD_DELAY": 1.5, + "CONCURRENT_REQUESTS": 1, + "USER_AGENT": f"URAAS/1.0 (+SC discovery; mailto:{config.OPENALEX_MAILTO})", + } + + def __init__( + self, + institution="unilag", + target=50, + boost_special=True, + sc_only=False, + *args, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.target_limit = int(target) + _truthy = {"1", "true", "yes", "on"} + self.boost_special = ( + boost_special.lower() in _truthy + if isinstance(boost_special, str) + else bool(boost_special) + ) + self.sc_only = ( + sc_only.lower() in _truthy + if isinstance(sc_only, str) + else bool(sc_only) + ) + registry = get_registry() + self.institution_config = registry.get(institution) + if not self.institution_config: + raise ValueError(f"Institution '{institution}' not found in registry") + self.institution_name = self.institution_config.name + self.ror_id = self.institution_config.ror + self._accepted = 0 + self._seen_ids: set = set() + + def _build_url(self, query: str, start: int = 0) -> str: + params = {"search_query": query, "start": start, "max_results": _BATCH} + return f"{_BASE}?{urlencode(params)}" + + async def start(self): + seen_queries: set = set() + + if not self.sc_only: + # Primary institution wave — exact phrase in all fields + q = f'all:"{self.institution_name}"' + seen_queries.add(q) + yield scrapy.Request( + self._build_url(q, 0), + callback=self.parse, + meta={"query": q, "start": 0}, + ) + + if self.boost_special: + # Combine institution with SC seeds that are relevant to arXiv + inst_q = f'all:"{self.institution_name}"' + priority_seeds = [ + s for s in SC_SEED_KEYWORDS + if any(k in s.lower() for k in _SC_RELEVANT) + ][:8] + for seed in priority_seeds: + q = f'{inst_q} AND all:"{seed}"' + if q not in seen_queries: + seen_queries.add(q) + yield scrapy.Request( + self._build_url(q, 0), + callback=self.parse, + meta={"query": q, "start": 0}, + priority=5, + ) + + def parse(self, response): + if self._accepted >= self.target_limit: + return + + try: + root = ElementTree.fromstring(response.text) + except ElementTree.ParseError: + self.logger.warning(f"arXiv XML parse error: {response.url[:120]}") + return + + entries = root.findall("atom:entry", _NS) + for entry in entries: + if self._accepted >= self.target_limit: + return + + arxiv_id = (entry.findtext("atom:id", "", _NS) or "").strip() + if arxiv_id in self._seen_ids: + continue + self._seen_ids.add(arxiv_id) + + title = ( + (entry.findtext("atom:title", "", _NS) or "") + .strip() + .replace("\n", " ") + ) + abstract = ( + (entry.findtext("atom:summary", "", _NS) or "") + .strip() + .replace("\n", " ") + ) + if not title: + continue + + authors = [] + for a in entry.findall("atom:author", _NS): + name = (a.findtext("atom:name", "", _NS) or "").strip() + if name: + authors.append(name) + + doi = (entry.findtext("arxiv:doi", "", _NS) or "").strip() or None + pub_date = (entry.findtext("atom:published", "", _NS) or "")[:10] + + pdf_url = None + for link in entry.findall("atom:link", _NS): + if link.get("title") == "pdf": + pdf_url = link.get("href") + break + + # Extract affiliations from arxiv:affiliation elements + affiliations = [ + aff.text.strip() + for a in entry.findall("atom:author", _NS) + for aff in a.findall("arxiv:affiliation", _NS) + if aff.text + ] + raw_affiliation = "; ".join(affiliations) or self.institution_name + + self._accepted += 1 + yield { + "title": title, + "abstract": abstract, + "authors": authors, + "doi": doi, + "url": arxiv_id, + "pdf_url": pdf_url, + "publication_date": pub_date, + "source_repository": "arXiv", + "is_unilag_author": True, + "raw_affiliation": raw_affiliation, + "institution": self.institution_name, + "institution_ror": self.ror_id, + } + + # Pagination + total_text = root.findtext("opensearch:totalResults", "0", _NS) or "0" + total = int(total_text) if total_text.isdigit() else 0 + query = response.meta.get("query", "") + start = response.meta.get("start", 0) + next_start = start + _BATCH + if next_start < min(total, 500) and self._accepted < self.target_limit: + yield scrapy.Request( + self._build_url(query, next_start), + callback=self.parse, + meta={"query": query, "start": next_start}, + ) diff --git a/uraas/spiders/sources/core_spider.py b/uraas/spiders/sources/core_spider.py index 76230aee0424ca0f604dac359c9659287e8fd054..c1b9c2266d31551b21b6d6f548be06d102bdca48 100644 --- a/uraas/spiders/sources/core_spider.py +++ b/uraas/spiders/sources/core_spider.py @@ -1,117 +1,117 @@ -""" -CORE spider — queries core.ac.uk (250M+ open-access papers from repos worldwide). - -CORE aggregates content from thousands of institutional repositories and OA journals -globally, including many African university repositories. For URAAS it surfaces grey -literature and theses that are not yet indexed by OpenAlex or Crossref. - -Requires a free CORE API key: https://core.ac.uk/api-keys/register -Set CORE_API_KEY in .env. Without a key the spider logs a warning and exits. -""" - -import os -import sys -from urllib.parse import urlencode, quote - -import scrapy - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) - -from uraas.config import config -from uraas.config.institutions import get_registry -from uraas.config.special_collections import SC_SEED_KEYWORDS - -_CORE_BASE = "https://api.core.ac.uk/v3/search/works" - - -class CORESpider(scrapy.Spider): - name = "core" - custom_settings = { - "DOWNLOAD_DELAY": 1.0, - "CONCURRENT_REQUESTS": 1, - "USER_AGENT": f"URAAS/1.0 (+SC discovery; mailto:{config.OPENALEX_MAILTO})", - } - - def __init__(self, institution="unilag", target=50, boost_special=True, sc_only=False, *args, **kwargs): - super().__init__(*args, **kwargs) - self.target_limit = int(target) - _truthy = {"1", "true", "yes", "on"} - self.boost_special = boost_special.lower() in _truthy if isinstance(boost_special, str) else bool(boost_special) - self.sc_only = sc_only.lower() in _truthy if isinstance(sc_only, str) else bool(sc_only) - - self.api_key = getattr(config, "CORE_API_KEY", "") or os.environ.get("CORE_API_KEY", "") - if not self.api_key: - self.logger.warning( - "CORE_API_KEY not set — CORE spider will not run. " - "Get a free key at https://core.ac.uk/api-keys/register" - ) - - registry = get_registry() - self.institution_config = registry.get(institution) - if not self.institution_config: - raise ValueError(f"Institution '{institution}' not found") - self.institution_name = self.institution_config.name - self.ror_id = self.institution_config.ror - self._accepted = 0 - - def _headers(self): - return {"Authorization": f"Bearer {self.api_key}", "Accept": "application/json"} - - def _build_url(self, query: str, offset: int = 0) -> str: - params = {"q": query, "limit": 100, "offset": offset} - return f"{_CORE_BASE}?{urlencode(params)}" - - async def start(self): - if not self.api_key: - return - - if not self.sc_only: - url = self._build_url(f'"{self.institution_name}"') - yield scrapy.Request(url=url, headers=self._headers(), callback=self.parse, - meta={"wave": "general", "query": f'"{self.institution_name}"', "offset": 0}) - - if self.boost_special: - for seed in SC_SEED_KEYWORDS: - query = f'"{self.institution_name}" "{seed}"' - url = self._build_url(query) - yield scrapy.Request(url=url, headers=self._headers(), callback=self.parse, - meta={"wave": f"sc:{seed}", "query": query, "offset": 0}, priority=10) - - def parse(self, response): - if self._accepted >= self.target_limit: - return - data = response.json() - results = data.get("results", []) - wave = response.meta.get("wave", "general") - self.logger.info("[CORE:%s] received %d results", wave, len(results)) - - for r in results: - if self._accepted >= self.target_limit: - return - title = (r.get("title") or "").strip() - if not title: - continue - doi = (r.get("doi") or "").strip() - authors = [a.get("name", "") for a in (r.get("authors") or []) if a.get("name")] - abstract = (r.get("abstract") or "").strip() - pub_year = r.get("yearPublished") or "" - url_val = r.get("sourceFulltextUrls", [None])[0] or (f"https://doi.org/{doi}" if doi else "") - pdf_url = r.get("downloadUrl") or None - doc_type = r.get("documentType") or "" - - self._accepted += 1 - yield { - "title": title, "abstract": abstract, "authors": authors, "doi": doi, - "url": url_val, "pdf_url": pdf_url, "publication_date": str(pub_year), - "source_repository": "CORE", "is_unilag_author": True, - "raw_affiliation": self.institution_name, "institution": self.institution_name, - "institution_ror": self.ror_id, "content_type": doc_type, - } - - offset = response.meta.get("offset", 0) + 100 - total = data.get("totalHits", 0) - if offset < min(total, 500) and self._accepted < self.target_limit: - query = response.meta["query"] - next_url = self._build_url(query, offset) - yield scrapy.Request(url=next_url, headers=self._headers(), callback=self.parse, - meta={"wave": wave, "query": query, "offset": offset}) +""" +CORE spider — queries core.ac.uk (250M+ open-access papers from repos worldwide). + +CORE aggregates content from thousands of institutional repositories and OA journals +globally, including many African university repositories. For URAAS it surfaces grey +literature and theses that are not yet indexed by OpenAlex or Crossref. + +Requires a free CORE API key: https://core.ac.uk/api-keys/register +Set CORE_API_KEY in .env. Without a key the spider logs a warning and exits. +""" + +import os +import sys +from urllib.parse import urlencode, quote + +import scrapy + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) + +from uraas.config import config +from uraas.config.institutions import get_registry +from uraas.config.special_collections import SC_SEED_KEYWORDS + +_CORE_BASE = "https://api.core.ac.uk/v3/search/works" + + +class CORESpider(scrapy.Spider): + name = "core" + custom_settings = { + "DOWNLOAD_DELAY": 1.0, + "CONCURRENT_REQUESTS": 1, + "USER_AGENT": f"URAAS/1.0 (+SC discovery; mailto:{config.OPENALEX_MAILTO})", + } + + def __init__(self, institution="unilag", target=50, boost_special=True, sc_only=False, *args, **kwargs): + super().__init__(*args, **kwargs) + self.target_limit = int(target) + _truthy = {"1", "true", "yes", "on"} + self.boost_special = boost_special.lower() in _truthy if isinstance(boost_special, str) else bool(boost_special) + self.sc_only = sc_only.lower() in _truthy if isinstance(sc_only, str) else bool(sc_only) + + self.api_key = getattr(config, "CORE_API_KEY", "") or os.environ.get("CORE_API_KEY", "") + if not self.api_key: + self.logger.warning( + "CORE_API_KEY not set — CORE spider will not run. " + "Get a free key at https://core.ac.uk/api-keys/register" + ) + + registry = get_registry() + self.institution_config = registry.get(institution) + if not self.institution_config: + raise ValueError(f"Institution '{institution}' not found") + self.institution_name = self.institution_config.name + self.ror_id = self.institution_config.ror + self._accepted = 0 + + def _headers(self): + return {"Authorization": f"Bearer {self.api_key}", "Accept": "application/json"} + + def _build_url(self, query: str, offset: int = 0) -> str: + params = {"q": query, "limit": 100, "offset": offset} + return f"{_CORE_BASE}?{urlencode(params)}" + + async def start(self): + if not self.api_key: + return + + if not self.sc_only: + url = self._build_url(f'"{self.institution_name}"') + yield scrapy.Request(url=url, headers=self._headers(), callback=self.parse, + meta={"wave": "general", "query": f'"{self.institution_name}"', "offset": 0}) + + if self.boost_special: + for seed in SC_SEED_KEYWORDS: + query = f'"{self.institution_name}" "{seed}"' + url = self._build_url(query) + yield scrapy.Request(url=url, headers=self._headers(), callback=self.parse, + meta={"wave": f"sc:{seed}", "query": query, "offset": 0}, priority=10) + + def parse(self, response): + if self._accepted >= self.target_limit: + return + data = response.json() + results = data.get("results", []) + wave = response.meta.get("wave", "general") + self.logger.info("[CORE:%s] received %d results", wave, len(results)) + + for r in results: + if self._accepted >= self.target_limit: + return + title = (r.get("title") or "").strip() + if not title: + continue + doi = (r.get("doi") or "").strip() + authors = [a.get("name", "") for a in (r.get("authors") or []) if a.get("name")] + abstract = (r.get("abstract") or "").strip() + pub_year = r.get("yearPublished") or "" + url_val = r.get("sourceFulltextUrls", [None])[0] or (f"https://doi.org/{doi}" if doi else "") + pdf_url = r.get("downloadUrl") or None + doc_type = r.get("documentType") or "" + + self._accepted += 1 + yield { + "title": title, "abstract": abstract, "authors": authors, "doi": doi, + "url": url_val, "pdf_url": pdf_url, "publication_date": str(pub_year), + "source_repository": "CORE", "is_unilag_author": True, + "raw_affiliation": self.institution_name, "institution": self.institution_name, + "institution_ror": self.ror_id, "content_type": doc_type, + } + + offset = response.meta.get("offset", 0) + 100 + total = data.get("totalHits", 0) + if offset < min(total, 500) and self._accepted < self.target_limit: + query = response.meta["query"] + next_url = self._build_url(query, offset) + yield scrapy.Request(url=next_url, headers=self._headers(), callback=self.parse, + meta={"wave": wave, "query": query, "offset": offset}) diff --git a/uraas/spiders/sources/crossref_spider.py b/uraas/spiders/sources/crossref_spider.py index 1b6184751e6e1ceb0064076c85f236db177b5da1..16995b647adbba4615462d24ccae223dcb0998e8 100644 --- a/uraas/spiders/sources/crossref_spider.py +++ b/uraas/spiders/sources/crossref_spider.py @@ -1,187 +1,187 @@ -import os -import sys - -import scrapy - -# Add project root to path for imports -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) - -from uraas.config import config -from uraas.config.institutions import get_registry -from uraas.config.special_collections import SC_SEED_KEYWORDS - - -class CrossrefSpider(scrapy.Spider): - name = "crossref_multi" - custom_settings = { - "DOWNLOAD_DELAY": 1.0, - # Crossref etiquette: identify the bot + a contact so we land in the - # "polite pool" (https://www.crossref.org/documentation/retrieve-metadata/rest-api/). - "USER_AGENT": ( - f"URAAS/1.0 (+https://github.com; mailto:{config.OPENALEX_MAILTO})" - ), - } - - SELECT_FIELDS = "DOI,title,abstract,author,issued,URL,link" - - def __init__( - self, - institution="unilag", - target=20, - boost_special=True, - sc_only=False, - *args, - **kwargs, - ): - """ - boost_special: fan out extra Crossref queries seeded with SC keywords - (default True — heavy SC weight). - sc_only: skip the plain-affiliation query; crawl only SC-seeded fan-outs. - """ - super().__init__(*args, **kwargs) - self.target_limit = int(target) - self.boost_special = str(boost_special).lower() not in ( - "false", - "0", - "no", - "off", - ) - self.sc_only = str(sc_only).lower() in ("true", "1", "yes", "on") - - registry = get_registry() - self.institution_config = registry.get(institution) - - if not self.institution_config: - raise ValueError(f"Institution '{institution}' not found in registry") - - self.institution_name = self.institution_config.name - self.ror_id = self.institution_config.ror - self._accepted = 0 - self._rejected_aff = 0 - - self.logger.info(f"Initialized Crossref spider for {self.institution_name}") - self.logger.info( - f"ROR ID: {self.ror_id} | boost_special={self.boost_special} | sc_only={self.sc_only} | target={self.target_limit}" - ) - - def _build_url(self, *, query: str = "", offset: int = 0) -> str: - import urllib.parse - - encoded_name = urllib.parse.quote(self.institution_name) - q = f"&query={urllib.parse.quote(query)}" if query else "" - mailto = urllib.parse.quote(config.OPENALEX_MAILTO) - return ( - f"https://api.crossref.org/works" - f"?query.affiliation={encoded_name}" - f"{q}" - f"&select={self.SELECT_FIELDS}" - f"&rows=50&offset={offset}" - f"&mailto={mailto}" - ) - - async def start(self): - # Wave 1 — plain affiliation query - if not self.sc_only: - url = self._build_url() - self.logger.info(f"[ROR wave] {url}") - yield scrapy.Request( - url=url, callback=self.parse, meta={"wave": "ror", "query": "", "offset": 0} - ) - - # Wave 2 — one fan-out request per SC seed phrase, AND-ed with affiliation. - if self.boost_special: - for seed in SC_SEED_KEYWORDS: - url = self._build_url(query=seed) - self.logger.info(f"[SC wave seed={seed!r}] {url}") - yield scrapy.Request( - url=url, - callback=self.parse, - meta={"wave": f"sc:{seed}", "query": seed, "offset": 0}, - ) - - def parse(self, response): - if self._accepted >= self.target_limit: - return - - data = response.json() - items = data.get("message", {}).get("items", []) - wave = response.meta.get("wave", "ror") - self.logger.info(f"[{wave}] received {len(items)} works") - - for work in items: - if self._accepted >= self.target_limit: - break - - title = work.get("title", [""])[0] if work.get("title") else "" - if not title: - continue - - doi = work.get("DOI", "") - url = work.get("URL", "") - abstract = work.get("abstract", "") - - # Affiliation check using Crossref author.affiliation field. - # Crossref returns affiliation data when available; if empty we - # accept the paper (institution filter on the query is still active). - raw_affs = [] - authors = [] - for author in work.get("author", []): - given = author.get("given", "") - family = author.get("family", "") - if given or family: - authors.append(f"{given} {family}".strip()) - for aff in author.get("affiliation", []): - name = aff.get("name", "") - if name: - raw_affs.append(name) - - if raw_affs: - aff_text = " ".join(raw_affs) - if not self.institution_config.matches_affiliation(aff_text): - self._rejected_aff += 1 - self.logger.debug(f"Crossref aff FAIL: {title[:60]}") - continue - - # Try to find a PDF link in the 'link' array if open access - pdf_url = None - for link in work.get("link", []): - if link.get("content-type") == "application/pdf": - pdf_url = link.get("URL") - break - - self._accepted += 1 - yield { - "title": title, - "authors": authors, - "abstract": abstract, - "doi": doi, - "url": url, - "pdf_url": pdf_url, - "source_repository": "Crossref", - "is_unilag_author": True, - "raw_affiliation": " | ".join(raw_affs) if raw_affs else self.institution_name, - "institution": self.institution_name, - "institution_ror": self.ror_id, - } - - # Deep pagination — stop when target reached or no more results. - offset = response.meta.get("offset", 0) + 50 - if ( - items - and offset < 500 - and self._accepted < self.target_limit - ): - wave = response.meta.get("wave", "ror") - query = response.meta.get("query", "") - next_url = self._build_url(query=query, offset=offset) - yield scrapy.Request( - url=next_url, - callback=self.parse, - meta={"wave": wave, "query": query, "offset": offset}, - ) - - def closed(self, reason): - self.logger.info( - f"Crossref spider closed | {self.institution_name} | " - f"accepted={self._accepted} | rejected_aff={self._rejected_aff} | reason={reason}" - ) +import os +import sys + +import scrapy + +# Add project root to path for imports +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) + +from uraas.config import config +from uraas.config.institutions import get_registry +from uraas.config.special_collections import SC_SEED_KEYWORDS + + +class CrossrefSpider(scrapy.Spider): + name = "crossref_multi" + custom_settings = { + "DOWNLOAD_DELAY": 1.0, + # Crossref etiquette: identify the bot + a contact so we land in the + # "polite pool" (https://www.crossref.org/documentation/retrieve-metadata/rest-api/). + "USER_AGENT": ( + f"URAAS/1.0 (+https://github.com; mailto:{config.OPENALEX_MAILTO})" + ), + } + + SELECT_FIELDS = "DOI,title,abstract,author,issued,URL,link" + + def __init__( + self, + institution="unilag", + target=20, + boost_special=True, + sc_only=False, + *args, + **kwargs, + ): + """ + boost_special: fan out extra Crossref queries seeded with SC keywords + (default True — heavy SC weight). + sc_only: skip the plain-affiliation query; crawl only SC-seeded fan-outs. + """ + super().__init__(*args, **kwargs) + self.target_limit = int(target) + self.boost_special = str(boost_special).lower() not in ( + "false", + "0", + "no", + "off", + ) + self.sc_only = str(sc_only).lower() in ("true", "1", "yes", "on") + + registry = get_registry() + self.institution_config = registry.get(institution) + + if not self.institution_config: + raise ValueError(f"Institution '{institution}' not found in registry") + + self.institution_name = self.institution_config.name + self.ror_id = self.institution_config.ror + self._accepted = 0 + self._rejected_aff = 0 + + self.logger.info(f"Initialized Crossref spider for {self.institution_name}") + self.logger.info( + f"ROR ID: {self.ror_id} | boost_special={self.boost_special} | sc_only={self.sc_only} | target={self.target_limit}" + ) + + def _build_url(self, *, query: str = "", offset: int = 0) -> str: + import urllib.parse + + encoded_name = urllib.parse.quote(self.institution_name) + q = f"&query={urllib.parse.quote(query)}" if query else "" + mailto = urllib.parse.quote(config.OPENALEX_MAILTO) + return ( + f"https://api.crossref.org/works" + f"?query.affiliation={encoded_name}" + f"{q}" + f"&select={self.SELECT_FIELDS}" + f"&rows=50&offset={offset}" + f"&mailto={mailto}" + ) + + async def start(self): + # Wave 1 — plain affiliation query + if not self.sc_only: + url = self._build_url() + self.logger.info(f"[ROR wave] {url}") + yield scrapy.Request( + url=url, callback=self.parse, meta={"wave": "ror", "query": "", "offset": 0} + ) + + # Wave 2 — one fan-out request per SC seed phrase, AND-ed with affiliation. + if self.boost_special: + for seed in SC_SEED_KEYWORDS: + url = self._build_url(query=seed) + self.logger.info(f"[SC wave seed={seed!r}] {url}") + yield scrapy.Request( + url=url, + callback=self.parse, + meta={"wave": f"sc:{seed}", "query": seed, "offset": 0}, + ) + + def parse(self, response): + if self._accepted >= self.target_limit: + return + + data = response.json() + items = data.get("message", {}).get("items", []) + wave = response.meta.get("wave", "ror") + self.logger.info(f"[{wave}] received {len(items)} works") + + for work in items: + if self._accepted >= self.target_limit: + break + + title = work.get("title", [""])[0] if work.get("title") else "" + if not title: + continue + + doi = work.get("DOI", "") + url = work.get("URL", "") + abstract = work.get("abstract", "") + + # Affiliation check using Crossref author.affiliation field. + # Crossref returns affiliation data when available; if empty we + # accept the paper (institution filter on the query is still active). + raw_affs = [] + authors = [] + for author in work.get("author", []): + given = author.get("given", "") + family = author.get("family", "") + if given or family: + authors.append(f"{given} {family}".strip()) + for aff in author.get("affiliation", []): + name = aff.get("name", "") + if name: + raw_affs.append(name) + + if raw_affs: + aff_text = " ".join(raw_affs) + if not self.institution_config.matches_affiliation(aff_text): + self._rejected_aff += 1 + self.logger.debug(f"Crossref aff FAIL: {title[:60]}") + continue + + # Try to find a PDF link in the 'link' array if open access + pdf_url = None + for link in work.get("link", []): + if link.get("content-type") == "application/pdf": + pdf_url = link.get("URL") + break + + self._accepted += 1 + yield { + "title": title, + "authors": authors, + "abstract": abstract, + "doi": doi, + "url": url, + "pdf_url": pdf_url, + "source_repository": "Crossref", + "is_unilag_author": True, + "raw_affiliation": " | ".join(raw_affs) if raw_affs else self.institution_name, + "institution": self.institution_name, + "institution_ror": self.ror_id, + } + + # Deep pagination — stop when target reached or no more results. + offset = response.meta.get("offset", 0) + 50 + if ( + items + and offset < 500 + and self._accepted < self.target_limit + ): + wave = response.meta.get("wave", "ror") + query = response.meta.get("query", "") + next_url = self._build_url(query=query, offset=offset) + yield scrapy.Request( + url=next_url, + callback=self.parse, + meta={"wave": wave, "query": query, "offset": offset}, + ) + + def closed(self, reason): + self.logger.info( + f"Crossref spider closed | {self.institution_name} | " + f"accepted={self._accepted} | rejected_aff={self._rejected_aff} | reason={reason}" + ) diff --git a/uraas/spiders/sources/doaj_spider.py b/uraas/spiders/sources/doaj_spider.py index 9f64464f1a8a3f013835b6aaab22cbf075d70697..ddac62f81fc70028e5f1cbc661498990c33647ac 100644 --- a/uraas/spiders/sources/doaj_spider.py +++ b/uraas/spiders/sources/doaj_spider.py @@ -1,173 +1,173 @@ -""" -DOAJ spider — Directory of Open Access Journals API v3. - -DOAJ indexes 20,000+ OA journals and is particularly strong for African -humanities, social sciences, and indigenous knowledge journals. Many -AJOL-listed journals appear in DOAJ. - -Free API, no key required. Docs: doaj.org/api/v3/docs -Rate limit: 60 req/min — we use 1.5s delay. -""" - -import os -import sys -from urllib.parse import quote, urlencode - -import scrapy - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) - -from uraas.config import config -from uraas.config.institutions import get_registry -from uraas.config.special_collections import SC_SEED_KEYWORDS - -_BASE = "https://doaj.org/api/v3/search/articles" -_SC_AFFINITY = { - "indigenous", "cultural", "traditional", "oral", "heritage", - "african", "yoruba", "igbo", "hausa", "postcolonial", -} - - -class DOAJSpider(scrapy.Spider): - name = "doaj" - custom_settings = { - "DOWNLOAD_DELAY": 1.5, - "CONCURRENT_REQUESTS": 1, - "USER_AGENT": f"URAAS/1.0 (+SC discovery; mailto:{config.OPENALEX_MAILTO})", - } - - def __init__( - self, institution="unilag", target=50, boost_special=True, sc_only=False, - *args, **kwargs, - ): - super().__init__(*args, **kwargs) - self.target_limit = int(target) - _truthy = {"1", "true", "yes", "on"} - self.boost_special = ( - boost_special.lower() in _truthy - if isinstance(boost_special, str) - else bool(boost_special) - ) - self.sc_only = ( - sc_only.lower() in _truthy - if isinstance(sc_only, str) - else bool(sc_only) - ) - registry = get_registry() - self.institution_config = registry.get(institution) - if not self.institution_config: - raise ValueError(f"Institution '{institution}' not found") - self.institution_name = self.institution_config.name - self.ror_id = self.institution_config.ror - self._accepted = 0 - - def _build_url(self, query: str, page: int = 1) -> str: - # DOAJ v3: query is a path segment, must be URL-encoded. - # Avoid field:value syntax — DOAJ rejects compound field queries (400). - # Use plain free-text so the engine handles field matching internally. - encoded_q = quote(query, safe="") - params = urlencode({"pageSize": 50, "page": page, "sort": "relevance"}) - return f"{_BASE}/{encoded_q}?{params}" - - async def start(self): - seen: set = set() - - if not self.sc_only: - # Simple affiliation free-text — no field: syntax - q = f'"{self.institution_name}"' - seen.add(q) - yield scrapy.Request( - self._build_url(q), callback=self.parse, - meta={"query": q, "page": 1}, - ) - - if self.boost_special: - priority_seeds = [ - s for s in SC_SEED_KEYWORDS - if any(k in s.lower() for k in _SC_AFFINITY) - ][:6] - for seed in priority_seeds: - # Plain conjunction — no field: prefix that causes 400 - q = f'"{self.institution_name}" "{seed}"' - if q not in seen: - seen.add(q) - yield scrapy.Request( - self._build_url(q), callback=self.parse, - meta={"query": q, "page": 1}, priority=5, - ) - - def parse(self, response): - if self._accepted >= self.target_limit: - return - try: - data = response.json() - except Exception: - self.logger.warning(f"DOAJ JSON parse error: {response.url[:120]}") - return - - inst_lower = self.institution_name.lower() - - for r in data.get("results", []): - if self._accepted >= self.target_limit: - return - bib = r.get("bibjson", {}) - title = (bib.get("title") or "").strip() - if not title: - continue - - abstract = (bib.get("abstract") or "").strip() - - # Verify institution affiliation via author records - affil_text = " ".join( - (a.get("affiliation") or "") for a in bib.get("author", []) - ).lower() - if inst_lower not in affil_text and inst_lower not in title.lower() and inst_lower not in abstract.lower(): - continue - - doi = "" - for ident in bib.get("identifier", []): - if ident.get("type") == "doi": - doi = (ident.get("id") or "").strip() - break - - url_val = "" - for link in bib.get("link", []): - if link.get("type") in ("fulltext", "article"): - url_val = link.get("url", "") - break - if not url_val and doi: - url_val = f"https://doi.org/{doi}" - - authors = [ - (a.get("name") or "").strip() - for a in bib.get("author", []) - if a.get("name") - ] - - year = str(bib.get("year") or "") - journal = (bib.get("journal") or {}).get("title", "") - - self._accepted += 1 - yield { - "title": title, - "abstract": abstract, - "authors": authors, - "doi": doi or None, - "url": url_val or None, - "pdf_url": None, - "publication_date": year, - "source_repository": f"DOAJ/{journal}" if journal else "DOAJ", - "is_unilag_author": True, - "raw_affiliation": self.institution_name, - "institution": self.institution_name, - "institution_ror": self.ror_id, - } - - total = int(data.get("total") or 0) - page = response.meta.get("page", 1) - query = response.meta.get("query", "") - if page * 50 < min(total, 500) and self._accepted < self.target_limit: - yield scrapy.Request( - self._build_url(query, page + 1), callback=self.parse, - meta={"query": query, "page": page + 1}, - ) +""" +DOAJ spider — Directory of Open Access Journals API v3. + +DOAJ indexes 20,000+ OA journals and is particularly strong for African +humanities, social sciences, and indigenous knowledge journals. Many +AJOL-listed journals appear in DOAJ. + +Free API, no key required. Docs: doaj.org/api/v3/docs +Rate limit: 60 req/min — we use 1.5s delay. +""" + +import os +import sys +from urllib.parse import quote, urlencode + +import scrapy + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) + +from uraas.config import config +from uraas.config.institutions import get_registry +from uraas.config.special_collections import SC_SEED_KEYWORDS + +_BASE = "https://doaj.org/api/v3/search/articles" +_SC_AFFINITY = { + "indigenous", "cultural", "traditional", "oral", "heritage", + "african", "yoruba", "igbo", "hausa", "postcolonial", +} + + +class DOAJSpider(scrapy.Spider): + name = "doaj" + custom_settings = { + "DOWNLOAD_DELAY": 1.5, + "CONCURRENT_REQUESTS": 1, + "USER_AGENT": f"URAAS/1.0 (+SC discovery; mailto:{config.OPENALEX_MAILTO})", + } + + def __init__( + self, institution="unilag", target=50, boost_special=True, sc_only=False, + *args, **kwargs, + ): + super().__init__(*args, **kwargs) + self.target_limit = int(target) + _truthy = {"1", "true", "yes", "on"} + self.boost_special = ( + boost_special.lower() in _truthy + if isinstance(boost_special, str) + else bool(boost_special) + ) + self.sc_only = ( + sc_only.lower() in _truthy + if isinstance(sc_only, str) + else bool(sc_only) + ) + registry = get_registry() + self.institution_config = registry.get(institution) + if not self.institution_config: + raise ValueError(f"Institution '{institution}' not found") + self.institution_name = self.institution_config.name + self.ror_id = self.institution_config.ror + self._accepted = 0 + + def _build_url(self, query: str, page: int = 1) -> str: + # DOAJ v3: query is a path segment, must be URL-encoded. + # Avoid field:value syntax — DOAJ rejects compound field queries (400). + # Use plain free-text so the engine handles field matching internally. + encoded_q = quote(query, safe="") + params = urlencode({"pageSize": 50, "page": page, "sort": "relevance"}) + return f"{_BASE}/{encoded_q}?{params}" + + async def start(self): + seen: set = set() + + if not self.sc_only: + # Simple affiliation free-text — no field: syntax + q = f'"{self.institution_name}"' + seen.add(q) + yield scrapy.Request( + self._build_url(q), callback=self.parse, + meta={"query": q, "page": 1}, + ) + + if self.boost_special: + priority_seeds = [ + s for s in SC_SEED_KEYWORDS + if any(k in s.lower() for k in _SC_AFFINITY) + ][:6] + for seed in priority_seeds: + # Plain conjunction — no field: prefix that causes 400 + q = f'"{self.institution_name}" "{seed}"' + if q not in seen: + seen.add(q) + yield scrapy.Request( + self._build_url(q), callback=self.parse, + meta={"query": q, "page": 1}, priority=5, + ) + + def parse(self, response): + if self._accepted >= self.target_limit: + return + try: + data = response.json() + except Exception: + self.logger.warning(f"DOAJ JSON parse error: {response.url[:120]}") + return + + inst_lower = self.institution_name.lower() + + for r in data.get("results", []): + if self._accepted >= self.target_limit: + return + bib = r.get("bibjson", {}) + title = (bib.get("title") or "").strip() + if not title: + continue + + abstract = (bib.get("abstract") or "").strip() + + # Verify institution affiliation via author records + affil_text = " ".join( + (a.get("affiliation") or "") for a in bib.get("author", []) + ).lower() + if inst_lower not in affil_text and inst_lower not in title.lower() and inst_lower not in abstract.lower(): + continue + + doi = "" + for ident in bib.get("identifier", []): + if ident.get("type") == "doi": + doi = (ident.get("id") or "").strip() + break + + url_val = "" + for link in bib.get("link", []): + if link.get("type") in ("fulltext", "article"): + url_val = link.get("url", "") + break + if not url_val and doi: + url_val = f"https://doi.org/{doi}" + + authors = [ + (a.get("name") or "").strip() + for a in bib.get("author", []) + if a.get("name") + ] + + year = str(bib.get("year") or "") + journal = (bib.get("journal") or {}).get("title", "") + + self._accepted += 1 + yield { + "title": title, + "abstract": abstract, + "authors": authors, + "doi": doi or None, + "url": url_val or None, + "pdf_url": None, + "publication_date": year, + "source_repository": f"DOAJ/{journal}" if journal else "DOAJ", + "is_unilag_author": True, + "raw_affiliation": self.institution_name, + "institution": self.institution_name, + "institution_ror": self.ror_id, + } + + total = int(data.get("total") or 0) + page = response.meta.get("page", 1) + query = response.meta.get("query", "") + if page * 50 < min(total, 500) and self._accepted < self.target_limit: + yield scrapy.Request( + self._build_url(query, page + 1), callback=self.parse, + meta={"query": query, "page": page + 1}, + ) diff --git a/uraas/spiders/sources/europepmc_spider.py b/uraas/spiders/sources/europepmc_spider.py index 7b4049be4f9c3988eae031a7ae4405c667f2d8d2..529a77315a660e495ac109b7b8e424c92d07d4a6 100644 --- a/uraas/spiders/sources/europepmc_spider.py +++ b/uraas/spiders/sources/europepmc_spider.py @@ -1,212 +1,212 @@ -""" -EuropePMC spider — queries the Europe PubMed Central REST API. - -EuropePMC (europepmc.org) aggregates life-sciences and biomedical literature from -PubMed, PMC, WHO, and many other sources. For URAAS Special Collections it is -particularly valuable for the **Indigenous Knowledge** and **Ethnobotany** subcategories -— traditional plant medicine, ethno-pharmacology, and indigenous health practices -feature heavily in UNILAG research and are well-indexed here. - -The API is free, no key required. -""" - -import os -import sys -from urllib.parse import urlencode, quote - -import scrapy - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) - -from uraas.config import config -from uraas.config.institutions import get_registry -from uraas.config.special_collections import SC_SEED_KEYWORDS - -_EPMC_BASE = "https://www.ebi.ac.uk/europepmc/webservices/rest/search" -_PAGE_SIZE = 100 - - -class EuropePMCSpider(scrapy.Spider): - name = "europepmc" - custom_settings = { - "DOWNLOAD_DELAY": 1.0, - "CONCURRENT_REQUESTS": 1, - "AUTOTHROTTLE_ENABLED": True, - "USER_AGENT": ( - f"URAAS/1.0 (+read-only SC discovery; mailto:{config.OPENALEX_MAILTO})" - ), - } - - def __init__( - self, - institution="unilag", - target=50, - boost_special=True, - sc_only=False, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - self.target_limit = int(target) - _truthy = {"1", "true", "yes", "on"} - self.boost_special = ( - boost_special.lower() in _truthy - if isinstance(boost_special, str) - else bool(boost_special) - ) - self.sc_only = ( - sc_only.lower() in _truthy - if isinstance(sc_only, str) - else bool(sc_only) - ) - registry = get_registry() - self.institution_config = registry.get(institution) - if not self.institution_config: - raise ValueError(f"Institution '{institution}' not found in registry") - self.institution_name = self.institution_config.name - self.ror_id = self.institution_config.ror - self._accepted = 0 - - # Use primary affiliation patterns to widen EPMC search - self._affiliation_patterns = self.institution_config.affiliation_patterns or [self.institution_name] - - self.logger.info( - "EuropePMC spider | %s | boost_special=%s | target=%d", - self.institution_name, - self.boost_special, - self.target_limit, - ) - - def _build_url(self, query: str, cursor_mark: str = "*") -> str: - params = { - "query": query, - "format": "json", - "pageSize": _PAGE_SIZE, - "resultType": "core", - "cursorMark": cursor_mark, - } - return f"{_EPMC_BASE}?{urlencode(params)}" - - def _affil_query(self, seed: str = "") -> str: - # EPMC affiliation filter — OR across all known institution name patterns - affil_parts = " OR ".join( - f'AFFILIATION:"{p}"' for p in self._affiliation_patterns[:3] - ) - affil = f"({affil_parts})" - if seed: - return f'{affil} AND ("{seed}")' - return affil - - async def start(self): - if not self.sc_only: - url = self._build_url(self._affil_query()) - yield scrapy.Request( - url=url, - callback=self.parse, - meta={"wave": "general", "query": self._affil_query(), "cursor": "*"}, - ) - - if self.boost_special: - # Only run the most SC-relevant seeds for EPMC — ethnobotany, traditional - # knowledge, and cultural heritage are where EPMC adds the most value. - priority_seeds = [ - s for s in SC_SEED_KEYWORDS - if any(k in s.lower() for k in ( - "indigenous", "traditional", "ethnobotany", "cultural", "oral", - "decolonial", "ubuntu", "ethnomusicology", - )) - ] - for seed in priority_seeds: - query = self._affil_query(seed) - url = self._build_url(query) - yield scrapy.Request( - url=url, - callback=self.parse, - meta={"wave": f"sc:{seed}", "query": query, "cursor": "*"}, - priority=10, - ) - - def parse(self, response): - if self._accepted >= self.target_limit: - return - - data = response.json() - results = data.get("resultList", {}).get("result", []) - wave = response.meta.get("wave", "general") - self.logger.info("[EPMC:%s] received %d results", wave, len(results)) - - for r in results: - if self._accepted >= self.target_limit: - return - - title = (r.get("title") or "").strip().rstrip(".") - if not title: - continue - - abstract = (r.get("abstractText") or "").strip() - doi = (r.get("doi") or "").strip() - pmid = r.get("pmid") or "" - pub_date = r.get("firstPublicationDate") or r.get("pubYear") or "" - doc_type = r.get("pubType") or r.get("source") or "" - - url_val = ( - f"https://doi.org/{doi}" if doi - else (f"https://europepmc.org/article/med/{pmid}" if pmid else "") - ) - pdf_url = None - if r.get("isOpenAccess") == "Y" and r.get("fullTextUrlList"): - for ft in (r.get("fullTextUrlList", {}).get("fullTextUrl") or []): - if ft.get("documentStyle") == "pdf": - pdf_url = ft.get("url") - break - - authors_raw = r.get("authorList", {}).get("author") or [] - authors = [ - f"{a.get('firstName','')} {a.get('lastName','')}".strip() - for a in authors_raw - if a.get("lastName") - ] - - self._accepted += 1 - yield { - "title": title, - "abstract": abstract, - "authors": authors, - "doi": doi, - "url": url_val, - "pdf_url": pdf_url, - "publication_date": pub_date, - "source_repository": "EuropePMC", - "is_unilag_author": True, - "raw_affiliation": self.institution_name, - "institution": self.institution_name, - "institution_ror": self.ror_id, - "content_type": doc_type, - } - - # Cursor-based pagination - next_cursor = data.get("nextCursorMark") - hit_count = data.get("hitCount", 0) - if ( - next_cursor - and next_cursor != response.meta.get("cursor") - and results - and self._accepted < self.target_limit - and hit_count > _PAGE_SIZE - ): - query = response.meta["query"] - wave = response.meta["wave"] - next_url = self._build_url(query, next_cursor) - yield scrapy.Request( - url=next_url, - callback=self.parse, - meta={"wave": wave, "query": query, "cursor": next_cursor}, - ) - - def closed(self, reason): - self.logger.info( - "EuropePMC spider closed | %s | accepted=%d | reason=%s", - self.institution_name, - self._accepted, - reason, - ) +""" +EuropePMC spider — queries the Europe PubMed Central REST API. + +EuropePMC (europepmc.org) aggregates life-sciences and biomedical literature from +PubMed, PMC, WHO, and many other sources. For URAAS Special Collections it is +particularly valuable for the **Indigenous Knowledge** and **Ethnobotany** subcategories +— traditional plant medicine, ethno-pharmacology, and indigenous health practices +feature heavily in UNILAG research and are well-indexed here. + +The API is free, no key required. +""" + +import os +import sys +from urllib.parse import urlencode, quote + +import scrapy + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) + +from uraas.config import config +from uraas.config.institutions import get_registry +from uraas.config.special_collections import SC_SEED_KEYWORDS + +_EPMC_BASE = "https://www.ebi.ac.uk/europepmc/webservices/rest/search" +_PAGE_SIZE = 100 + + +class EuropePMCSpider(scrapy.Spider): + name = "europepmc" + custom_settings = { + "DOWNLOAD_DELAY": 1.0, + "CONCURRENT_REQUESTS": 1, + "AUTOTHROTTLE_ENABLED": True, + "USER_AGENT": ( + f"URAAS/1.0 (+read-only SC discovery; mailto:{config.OPENALEX_MAILTO})" + ), + } + + def __init__( + self, + institution="unilag", + target=50, + boost_special=True, + sc_only=False, + *args, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.target_limit = int(target) + _truthy = {"1", "true", "yes", "on"} + self.boost_special = ( + boost_special.lower() in _truthy + if isinstance(boost_special, str) + else bool(boost_special) + ) + self.sc_only = ( + sc_only.lower() in _truthy + if isinstance(sc_only, str) + else bool(sc_only) + ) + registry = get_registry() + self.institution_config = registry.get(institution) + if not self.institution_config: + raise ValueError(f"Institution '{institution}' not found in registry") + self.institution_name = self.institution_config.name + self.ror_id = self.institution_config.ror + self._accepted = 0 + + # Use primary affiliation patterns to widen EPMC search + self._affiliation_patterns = self.institution_config.affiliation_patterns or [self.institution_name] + + self.logger.info( + "EuropePMC spider | %s | boost_special=%s | target=%d", + self.institution_name, + self.boost_special, + self.target_limit, + ) + + def _build_url(self, query: str, cursor_mark: str = "*") -> str: + params = { + "query": query, + "format": "json", + "pageSize": _PAGE_SIZE, + "resultType": "core", + "cursorMark": cursor_mark, + } + return f"{_EPMC_BASE}?{urlencode(params)}" + + def _affil_query(self, seed: str = "") -> str: + # EPMC affiliation filter — OR across all known institution name patterns + affil_parts = " OR ".join( + f'AFFILIATION:"{p}"' for p in self._affiliation_patterns[:3] + ) + affil = f"({affil_parts})" + if seed: + return f'{affil} AND ("{seed}")' + return affil + + async def start(self): + if not self.sc_only: + url = self._build_url(self._affil_query()) + yield scrapy.Request( + url=url, + callback=self.parse, + meta={"wave": "general", "query": self._affil_query(), "cursor": "*"}, + ) + + if self.boost_special: + # Only run the most SC-relevant seeds for EPMC — ethnobotany, traditional + # knowledge, and cultural heritage are where EPMC adds the most value. + priority_seeds = [ + s for s in SC_SEED_KEYWORDS + if any(k in s.lower() for k in ( + "indigenous", "traditional", "ethnobotany", "cultural", "oral", + "decolonial", "ubuntu", "ethnomusicology", + )) + ] + for seed in priority_seeds: + query = self._affil_query(seed) + url = self._build_url(query) + yield scrapy.Request( + url=url, + callback=self.parse, + meta={"wave": f"sc:{seed}", "query": query, "cursor": "*"}, + priority=10, + ) + + def parse(self, response): + if self._accepted >= self.target_limit: + return + + data = response.json() + results = data.get("resultList", {}).get("result", []) + wave = response.meta.get("wave", "general") + self.logger.info("[EPMC:%s] received %d results", wave, len(results)) + + for r in results: + if self._accepted >= self.target_limit: + return + + title = (r.get("title") or "").strip().rstrip(".") + if not title: + continue + + abstract = (r.get("abstractText") or "").strip() + doi = (r.get("doi") or "").strip() + pmid = r.get("pmid") or "" + pub_date = r.get("firstPublicationDate") or r.get("pubYear") or "" + doc_type = r.get("pubType") or r.get("source") or "" + + url_val = ( + f"https://doi.org/{doi}" if doi + else (f"https://europepmc.org/article/med/{pmid}" if pmid else "") + ) + pdf_url = None + if r.get("isOpenAccess") == "Y" and r.get("fullTextUrlList"): + for ft in (r.get("fullTextUrlList", {}).get("fullTextUrl") or []): + if ft.get("documentStyle") == "pdf": + pdf_url = ft.get("url") + break + + authors_raw = r.get("authorList", {}).get("author") or [] + authors = [ + f"{a.get('firstName','')} {a.get('lastName','')}".strip() + for a in authors_raw + if a.get("lastName") + ] + + self._accepted += 1 + yield { + "title": title, + "abstract": abstract, + "authors": authors, + "doi": doi, + "url": url_val, + "pdf_url": pdf_url, + "publication_date": pub_date, + "source_repository": "EuropePMC", + "is_unilag_author": True, + "raw_affiliation": self.institution_name, + "institution": self.institution_name, + "institution_ror": self.ror_id, + "content_type": doc_type, + } + + # Cursor-based pagination + next_cursor = data.get("nextCursorMark") + hit_count = data.get("hitCount", 0) + if ( + next_cursor + and next_cursor != response.meta.get("cursor") + and results + and self._accepted < self.target_limit + and hit_count > _PAGE_SIZE + ): + query = response.meta["query"] + wave = response.meta["wave"] + next_url = self._build_url(query, next_cursor) + yield scrapy.Request( + url=next_url, + callback=self.parse, + meta={"wave": wave, "query": query, "cursor": next_cursor}, + ) + + def closed(self, reason): + self.logger.info( + "EuropePMC spider closed | %s | accepted=%d | reason=%s", + self.institution_name, + self._accepted, + reason, + ) diff --git a/uraas/spiders/sources/faculty_directory_spider.py b/uraas/spiders/sources/faculty_directory_spider.py index 4b062e93ffa03a7276094b9704d347cbf6643950..80ef5b138f08aab53d6c74059b56947b41bef2b2 100644 --- a/uraas/spiders/sources/faculty_directory_spider.py +++ b/uraas/spiders/sources/faculty_directory_spider.py @@ -1,200 +1,200 @@ -""" -Faculty Directory Spider — crawls UNILAG staff pages and extracts -name + faculty + department for each staff member. -Saves to data/unilag_staff_detailed.json for accurate classification. -""" - -import json -import os - -import scrapy -from scrapy.http import Request - -STAFF_CACHE = os.path.join( - os.path.dirname(__file__), "..", "..", "..", "data", "unilag_staff.json" -) -DETAILED_CACHE = os.path.join( - os.path.dirname(__file__), "..", "..", "..", "data", "unilag_staff_detailed.json" -) - -# Known UNILAG faculty URLs — direct seeds so we don't rely on the homepage nav -FACULTY_SEEDS = [ - ("College of Medicine", "https://medicine.unilag.edu.ng/academic-staff/"), - ("Faculty of Engineering", "https://engineering.unilag.edu.ng/staff/"), - ("Faculty of Science", "https://science.unilag.edu.ng/staff/"), - ("Faculty of Arts", "https://arts.unilag.edu.ng/staff/"), - ("Faculty of Social Sciences", "https://socialsciences.unilag.edu.ng/staff/"), - ("Faculty of Law", "https://law.unilag.edu.ng/staff/"), - ("Faculty of Education", "https://education.unilag.edu.ng/staff/"), - ( - "Faculty of Environmental Sciences", - "https://environmentalsciences.unilag.edu.ng/staff/", - ), - ("Faculty of Management Sciences", "https://management.unilag.edu.ng/staff/"), - ("Faculty of Pharmacy", "https://pharmacy.unilag.edu.ng/staff/"), - ("Faculty of Dental Sciences", "https://dentistry.unilag.edu.ng/staff/"), - ("Faculty of Basic Medical Sciences", "https://basicmedical.unilag.edu.ng/staff/"), -] - -NOISE = { - "click", - "view", - "department", - "faculty", - "university", - "college", - "profile", - "contact", - "email", - "phone", - "office", - "research", - "publications", - "more", - "read", - "about", - "staff", - "list", - "home", - "menu", - "search", - "login", - "logout", -} - - -class FacultyDirectorySpider(scrapy.Spider): - name = "unilag_faculty_directory" - custom_settings = { - "DOWNLOAD_DELAY": 1.5, - # Honor robots.txt — the institution's machine-readable usage signal. - "ROBOTSTXT_OBEY": True, - "RETRY_ENABLED": True, - "RETRY_TIMES": 2, - "LOG_LEVEL": "WARNING", - } - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.staff_records = [] # list of {name, faculty, department} - self.staff_names = [] # flat list for backwards compat - - def start_requests(self): - headers = { - "User-Agent": "Mozilla/5.0 (compatible; URAAS/1.0; mailto:library@unilag.edu.ng)" - } - for faculty_name, url in FACULTY_SEEDS: - yield Request( - url, - headers=headers, - callback=self.parse_staff_page, - errback=self.handle_error, - meta={"faculty": faculty_name, "department": None}, - ) - - def handle_error(self, failure): - self.logger.warning(f"Failed: {failure.request.url}") - - def parse_staff_page(self, response): - faculty = response.meta.get("faculty", "Unknown") - department = response.meta.get("department") - - # Try to detect department from page heading - heading = ( - response.css("h1::text, h2::text, .page-title::text").get(default="") or "" - ).strip() - if heading and len(heading) < 80 and "staff" not in heading.lower(): - department = heading - - found = 0 - - # Strategy 1: Elementor headings - for name in response.css( - "h2.elementor-cta__title::text, h3.elementor-heading-title::text, h4.elementor-heading-title::text" - ).getall(): - if self._add(name, faculty, department): - found += 1 - - # Strategy 2: Bold text in content blocks - for name in response.css( - "div.kc_text_block strong::text, div.entry-content strong::text, .wp-block-column strong::text" - ).getall(): - if self._add(name, faculty, department): - found += 1 - - # Strategy 3: Table first column - for name in response.css( - "table td:first-child::text, table td:nth-child(2)::text" - ).getall(): - if self._add(name, faculty, department): - found += 1 - - # Strategy 4: List items - for name in response.css( - "li strong::text, .staff-name::text, .member-name::text, .team-member-name::text" - ).getall(): - if self._add(name, faculty, department): - found += 1 - - # Strategy 5: Generic headings (h3/h4/h5) - for name in response.css("h3::text, h4::text, h5::text").getall(): - if self._add(name, faculty, department): - found += 1 - - self.logger.info(f"[{faculty}] {found} names from {response.url}") - - # Follow department sub-pages - for link in response.css("a"): - text = (link.css("::text").get(default="")).strip() - href = link.attrib.get("href", "") - if any( - kw in text.lower() for kw in ["department", "dept", "staff", "academic"] - ): - yield response.follow( - href, - callback=self.parse_staff_page, - errback=self.handle_error, - meta={"faculty": faculty, "department": text}, - ) - - def _add(self, name: str, faculty: str, department) -> bool: - name = name.strip() - if not name or len(name) < 4: - return False - words = name.split() - if not (2 <= len(words) <= 6): - return False - if not any(c.isalpha() for c in name): - return False - if any(kw in name.lower() for kw in NOISE): - return False - # Avoid duplicates - if any(r["name"] == name for r in self.staff_records): - return False - self.staff_records.append( - {"name": name, "faculty": faculty, "department": department or faculty} - ) - self.staff_names.append(name) - return True - - def closed(self, reason): - if not self.staff_records: - self.logger.warning( - "No staff found — UNILAG site structure may have changed." - ) - return - - os.makedirs(os.path.dirname(STAFF_CACHE), exist_ok=True) - - # Save flat name list (backwards compat) - unique_names = sorted(set(self.staff_names)) - with open(STAFF_CACHE, "w", encoding="utf-8") as f: - json.dump(unique_names, f, indent=2, ensure_ascii=False) - - # Save detailed records - with open(DETAILED_CACHE, "w", encoding="utf-8") as f: - json.dump(self.staff_records, f, indent=2, ensure_ascii=False) - - self.logger.warning( - f"Saved {len(unique_names)} staff names + {len(self.staff_records)} detailed records" - ) +""" +Faculty Directory Spider — crawls UNILAG staff pages and extracts +name + faculty + department for each staff member. +Saves to data/unilag_staff_detailed.json for accurate classification. +""" + +import json +import os + +import scrapy +from scrapy.http import Request + +STAFF_CACHE = os.path.join( + os.path.dirname(__file__), "..", "..", "..", "data", "unilag_staff.json" +) +DETAILED_CACHE = os.path.join( + os.path.dirname(__file__), "..", "..", "..", "data", "unilag_staff_detailed.json" +) + +# Known UNILAG faculty URLs — direct seeds so we don't rely on the homepage nav +FACULTY_SEEDS = [ + ("College of Medicine", "https://medicine.unilag.edu.ng/academic-staff/"), + ("Faculty of Engineering", "https://engineering.unilag.edu.ng/staff/"), + ("Faculty of Science", "https://science.unilag.edu.ng/staff/"), + ("Faculty of Arts", "https://arts.unilag.edu.ng/staff/"), + ("Faculty of Social Sciences", "https://socialsciences.unilag.edu.ng/staff/"), + ("Faculty of Law", "https://law.unilag.edu.ng/staff/"), + ("Faculty of Education", "https://education.unilag.edu.ng/staff/"), + ( + "Faculty of Environmental Sciences", + "https://environmentalsciences.unilag.edu.ng/staff/", + ), + ("Faculty of Management Sciences", "https://management.unilag.edu.ng/staff/"), + ("Faculty of Pharmacy", "https://pharmacy.unilag.edu.ng/staff/"), + ("Faculty of Dental Sciences", "https://dentistry.unilag.edu.ng/staff/"), + ("Faculty of Basic Medical Sciences", "https://basicmedical.unilag.edu.ng/staff/"), +] + +NOISE = { + "click", + "view", + "department", + "faculty", + "university", + "college", + "profile", + "contact", + "email", + "phone", + "office", + "research", + "publications", + "more", + "read", + "about", + "staff", + "list", + "home", + "menu", + "search", + "login", + "logout", +} + + +class FacultyDirectorySpider(scrapy.Spider): + name = "unilag_faculty_directory" + custom_settings = { + "DOWNLOAD_DELAY": 1.5, + # Honor robots.txt — the institution's machine-readable usage signal. + "ROBOTSTXT_OBEY": True, + "RETRY_ENABLED": True, + "RETRY_TIMES": 2, + "LOG_LEVEL": "WARNING", + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.staff_records = [] # list of {name, faculty, department} + self.staff_names = [] # flat list for backwards compat + + def start_requests(self): + headers = { + "User-Agent": "Mozilla/5.0 (compatible; URAAS/1.0; mailto:library@unilag.edu.ng)" + } + for faculty_name, url in FACULTY_SEEDS: + yield Request( + url, + headers=headers, + callback=self.parse_staff_page, + errback=self.handle_error, + meta={"faculty": faculty_name, "department": None}, + ) + + def handle_error(self, failure): + self.logger.warning(f"Failed: {failure.request.url}") + + def parse_staff_page(self, response): + faculty = response.meta.get("faculty", "Unknown") + department = response.meta.get("department") + + # Try to detect department from page heading + heading = ( + response.css("h1::text, h2::text, .page-title::text").get(default="") or "" + ).strip() + if heading and len(heading) < 80 and "staff" not in heading.lower(): + department = heading + + found = 0 + + # Strategy 1: Elementor headings + for name in response.css( + "h2.elementor-cta__title::text, h3.elementor-heading-title::text, h4.elementor-heading-title::text" + ).getall(): + if self._add(name, faculty, department): + found += 1 + + # Strategy 2: Bold text in content blocks + for name in response.css( + "div.kc_text_block strong::text, div.entry-content strong::text, .wp-block-column strong::text" + ).getall(): + if self._add(name, faculty, department): + found += 1 + + # Strategy 3: Table first column + for name in response.css( + "table td:first-child::text, table td:nth-child(2)::text" + ).getall(): + if self._add(name, faculty, department): + found += 1 + + # Strategy 4: List items + for name in response.css( + "li strong::text, .staff-name::text, .member-name::text, .team-member-name::text" + ).getall(): + if self._add(name, faculty, department): + found += 1 + + # Strategy 5: Generic headings (h3/h4/h5) + for name in response.css("h3::text, h4::text, h5::text").getall(): + if self._add(name, faculty, department): + found += 1 + + self.logger.info(f"[{faculty}] {found} names from {response.url}") + + # Follow department sub-pages + for link in response.css("a"): + text = (link.css("::text").get(default="")).strip() + href = link.attrib.get("href", "") + if any( + kw in text.lower() for kw in ["department", "dept", "staff", "academic"] + ): + yield response.follow( + href, + callback=self.parse_staff_page, + errback=self.handle_error, + meta={"faculty": faculty, "department": text}, + ) + + def _add(self, name: str, faculty: str, department) -> bool: + name = name.strip() + if not name or len(name) < 4: + return False + words = name.split() + if not (2 <= len(words) <= 6): + return False + if not any(c.isalpha() for c in name): + return False + if any(kw in name.lower() for kw in NOISE): + return False + # Avoid duplicates + if any(r["name"] == name for r in self.staff_records): + return False + self.staff_records.append( + {"name": name, "faculty": faculty, "department": department or faculty} + ) + self.staff_names.append(name) + return True + + def closed(self, reason): + if not self.staff_records: + self.logger.warning( + "No staff found — UNILAG site structure may have changed." + ) + return + + os.makedirs(os.path.dirname(STAFF_CACHE), exist_ok=True) + + # Save flat name list (backwards compat) + unique_names = sorted(set(self.staff_names)) + with open(STAFF_CACHE, "w", encoding="utf-8") as f: + json.dump(unique_names, f, indent=2, ensure_ascii=False) + + # Save detailed records + with open(DETAILED_CACHE, "w", encoding="utf-8") as f: + json.dump(self.staff_records, f, indent=2, ensure_ascii=False) + + self.logger.warning( + f"Saved {len(unique_names)} staff names + {len(self.staff_records)} detailed records" + ) diff --git a/uraas/spiders/sources/oai_spider.py b/uraas/spiders/sources/oai_spider.py index 3ee9c405303da8ca9ea81e8aaa647fb97acd30d8..550ed8eaf925f1b6f718bc596ce6b9a0ba0d9358 100644 --- a/uraas/spiders/sources/oai_spider.py +++ b/uraas/spiders/sources/oai_spider.py @@ -1,344 +1,344 @@ -""" -Read-only OAI-PMH harvester for an institution's DSpace repository. - -This is the ONLY URAAS spider that talks to an institution's *own* repository -server (e.g. UNILAG's ``api-ir.unilag.edu.ng``). It uses the OAI-PMH protocol, -which is **read-only by specification** — it has no verbs that create, modify, or -delete repository content — so it cannot harm the source repository. It only -issues ``ListRecords`` GETs and follows ``resumptionToken`` pages. - -Why this spider exists: the aggregator spiders (OpenAlex/Crossref/arXiv/ORCID) -have broad citation/OA coverage but miss locally-deposited **theses, -dissertations and grey literature** that only live in the institutional -repository. This harvester complements them. - -Behaviour notes: -* **Always bounded.** The endpoint is harvested incrementally with ``from`` (and - optional ``until``). An unbounded full harvest can time out the server, so a - ``from`` lower bound is always sent (defaulting to a recent look-back window). -* **Polite.** One request at a time, a download delay, AutoThrottle, and a - contact ``User-Agent`` so the repository admin can identify URAAS traffic. -* **SC-gated downstream.** Like every other source, harvested records flow through - ``DatabaseStoragePipeline``, which keeps only items the Special-Collections - classifier scores > 0 — exactly the indigenous-knowledge / cultural-heritage / - local material the aggregators omit. -""" - -import logging -import os -import sys -from datetime import datetime, timedelta, timezone - -import scrapy -from scrapy.selector import Selector - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) - -from uraas.config import config -from uraas.config.institutions import get_registry - -log = logging.getLogger(__name__) - -# OAI-PMH XML namespaces. -_NS = { - "oai": "http://www.openarchives.org/OAI/2.0/", - "oai_dc": "http://www.openarchives.org/OAI/2.0/oai_dc/", - "dc": "http://purl.org/dc/elements/1.1/", -} - -# Default look-back when no from_date is supplied. -# Keep at ~5 years: covers the bulk of active IR deposits without causing 500s -# on DSpace servers that struggle with very large date-range requests. -# For a full back-catalogue harvest, pass --from-date 2000-01-01 explicitly. -_DEFAULT_LOOKBACK_DAYS = 1825 # ~5 years - - -class OAISpider(scrapy.Spider): - """Harvest oai_dc metadata from an institution's public OAI-PMH endpoint.""" - - name = "oai_repository" - custom_settings = { - # Deliberately gentle on the institution's own server. - "DOWNLOAD_DELAY": 2.0, - "CONCURRENT_REQUESTS": 1, - "AUTOTHROTTLE_ENABLED": True, - "AUTOTHROTTLE_START_DELAY": 2.0, - "AUTOTHROTTLE_MAX_DELAY": 15.0, - "RETRY_ENABLED": True, - "RETRY_TIMES": 2, - "ROBOTSTXT_OBEY": True, - "USER_AGENT": ( - f"URAAS/1.0 (+read-only OAI-PMH harvester; " - f"mailto:{config.OPENALEX_MAILTO})" - ), - } - - def __init__( - self, - institution="unilag", - target=200, - from_date=None, - until_date=None, - oai_set=None, - *args, - **kwargs, - ): - """ - institution: registry short name; must have ``oai_endpoint`` configured. - target: max records to accept this run (hard stop). - from_date: lower bound ``YYYY-MM-DD`` (defaults to a recent look-back). - until_date: optional upper bound ``YYYY-MM-DD``. - oai_set: optional OAI-PMH set spec (e.g. a DSpace community/collection - handle like ``com_1234_56``) to filter at source. When set, - only records belonging to that set are returned by the server — - drastically reducing traffic for focused harvests. If None, - the institution config's ``oai_set`` field is used if present. - """ - super().__init__(*args, **kwargs) - self.target_limit = int(target) - - registry = get_registry() - self.institution_config = registry.get(institution) - if not self.institution_config: - raise ValueError(f"Institution '{institution}' not found in registry") - - self.oai_endpoint = self.institution_config.oai_endpoint - if not self.oai_endpoint: - raise ValueError( - f"Institution '{institution}' has no oai_endpoint configured. " - f"Add it to config/institutions/{institution}.json to enable " - f"OAI-PMH harvesting." - ) - - # Read by DatabaseStoragePipeline via getattr(). - self.institution_name = self.institution_config.name - self.ror_id = self.institution_config.ror - - self.from_date = self._normalize_date(from_date) or self._default_from() - self.until_date = self._normalize_date(until_date) - # OAI set: explicit arg > institution config > None (no set filter) - self.oai_set = ( - oai_set - or getattr(self.institution_config, "oai_set", None) - or None - ) - - self._accepted = 0 - - self.logger.info( - "OAI harvester | %s | endpoint=%s | from=%s until=%s | set=%s | target=%d", - self.institution_name, - self.oai_endpoint, - self.from_date, - self.until_date or "(now)", - self.oai_set or "(all)", - self.target_limit, - ) - - # ── URL building ───────────────────────────────────────────────────────── - @staticmethod - def _normalize_date(value): - """Accept YYYY-MM-DD (or full ISO) and return YYYY-MM-DD, else None.""" - if not value: - return None - text = str(value).strip() - if not text: - return None - # Keep only the date part; OAI granularity is fine with YYYY-MM-DD. - return text[:10] - - def _default_from(self) -> str: - cutoff = datetime.now(timezone.utc) - timedelta(days=_DEFAULT_LOOKBACK_DAYS) - return cutoff.strftime("%Y-%m-%d") - - def _list_records_url(self) -> str: - from urllib.parse import urlencode - - params = { - "verb": "ListRecords", - "metadataPrefix": "oai_dc", - "from": self.from_date, - } - if self.until_date: - params["until"] = self.until_date - if self.oai_set: - params["set"] = self.oai_set - return f"{self.oai_endpoint}?{urlencode(params)}" - - def _resume_url(self, token: str) -> str: - from urllib.parse import urlencode - - # Per OAI-PMH spec, resumptionToken is sent alone with the verb. - return f"{self.oai_endpoint}?{urlencode({'verb': 'ListRecords', 'resumptionToken': token})}" - - async def start(self): - url = self._list_records_url() - self.logger.info("[OAI ListRecords] %s", url) - yield scrapy.Request(url=url, callback=self.parse, meta={"oai": True}) - - # ── Parsing ────────────────────────────────────────────────────────────── - def parse(self, response): - if self._accepted >= self.target_limit: - return - - # Parse explicitly as XML. OAI-PMH always returns XML, but depending on the - # Content-Type header Scrapy may otherwise build an HTML selector (which - # silently fails to match the namespaced OAI/DC nodes). - sel = Selector(text=response.text, type="xml") - for prefix, uri in _NS.items(): - sel.register_namespace(prefix, uri) - - # OAI-level error (badArgument, noRecordsMatch, etc.) — log and stop. - error = sel.xpath("//oai:error/@code").get() - if error: - self.logger.warning( - "OAI error '%s': %s", - error, - sel.xpath("//oai:error/text()").get() or "", - ) - return - - records = sel.xpath("//oai:ListRecords/oai:record") - self.logger.info("[OAI] received %d records", len(records)) - - for record in records: - if self._accepted >= self.target_limit: - break - - # Skip deleted records (header status="deleted", no metadata body). - if record.xpath("./oai:header/@status").get() == "deleted": - continue - - dc = record.xpath("./oai:metadata/oai_dc:dc") - if not dc: - continue - dc = dc[0] - - title = (dc.xpath("./dc:title/text()").get() or "").strip() - if not title: - continue - - creators = [ - c.strip() - for c in dc.xpath("./dc:creator/text()").getall() - if c and c.strip() - ] - subjects = [ - s.strip() - for s in dc.xpath("./dc:subject/text()").getall() - if s and s.strip() - ] - descriptions = [ - d.strip() - for d in dc.xpath("./dc:description/text()").getall() - if d and d.strip() - ] - identifiers = [ - i.strip() - for i in dc.xpath("./dc:identifier/text()").getall() - if i and i.strip() - ] - dates = [ - d.strip() - for d in dc.xpath("./dc:date/text()").getall() - if d and d.strip() - ] - rights = [ - r.strip() - for r in dc.xpath("./dc:rights/text()").getall() - if r and r.strip() - ] - doc_type = (dc.xpath("./dc:type/text()").get() or "").strip() - - url, doi = self._pick_url_and_doi(identifiers) - pub_date = self._pick_publication_date(dates) - abstract = max(descriptions, key=len) if descriptions else "" - - self._accepted += 1 - yield { - "title": title, - "abstract": abstract, - "authors": creators, - "authors_full": [ - {"name": c, "orcid": "", "ror": ""} for c in creators - ], - "doi": doi, - "url": url, - "pdf_url": None, # OAI metadata only; do not fetch IR bitstreams. - "publication_date": pub_date, - "source_repository": f"{self.institution_config.short_name} IR (OAI-PMH)", - "is_unilag_author": True, # Legacy field expected by pipeline. - "raw_affiliation": self.institution_name, - "institution": self.institution_name, - "institution_ror": self.ror_id, - "dc_subject": ", ".join(subjects[:8]), - "dc_rights": self._map_rights(rights), - "content_type": doc_type or None, - "sdg_tags": None, - } - - # ── Pagination via resumptionToken ─────────────────────────────────── - token = sel.xpath("//oai:ListRecords/oai:resumptionToken/text()").get() - if token and token.strip() and self._accepted < self.target_limit: - next_url = self._resume_url(token.strip()) - yield scrapy.Request(url=next_url, callback=self.parse, meta={"oai": True}) - - # ── Field helpers ───────────────────────────────────────────────────────── - @staticmethod - def _pick_url_and_doi(identifiers): - """From dc:identifier values, pick a landing URL (prefer the Handle) and a DOI.""" - url = "" - doi = "" - for ident in identifiers: - low = ident.lower() - if "doi.org/" in low or low.startswith("10."): - doi = ident - if ident.startswith("http") and not url: - url = ident - # Prefer a real handle/landing page over a citation string. - if "/handle/" in low: - url = ident - return url, doi - - @staticmethod - def _pick_publication_date(dates): - """Pick the most plausible publication date. - - DSpace emits accessioned/available timestamps plus an 'issued' value - (often just a year). Prefer a bare year/short date (the issued one) over - the long accession timestamps. - """ - if not dates: - return "" - # A YYYY or YYYY-MM-DD style value is the issued date; timestamps contain 'T'. - issued = [d for d in dates if "T" not in d] - return (issued[0] if issued else dates[-1]) - - @staticmethod - def _map_rights(rights): - """Map dc:rights to the repo's rights convention. - - Marks clearly-open licences as open access so they aren't needlessly - gated; everything else stays restricted (the safe default). URAAS stores - only metadata + the landing URL here, never the IR's bitstreams. - """ - joined = " ".join(rights).lower() - open_markers = ( - "creativecommons.org", - "cc0", - "cc by", - "public domain", - "open access", - "openaccess", - ) - if any(m in joined for m in open_markers): - return "info:eu-repo/semantics/openAccess" - return "info:eu-repo/semantics/restrictedAccess" - - def closed(self, reason): - self.logger.info( - "OAI harvester closed: %s | accepted %d (reason=%s)", - self.institution_name, - self._accepted, - reason, - ) +""" +Read-only OAI-PMH harvester for an institution's DSpace repository. + +This is the ONLY URAAS spider that talks to an institution's *own* repository +server (e.g. UNILAG's ``api-ir.unilag.edu.ng``). It uses the OAI-PMH protocol, +which is **read-only by specification** — it has no verbs that create, modify, or +delete repository content — so it cannot harm the source repository. It only +issues ``ListRecords`` GETs and follows ``resumptionToken`` pages. + +Why this spider exists: the aggregator spiders (OpenAlex/Crossref/arXiv/ORCID) +have broad citation/OA coverage but miss locally-deposited **theses, +dissertations and grey literature** that only live in the institutional +repository. This harvester complements them. + +Behaviour notes: +* **Always bounded.** The endpoint is harvested incrementally with ``from`` (and + optional ``until``). An unbounded full harvest can time out the server, so a + ``from`` lower bound is always sent (defaulting to a recent look-back window). +* **Polite.** One request at a time, a download delay, AutoThrottle, and a + contact ``User-Agent`` so the repository admin can identify URAAS traffic. +* **SC-gated downstream.** Like every other source, harvested records flow through + ``DatabaseStoragePipeline``, which keeps only items the Special-Collections + classifier scores > 0 — exactly the indigenous-knowledge / cultural-heritage / + local material the aggregators omit. +""" + +import logging +import os +import sys +from datetime import datetime, timedelta, timezone + +import scrapy +from scrapy.selector import Selector + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) + +from uraas.config import config +from uraas.config.institutions import get_registry + +log = logging.getLogger(__name__) + +# OAI-PMH XML namespaces. +_NS = { + "oai": "http://www.openarchives.org/OAI/2.0/", + "oai_dc": "http://www.openarchives.org/OAI/2.0/oai_dc/", + "dc": "http://purl.org/dc/elements/1.1/", +} + +# Default look-back when no from_date is supplied. +# Keep at ~5 years: covers the bulk of active IR deposits without causing 500s +# on DSpace servers that struggle with very large date-range requests. +# For a full back-catalogue harvest, pass --from-date 2000-01-01 explicitly. +_DEFAULT_LOOKBACK_DAYS = 1825 # ~5 years + + +class OAISpider(scrapy.Spider): + """Harvest oai_dc metadata from an institution's public OAI-PMH endpoint.""" + + name = "oai_repository" + custom_settings = { + # Deliberately gentle on the institution's own server. + "DOWNLOAD_DELAY": 2.0, + "CONCURRENT_REQUESTS": 1, + "AUTOTHROTTLE_ENABLED": True, + "AUTOTHROTTLE_START_DELAY": 2.0, + "AUTOTHROTTLE_MAX_DELAY": 15.0, + "RETRY_ENABLED": True, + "RETRY_TIMES": 2, + "ROBOTSTXT_OBEY": True, + "USER_AGENT": ( + f"URAAS/1.0 (+read-only OAI-PMH harvester; " + f"mailto:{config.OPENALEX_MAILTO})" + ), + } + + def __init__( + self, + institution="unilag", + target=200, + from_date=None, + until_date=None, + oai_set=None, + *args, + **kwargs, + ): + """ + institution: registry short name; must have ``oai_endpoint`` configured. + target: max records to accept this run (hard stop). + from_date: lower bound ``YYYY-MM-DD`` (defaults to a recent look-back). + until_date: optional upper bound ``YYYY-MM-DD``. + oai_set: optional OAI-PMH set spec (e.g. a DSpace community/collection + handle like ``com_1234_56``) to filter at source. When set, + only records belonging to that set are returned by the server — + drastically reducing traffic for focused harvests. If None, + the institution config's ``oai_set`` field is used if present. + """ + super().__init__(*args, **kwargs) + self.target_limit = int(target) + + registry = get_registry() + self.institution_config = registry.get(institution) + if not self.institution_config: + raise ValueError(f"Institution '{institution}' not found in registry") + + self.oai_endpoint = self.institution_config.oai_endpoint + if not self.oai_endpoint: + raise ValueError( + f"Institution '{institution}' has no oai_endpoint configured. " + f"Add it to config/institutions/{institution}.json to enable " + f"OAI-PMH harvesting." + ) + + # Read by DatabaseStoragePipeline via getattr(). + self.institution_name = self.institution_config.name + self.ror_id = self.institution_config.ror + + self.from_date = self._normalize_date(from_date) or self._default_from() + self.until_date = self._normalize_date(until_date) + # OAI set: explicit arg > institution config > None (no set filter) + self.oai_set = ( + oai_set + or getattr(self.institution_config, "oai_set", None) + or None + ) + + self._accepted = 0 + + self.logger.info( + "OAI harvester | %s | endpoint=%s | from=%s until=%s | set=%s | target=%d", + self.institution_name, + self.oai_endpoint, + self.from_date, + self.until_date or "(now)", + self.oai_set or "(all)", + self.target_limit, + ) + + # ── URL building ───────────────────────────────────────────────────────── + @staticmethod + def _normalize_date(value): + """Accept YYYY-MM-DD (or full ISO) and return YYYY-MM-DD, else None.""" + if not value: + return None + text = str(value).strip() + if not text: + return None + # Keep only the date part; OAI granularity is fine with YYYY-MM-DD. + return text[:10] + + def _default_from(self) -> str: + cutoff = datetime.now(timezone.utc) - timedelta(days=_DEFAULT_LOOKBACK_DAYS) + return cutoff.strftime("%Y-%m-%d") + + def _list_records_url(self) -> str: + from urllib.parse import urlencode + + params = { + "verb": "ListRecords", + "metadataPrefix": "oai_dc", + "from": self.from_date, + } + if self.until_date: + params["until"] = self.until_date + if self.oai_set: + params["set"] = self.oai_set + return f"{self.oai_endpoint}?{urlencode(params)}" + + def _resume_url(self, token: str) -> str: + from urllib.parse import urlencode + + # Per OAI-PMH spec, resumptionToken is sent alone with the verb. + return f"{self.oai_endpoint}?{urlencode({'verb': 'ListRecords', 'resumptionToken': token})}" + + async def start(self): + url = self._list_records_url() + self.logger.info("[OAI ListRecords] %s", url) + yield scrapy.Request(url=url, callback=self.parse, meta={"oai": True}) + + # ── Parsing ────────────────────────────────────────────────────────────── + def parse(self, response): + if self._accepted >= self.target_limit: + return + + # Parse explicitly as XML. OAI-PMH always returns XML, but depending on the + # Content-Type header Scrapy may otherwise build an HTML selector (which + # silently fails to match the namespaced OAI/DC nodes). + sel = Selector(text=response.text, type="xml") + for prefix, uri in _NS.items(): + sel.register_namespace(prefix, uri) + + # OAI-level error (badArgument, noRecordsMatch, etc.) — log and stop. + error = sel.xpath("//oai:error/@code").get() + if error: + self.logger.warning( + "OAI error '%s': %s", + error, + sel.xpath("//oai:error/text()").get() or "", + ) + return + + records = sel.xpath("//oai:ListRecords/oai:record") + self.logger.info("[OAI] received %d records", len(records)) + + for record in records: + if self._accepted >= self.target_limit: + break + + # Skip deleted records (header status="deleted", no metadata body). + if record.xpath("./oai:header/@status").get() == "deleted": + continue + + dc = record.xpath("./oai:metadata/oai_dc:dc") + if not dc: + continue + dc = dc[0] + + title = (dc.xpath("./dc:title/text()").get() or "").strip() + if not title: + continue + + creators = [ + c.strip() + for c in dc.xpath("./dc:creator/text()").getall() + if c and c.strip() + ] + subjects = [ + s.strip() + for s in dc.xpath("./dc:subject/text()").getall() + if s and s.strip() + ] + descriptions = [ + d.strip() + for d in dc.xpath("./dc:description/text()").getall() + if d and d.strip() + ] + identifiers = [ + i.strip() + for i in dc.xpath("./dc:identifier/text()").getall() + if i and i.strip() + ] + dates = [ + d.strip() + for d in dc.xpath("./dc:date/text()").getall() + if d and d.strip() + ] + rights = [ + r.strip() + for r in dc.xpath("./dc:rights/text()").getall() + if r and r.strip() + ] + doc_type = (dc.xpath("./dc:type/text()").get() or "").strip() + + url, doi = self._pick_url_and_doi(identifiers) + pub_date = self._pick_publication_date(dates) + abstract = max(descriptions, key=len) if descriptions else "" + + self._accepted += 1 + yield { + "title": title, + "abstract": abstract, + "authors": creators, + "authors_full": [ + {"name": c, "orcid": "", "ror": ""} for c in creators + ], + "doi": doi, + "url": url, + "pdf_url": None, # OAI metadata only; do not fetch IR bitstreams. + "publication_date": pub_date, + "source_repository": f"{self.institution_config.short_name} IR (OAI-PMH)", + "is_unilag_author": True, # Legacy field expected by pipeline. + "raw_affiliation": self.institution_name, + "institution": self.institution_name, + "institution_ror": self.ror_id, + "dc_subject": ", ".join(subjects[:8]), + "dc_rights": self._map_rights(rights), + "content_type": doc_type or None, + "sdg_tags": None, + } + + # ── Pagination via resumptionToken ─────────────────────────────────── + token = sel.xpath("//oai:ListRecords/oai:resumptionToken/text()").get() + if token and token.strip() and self._accepted < self.target_limit: + next_url = self._resume_url(token.strip()) + yield scrapy.Request(url=next_url, callback=self.parse, meta={"oai": True}) + + # ── Field helpers ───────────────────────────────────────────────────────── + @staticmethod + def _pick_url_and_doi(identifiers): + """From dc:identifier values, pick a landing URL (prefer the Handle) and a DOI.""" + url = "" + doi = "" + for ident in identifiers: + low = ident.lower() + if "doi.org/" in low or low.startswith("10."): + doi = ident + if ident.startswith("http") and not url: + url = ident + # Prefer a real handle/landing page over a citation string. + if "/handle/" in low: + url = ident + return url, doi + + @staticmethod + def _pick_publication_date(dates): + """Pick the most plausible publication date. + + DSpace emits accessioned/available timestamps plus an 'issued' value + (often just a year). Prefer a bare year/short date (the issued one) over + the long accession timestamps. + """ + if not dates: + return "" + # A YYYY or YYYY-MM-DD style value is the issued date; timestamps contain 'T'. + issued = [d for d in dates if "T" not in d] + return (issued[0] if issued else dates[-1]) + + @staticmethod + def _map_rights(rights): + """Map dc:rights to the repo's rights convention. + + Marks clearly-open licences as open access so they aren't needlessly + gated; everything else stays restricted (the safe default). URAAS stores + only metadata + the landing URL here, never the IR's bitstreams. + """ + joined = " ".join(rights).lower() + open_markers = ( + "creativecommons.org", + "cc0", + "cc by", + "public domain", + "open access", + "openaccess", + ) + if any(m in joined for m in open_markers): + return "info:eu-repo/semantics/openAccess" + return "info:eu-repo/semantics/restrictedAccess" + + def closed(self, reason): + self.logger.info( + "OAI harvester closed: %s | accepted %d (reason=%s)", + self.institution_name, + self._accepted, + reason, + ) diff --git a/uraas/spiders/sources/openaire_spider.py b/uraas/spiders/sources/openaire_spider.py index c7023aec947d4094f89a24ce90bb23b73d3880f6..66a499306371881a52fa949ee84923354491865b 100644 --- a/uraas/spiders/sources/openaire_spider.py +++ b/uraas/spiders/sources/openaire_spider.py @@ -1,125 +1,125 @@ -""" -OpenAIRE spider — queries the OpenAIRE Graph API. - -OpenAIRE aggregates research from EU-funded projects, repositories across 150+ -countries, and African research networks (NREN partnerships, African university -repositories). Particularly strong for: - • African development research - • Research from Nigerian/West African institutions - • Open access preprints and technical reports not in OpenAlex - -Free API, no key required. Rate limit: 7200 req/hour. -Docs: graph.openaire.eu/docs/apis/ -""" - -import os -import sys -from urllib.parse import urlencode - -import scrapy - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) - -from uraas.config import config -from uraas.config.institutions import get_registry -from uraas.config.special_collections import SC_SEED_KEYWORDS - -_BASE = "https://api.openaire.eu/search/publications" - - -class OpenAIRESpider(scrapy.Spider): - name = "openaire" - custom_settings = { - "DOWNLOAD_DELAY": 0.6, - "CONCURRENT_REQUESTS": 1, - "USER_AGENT": f"URAAS/1.0 (+SC discovery; mailto:{config.OPENALEX_MAILTO})", - } - - def __init__(self, institution="unilag", target=50, boost_special=True, sc_only=False, *args, **kwargs): - super().__init__(*args, **kwargs) - self.target_limit = int(target) - _truthy = {"1", "true", "yes", "on"} - self.boost_special = boost_special.lower() in _truthy if isinstance(boost_special, str) else bool(boost_special) - self.sc_only = sc_only.lower() in _truthy if isinstance(sc_only, str) else bool(sc_only) - registry = get_registry() - self.institution_config = registry.get(institution) - if not self.institution_config: - raise ValueError(f"Institution '{institution}' not found") - self.institution_name = self.institution_config.name - self.ror_id = self.institution_config.ror - self._accepted = 0 - - def _build_url(self, keywords: str = "", page: int = 1) -> str: - params = { - "affiliationORG": self.institution_name, - "format": "json", - "size": 50, - "page": page, - } - if keywords: - params["keywords"] = keywords - return f"{_BASE}?{urlencode(params)}" - - async def start(self): - if not self.sc_only: - yield scrapy.Request(self._build_url(), callback=self.parse, - meta={"keywords": "", "page": 1}) - if self.boost_special: - priority_seeds = [s for s in SC_SEED_KEYWORDS - if any(k in s for k in ("indigenous", "cultural", "postcolonial", "oral", "decolonial", "ubuntu"))] - for seed in priority_seeds: - yield scrapy.Request(self._build_url(seed), callback=self.parse, - meta={"keywords": seed, "page": 1}, priority=10) - - def parse(self, response): - if self._accepted >= self.target_limit: - return - data = response.json() - results = (data.get("response", {}).get("results", {}).get("result") or []) - if not isinstance(results, list): - results = [results] if results else [] - - for r in results: - if self._accepted >= self.target_limit: - return - metadata = r.get("metadata", {}).get("oaf:entity", {}).get("oaf:result", {}) - title_obj = metadata.get("title", {}) - title = (title_obj if isinstance(title_obj, str) else title_obj.get("$", "")).strip() - if not title: - continue - - desc = metadata.get("description", "") or "" - abstract = (desc if isinstance(desc, str) else desc.get("$", "")).strip() - - pids = metadata.get("pid") or [] - if not isinstance(pids, list): - pids = [pids] - doi = "" - for pid in pids: - if isinstance(pid, dict) and pid.get("@classid") == "doi": - doi = pid.get("$", "") - - creators_raw = metadata.get("creator") or [] - if not isinstance(creators_raw, list): - creators_raw = [creators_raw] - authors = [c.get("$", "") if isinstance(c, dict) else str(c) for c in creators_raw if c] - - year = str(metadata.get("dateofacceptance", "") or "")[:4] - url_val = f"https://doi.org/{doi}" if doi else "" - - self._accepted += 1 - yield { - "title": title, "abstract": abstract, "authors": authors, "doi": doi, - "url": url_val, "pdf_url": None, "publication_date": year, - "source_repository": "OpenAIRE", "is_unilag_author": True, - "raw_affiliation": self.institution_name, "institution": self.institution_name, - "institution_ror": self.ror_id, - } - - # Pagination - total = int(data.get("response", {}).get("header", {}).get("total", {}).get("$", 0) or 0) - page = response.meta.get("page", 1) - keywords = response.meta.get("keywords", "") - if page * 50 < min(total, 500) and self._accepted < self.target_limit: - yield scrapy.Request(self._build_url(keywords, page + 1), callback=self.parse, - meta={"keywords": keywords, "page": page + 1}) +""" +OpenAIRE spider — queries the OpenAIRE Graph API. + +OpenAIRE aggregates research from EU-funded projects, repositories across 150+ +countries, and African research networks (NREN partnerships, African university +repositories). Particularly strong for: + • African development research + • Research from Nigerian/West African institutions + • Open access preprints and technical reports not in OpenAlex + +Free API, no key required. Rate limit: 7200 req/hour. +Docs: graph.openaire.eu/docs/apis/ +""" + +import os +import sys +from urllib.parse import urlencode + +import scrapy + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) + +from uraas.config import config +from uraas.config.institutions import get_registry +from uraas.config.special_collections import SC_SEED_KEYWORDS + +_BASE = "https://api.openaire.eu/search/publications" + + +class OpenAIRESpider(scrapy.Spider): + name = "openaire" + custom_settings = { + "DOWNLOAD_DELAY": 0.6, + "CONCURRENT_REQUESTS": 1, + "USER_AGENT": f"URAAS/1.0 (+SC discovery; mailto:{config.OPENALEX_MAILTO})", + } + + def __init__(self, institution="unilag", target=50, boost_special=True, sc_only=False, *args, **kwargs): + super().__init__(*args, **kwargs) + self.target_limit = int(target) + _truthy = {"1", "true", "yes", "on"} + self.boost_special = boost_special.lower() in _truthy if isinstance(boost_special, str) else bool(boost_special) + self.sc_only = sc_only.lower() in _truthy if isinstance(sc_only, str) else bool(sc_only) + registry = get_registry() + self.institution_config = registry.get(institution) + if not self.institution_config: + raise ValueError(f"Institution '{institution}' not found") + self.institution_name = self.institution_config.name + self.ror_id = self.institution_config.ror + self._accepted = 0 + + def _build_url(self, keywords: str = "", page: int = 1) -> str: + params = { + "affiliationORG": self.institution_name, + "format": "json", + "size": 50, + "page": page, + } + if keywords: + params["keywords"] = keywords + return f"{_BASE}?{urlencode(params)}" + + async def start(self): + if not self.sc_only: + yield scrapy.Request(self._build_url(), callback=self.parse, + meta={"keywords": "", "page": 1}) + if self.boost_special: + priority_seeds = [s for s in SC_SEED_KEYWORDS + if any(k in s for k in ("indigenous", "cultural", "postcolonial", "oral", "decolonial", "ubuntu"))] + for seed in priority_seeds: + yield scrapy.Request(self._build_url(seed), callback=self.parse, + meta={"keywords": seed, "page": 1}, priority=10) + + def parse(self, response): + if self._accepted >= self.target_limit: + return + data = response.json() + results = (data.get("response", {}).get("results", {}).get("result") or []) + if not isinstance(results, list): + results = [results] if results else [] + + for r in results: + if self._accepted >= self.target_limit: + return + metadata = r.get("metadata", {}).get("oaf:entity", {}).get("oaf:result", {}) + title_obj = metadata.get("title", {}) + title = (title_obj if isinstance(title_obj, str) else title_obj.get("$", "")).strip() + if not title: + continue + + desc = metadata.get("description", "") or "" + abstract = (desc if isinstance(desc, str) else desc.get("$", "")).strip() + + pids = metadata.get("pid") or [] + if not isinstance(pids, list): + pids = [pids] + doi = "" + for pid in pids: + if isinstance(pid, dict) and pid.get("@classid") == "doi": + doi = pid.get("$", "") + + creators_raw = metadata.get("creator") or [] + if not isinstance(creators_raw, list): + creators_raw = [creators_raw] + authors = [c.get("$", "") if isinstance(c, dict) else str(c) for c in creators_raw if c] + + year = str(metadata.get("dateofacceptance", "") or "")[:4] + url_val = f"https://doi.org/{doi}" if doi else "" + + self._accepted += 1 + yield { + "title": title, "abstract": abstract, "authors": authors, "doi": doi, + "url": url_val, "pdf_url": None, "publication_date": year, + "source_repository": "OpenAIRE", "is_unilag_author": True, + "raw_affiliation": self.institution_name, "institution": self.institution_name, + "institution_ror": self.ror_id, + } + + # Pagination + total = int(data.get("response", {}).get("header", {}).get("total", {}).get("$", 0) or 0) + page = response.meta.get("page", 1) + keywords = response.meta.get("keywords", "") + if page * 50 < min(total, 500) and self._accepted < self.target_limit: + yield scrapy.Request(self._build_url(keywords, page + 1), callback=self.parse, + meta={"keywords": keywords, "page": page + 1}) diff --git a/uraas/spiders/sources/openalex_spider.py b/uraas/spiders/sources/openalex_spider.py index 3e893873ef2c3448498408bca06a6f0823983285..f8bc5a2345362531def73f5bd32ef44cee7408d0 100644 --- a/uraas/spiders/sources/openalex_spider.py +++ b/uraas/spiders/sources/openalex_spider.py @@ -1,365 +1,365 @@ -import logging -import os -import sys - -import scrapy - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) - -from uraas.config.institutions import get_registry -from uraas.config.special_collections import SC_OPENALEX_CONCEPTS, SC_SEED_KEYWORDS - -OPENALEX_BASE = "https://api.openalex.org/works" -MAILTO = "cokiki@unilag.edu.ng" - -log = logging.getLogger(__name__) - - -class OpenAlexSpider(scrapy.Spider): - """ - OpenAlex spider with 3-gate precision for 98% crawl accuracy. - - Gate 1: ROR-filtered API query (only papers from institution's ROR) - Gate 2: Per-paper authorship ROR verification (at least 1 author has target ROR) - Gate 3: Affiliation string pattern matching (belt-and-suspenders) - - Papers failing any gate are dropped — never mixed across institutions. - """ - - name = "openalex_multi" - custom_settings = { - # OpenAlex polite pool: ~10 req/s per IP. When multiple institution - # spiders run in parallel they share the IP budget, so we need a - # meaningful delay and generous autothrottle ceiling so 429s are - # absorbed gracefully rather than exhausting all retries. - "DOWNLOAD_DELAY": 2.0, - "AUTOTHROTTLE_ENABLED": True, - "AUTOTHROTTLE_START_DELAY": 2.0, - "AUTOTHROTTLE_MAX_DELAY": 60.0, - "AUTOTHROTTLE_TARGET_CONCURRENCY": 1, - "CONCURRENT_REQUESTS": 1, - "RETRY_ENABLED": True, - "RETRY_TIMES": 5, - "RETRY_HTTP_CODES": [429, 500, 502, 503, 504], - "HTTPERROR_ALLOWED_CODES": [429], - } - - def __init__( - self, - institution="unilag", - target=20, - boost_special=True, - sc_only=False, - *args, - **kwargs, - ): - """ - institution: registry short name (e.g. "unilag") - target: max SC papers to accept this run - boost_special: also run SC seed waves (topic+ROR) in addition to the - general ROR wave — keeps SC recall high (default ON) - sc_only: skip the general ROR wave and run ONLY SC seed waves; - use when you only want targeted SC discovery, no noise - """ - super().__init__(*args, **kwargs) - self.target_limit = int(target) - # Accept both bool and string ("true"/"false") — Scrapy passes CLI - # spider args as strings when launched via crawl_multi_institution.py. - _truthy = {"1", "true", "yes", "on"} - if isinstance(boost_special, str): - self.boost_special = boost_special.lower() in _truthy - else: - self.boost_special = bool(boost_special) - if isinstance(sc_only, str): - self.sc_only = sc_only.lower() in _truthy - else: - self.sc_only = bool(sc_only) - registry = get_registry() - self.institution_config = registry.get(institution) - if not self.institution_config: - raise ValueError(f"Institution '{institution}' not found in registry") - - self.institution_name = self.institution_config.name - self.ror_id = self.institution_config.ror - self.ror_short = self.ror_id.split("/")[-1] - - self._accepted = 0 - self._rejected_gate2 = 0 - self._rejected_gate3 = 0 - self._sc_accepted = 0 - - self.logger.info( - f"OpenAlex spider for {self.institution_name} | ROR: {self.ror_short} " - f"| boost_special={self.boost_special} | sc_only={self.sc_only}" - ) - - SELECT_FIELDS = ( - "id,doi,title,abstract_inverted_index,authorships," - "publication_date,open_access,primary_location,concepts" - ) - - def _build_url(self, *, filters: str, cursor: str = "*") -> str: - return ( - f"{OPENALEX_BASE}" - f"?filter={filters}" - f"&select={self.SELECT_FIELDS}" - f"&per-page=200" - f"&cursor={cursor}" - f"&mailto={MAILTO}" - ) - - async def start(self): - # Wave 1 — general ROR-only crawl (skipped in sc_only mode) - if not self.sc_only: - url = self._build_url(filters=f"institutions.ror:{self.ror_short}") - self.logger.info(f"[ROR wave] {url}") - yield scrapy.Request( - url=url, - callback=self.parse, - meta={"source": "ror", "wave": "ror"}, - priority=0, - ) - - # Wave 2 — SC-boosted waves: one request per SC seed phrase, AND-ed with ROR. - # OpenAlex combines filters with comma=AND. The valid free-text filter is - # title_and_abstract.search (concepts.display_name.search is not supported — - # only concepts.id is). We rely on free-text seeds; the in-pipeline classifier - # then scores the actual hits. - if self.boost_special: - seeds = set(SC_SEED_KEYWORDS) - for seed in sorted(seeds): - seed_q = seed.replace(" ", "%20") - filters = ( - f"institutions.ror:{self.ror_short}," - f"title_and_abstract.search:{seed_q}" - ) - url = self._build_url(filters=filters) - self.logger.info(f"[SC wave seed={seed!r}] {url}") - yield scrapy.Request( - url=url, - callback=self.parse, - meta={"source": "ror+seed", "wave": f"sc:{seed}"}, - priority=10, # Prioritize SC papers to fill target first - ) - - def parse(self, response): - assert self.institution_config is not None, "Institution config must be loaded" - - if response.status == 429: - self.logger.warning( - "OpenAlex rate-limited (429) on wave=%s — Scrapy retry will back off", - response.meta.get("wave", "?"), - ) - return - - # Hard stop if we've already reached the global target - if self._accepted >= self.target_limit: - return - - wave = response.meta.get("wave", "ror") - is_sc_wave = wave.startswith("sc:") - # Every wave respects the global target limit. No more 10x headroom. - wave_cap = self.target_limit - - data = response.json() - results = data.get("results", []) - self.logger.info(f"[{wave}] received {len(results)} works") - - wave_accepted = response.meta.get("wave_accepted", 0) - - for work in results: - # Check both wave-local cap and global target limit - if wave_accepted >= wave_cap or self._accepted >= self.target_limit: - break - - title = (work.get("title") or "").strip() - if not title: - continue - - authorships = work.get("authorships", []) - - # ── Gate 2: Authorship ROR verification ────────────────────────── - if not self.institution_config.verify_ror_in_authorships(authorships): - self._rejected_gate2 += 1 - self.logger.debug(f"Gate 2 FAIL (no ROR match): {title[:60]}") - continue - - authors = [] - author_orcids = [] - authors_full = [] - affiliations = [] - author_depts = [] - - for authorship in authorships: - author_name = authorship.get("author", {}).get("display_name", "") - author_orcid = authorship.get("author", {}).get("orcid", "") - if author_orcid: - author_orcid = author_orcid.replace("https://orcid.org/", "") - - author_ror = "" - - if author_name: - authors.append(author_name) - if author_orcid: - author_orcids.append(author_orcid) - - for inst in authorship.get("institutions", []): - inst_name = inst.get("display_name", "") - inst_ror = inst.get("ror", "") - if inst_ror: - author_ror = inst_ror.replace("https://ror.org/", "") - - if inst_name: - affiliations.append(inst_name) - # Collect sub-institution if available - sub = inst.get("lineage", []) - if sub and len(sub) > 1: - author_depts.append(sub[-1]) - - authors_full.append( - {"name": author_name, "orcid": author_orcid, "ror": author_ror} - ) - - raw_affiliation = ( - " | ".join(set(affiliations)) if affiliations else self.institution_name - ) - - # ── Gate 3: Affiliation pattern matching ────────────────────────── - if affiliations and not self.institution_config.matches_affiliation( - raw_affiliation - ): - self._rejected_gate3 += 1 - self.logger.debug(f"Gate 3 FAIL (pattern mismatch): {title[:60]}") - continue - - doi = work.get("doi", "") - abstract = self._reconstruct_abstract( - work.get("abstract_inverted_index", {}) - ) - concepts = work.get("concepts", []) - dc_subject = ", ".join(c.get("display_name", "") for c in concepts[:5] if c) - - self._accepted += 1 - wave_accepted += 1 - if is_sc_wave: - self._sc_accepted += 1 - - pub_date = work.get("publication_date", "") - - pdf_url = None - oa = work.get("open_access", {}) - if oa.get("is_oa") and oa.get("oa_url"): - pdf_url = oa["oa_url"] - - url = ( - (work.get("primary_location", {}) or {}).get("landing_page_url") - or doi - or "" - ) - if not url: - url = f"https://openalex.org/{work.get('id', '').replace('https://openalex.org/', '')}" - - # Extract SDG tags from concepts - sdg_tags = self._extract_sdg_from_concepts(concepts) - - yield { - "title": title, - "abstract": abstract, - "authors": authors, - "author_orcids": author_orcids, - "authors_full": authors_full, - "doi": doi, - "url": url, - "pdf_url": pdf_url, - "publication_date": pub_date, - "source_repository": "OpenAlex", - "is_unilag_author": True, # Legacy field - "raw_affiliation": raw_affiliation, - "institution": self.institution_name, - "institution_ror": self.ror_id, - "sdg_tags": sdg_tags, - "dc_subject": ", ".join( - c.get("display_name", "") for c in concepts[:5] if c - ), - } - - # Cursor-based pagination — keep paginating within the same wave until its - # cap is hit. Reuse the originating wave's filter (extracted from current URL) - # so SC waves don't degrade back into plain ROR queries. - meta = data.get("meta", {}) - next_cursor = meta.get("next_cursor") - if ( - next_cursor - and results - and wave_accepted < wave_cap - and self._accepted < self.target_limit - ): - from urllib.parse import parse_qs, urlparse - - qs = parse_qs(urlparse(response.url).query) - current_filters = ( - qs.get("filter") or [f"institutions.ror:{self.ror_short}"] - )[0] - next_url = self._build_url(filters=current_filters, cursor=next_cursor) - yield scrapy.Request( - url=next_url, - callback=self.parse, - meta={ - "source": response.meta.get("source", "ror"), - "wave": wave, - "wave_accepted": wave_accepted, - }, - ) - - def _reconstruct_abstract(self, inverted_index: dict) -> str: - """OpenAlex stores abstracts as word→[position] inverted index.""" - if not inverted_index: - return "" - word_positions = [] - for word, positions in inverted_index.items(): - for pos in positions: - word_positions.append((pos, word)) - word_positions.sort() - return " ".join(w for _, w in word_positions) - - def _extract_sdg_from_concepts(self, concepts: list) -> str: - """Map OpenAlex concepts to SDG numbers (rough heuristic).""" - sdg_concept_map = { - "Poverty": 1, - "Food security": 2, - "Health": 3, - "Medicine": 3, - "Education": 4, - "Gender studies": 5, - "Water resources": 6, - "Renewable energy": 7, - "Economic growth": 8, - "Engineering": 9, - "Inequality": 10, - "Urban planning": 11, - "Sustainability": 12, - "Climate change": 13, - "Marine biology": 14, - "Ecology": 15, - "Political science": 16, - "International development": 17, - } - matched_sdgs = set() - for concept in concepts: - name = concept.get("display_name", "") - for key, sdg_num in sdg_concept_map.items(): - if key.lower() in name.lower(): - matched_sdgs.add(str(sdg_num)) - return ",".join(sorted(matched_sdgs)) - - def closed(self, reason): - self.logger.info( - f"Spider closed: {self.institution_name} | " - f"Accepted: {self._accepted} (SC-wave: {self._sc_accepted}) | " - f"Rejected (gate2/ROR): {self._rejected_gate2} | " - f"Rejected (gate3/pattern): {self._rejected_gate3}" - ) - total_seen = self._accepted + self._rejected_gate2 + self._rejected_gate3 - if total_seen > 0: - precision = round(self._accepted / total_seen * 100, 1) - self.logger.info(f"Precision: {precision}%") +import logging +import os +import sys + +import scrapy + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) + +from uraas.config.institutions import get_registry +from uraas.config.special_collections import SC_OPENALEX_CONCEPTS, SC_SEED_KEYWORDS + +OPENALEX_BASE = "https://api.openalex.org/works" +MAILTO = "cokiki@unilag.edu.ng" + +log = logging.getLogger(__name__) + + +class OpenAlexSpider(scrapy.Spider): + """ + OpenAlex spider with 3-gate precision for 98% crawl accuracy. + + Gate 1: ROR-filtered API query (only papers from institution's ROR) + Gate 2: Per-paper authorship ROR verification (at least 1 author has target ROR) + Gate 3: Affiliation string pattern matching (belt-and-suspenders) + + Papers failing any gate are dropped — never mixed across institutions. + """ + + name = "openalex_multi" + custom_settings = { + # OpenAlex polite pool: ~10 req/s per IP. When multiple institution + # spiders run in parallel they share the IP budget, so we need a + # meaningful delay and generous autothrottle ceiling so 429s are + # absorbed gracefully rather than exhausting all retries. + "DOWNLOAD_DELAY": 2.0, + "AUTOTHROTTLE_ENABLED": True, + "AUTOTHROTTLE_START_DELAY": 2.0, + "AUTOTHROTTLE_MAX_DELAY": 60.0, + "AUTOTHROTTLE_TARGET_CONCURRENCY": 1, + "CONCURRENT_REQUESTS": 1, + "RETRY_ENABLED": True, + "RETRY_TIMES": 5, + "RETRY_HTTP_CODES": [429, 500, 502, 503, 504], + "HTTPERROR_ALLOWED_CODES": [429], + } + + def __init__( + self, + institution="unilag", + target=20, + boost_special=True, + sc_only=False, + *args, + **kwargs, + ): + """ + institution: registry short name (e.g. "unilag") + target: max SC papers to accept this run + boost_special: also run SC seed waves (topic+ROR) in addition to the + general ROR wave — keeps SC recall high (default ON) + sc_only: skip the general ROR wave and run ONLY SC seed waves; + use when you only want targeted SC discovery, no noise + """ + super().__init__(*args, **kwargs) + self.target_limit = int(target) + # Accept both bool and string ("true"/"false") — Scrapy passes CLI + # spider args as strings when launched via crawl_multi_institution.py. + _truthy = {"1", "true", "yes", "on"} + if isinstance(boost_special, str): + self.boost_special = boost_special.lower() in _truthy + else: + self.boost_special = bool(boost_special) + if isinstance(sc_only, str): + self.sc_only = sc_only.lower() in _truthy + else: + self.sc_only = bool(sc_only) + registry = get_registry() + self.institution_config = registry.get(institution) + if not self.institution_config: + raise ValueError(f"Institution '{institution}' not found in registry") + + self.institution_name = self.institution_config.name + self.ror_id = self.institution_config.ror + self.ror_short = self.ror_id.split("/")[-1] + + self._accepted = 0 + self._rejected_gate2 = 0 + self._rejected_gate3 = 0 + self._sc_accepted = 0 + + self.logger.info( + f"OpenAlex spider for {self.institution_name} | ROR: {self.ror_short} " + f"| boost_special={self.boost_special} | sc_only={self.sc_only}" + ) + + SELECT_FIELDS = ( + "id,doi,title,abstract_inverted_index,authorships," + "publication_date,open_access,primary_location,concepts" + ) + + def _build_url(self, *, filters: str, cursor: str = "*") -> str: + return ( + f"{OPENALEX_BASE}" + f"?filter={filters}" + f"&select={self.SELECT_FIELDS}" + f"&per-page=200" + f"&cursor={cursor}" + f"&mailto={MAILTO}" + ) + + async def start(self): + # Wave 1 — general ROR-only crawl (skipped in sc_only mode) + if not self.sc_only: + url = self._build_url(filters=f"institutions.ror:{self.ror_short}") + self.logger.info(f"[ROR wave] {url}") + yield scrapy.Request( + url=url, + callback=self.parse, + meta={"source": "ror", "wave": "ror"}, + priority=0, + ) + + # Wave 2 — SC-boosted waves: one request per SC seed phrase, AND-ed with ROR. + # OpenAlex combines filters with comma=AND. The valid free-text filter is + # title_and_abstract.search (concepts.display_name.search is not supported — + # only concepts.id is). We rely on free-text seeds; the in-pipeline classifier + # then scores the actual hits. + if self.boost_special: + seeds = set(SC_SEED_KEYWORDS) + for seed in sorted(seeds): + seed_q = seed.replace(" ", "%20") + filters = ( + f"institutions.ror:{self.ror_short}," + f"title_and_abstract.search:{seed_q}" + ) + url = self._build_url(filters=filters) + self.logger.info(f"[SC wave seed={seed!r}] {url}") + yield scrapy.Request( + url=url, + callback=self.parse, + meta={"source": "ror+seed", "wave": f"sc:{seed}"}, + priority=10, # Prioritize SC papers to fill target first + ) + + def parse(self, response): + assert self.institution_config is not None, "Institution config must be loaded" + + if response.status == 429: + self.logger.warning( + "OpenAlex rate-limited (429) on wave=%s — Scrapy retry will back off", + response.meta.get("wave", "?"), + ) + return + + # Hard stop if we've already reached the global target + if self._accepted >= self.target_limit: + return + + wave = response.meta.get("wave", "ror") + is_sc_wave = wave.startswith("sc:") + # Every wave respects the global target limit. No more 10x headroom. + wave_cap = self.target_limit + + data = response.json() + results = data.get("results", []) + self.logger.info(f"[{wave}] received {len(results)} works") + + wave_accepted = response.meta.get("wave_accepted", 0) + + for work in results: + # Check both wave-local cap and global target limit + if wave_accepted >= wave_cap or self._accepted >= self.target_limit: + break + + title = (work.get("title") or "").strip() + if not title: + continue + + authorships = work.get("authorships", []) + + # ── Gate 2: Authorship ROR verification ────────────────────────── + if not self.institution_config.verify_ror_in_authorships(authorships): + self._rejected_gate2 += 1 + self.logger.debug(f"Gate 2 FAIL (no ROR match): {title[:60]}") + continue + + authors = [] + author_orcids = [] + authors_full = [] + affiliations = [] + author_depts = [] + + for authorship in authorships: + author_name = authorship.get("author", {}).get("display_name", "") + author_orcid = authorship.get("author", {}).get("orcid", "") + if author_orcid: + author_orcid = author_orcid.replace("https://orcid.org/", "") + + author_ror = "" + + if author_name: + authors.append(author_name) + if author_orcid: + author_orcids.append(author_orcid) + + for inst in authorship.get("institutions", []): + inst_name = inst.get("display_name", "") + inst_ror = inst.get("ror", "") + if inst_ror: + author_ror = inst_ror.replace("https://ror.org/", "") + + if inst_name: + affiliations.append(inst_name) + # Collect sub-institution if available + sub = inst.get("lineage", []) + if sub and len(sub) > 1: + author_depts.append(sub[-1]) + + authors_full.append( + {"name": author_name, "orcid": author_orcid, "ror": author_ror} + ) + + raw_affiliation = ( + " | ".join(set(affiliations)) if affiliations else self.institution_name + ) + + # ── Gate 3: Affiliation pattern matching ────────────────────────── + if affiliations and not self.institution_config.matches_affiliation( + raw_affiliation + ): + self._rejected_gate3 += 1 + self.logger.debug(f"Gate 3 FAIL (pattern mismatch): {title[:60]}") + continue + + doi = work.get("doi", "") + abstract = self._reconstruct_abstract( + work.get("abstract_inverted_index", {}) + ) + concepts = work.get("concepts", []) + dc_subject = ", ".join(c.get("display_name", "") for c in concepts[:5] if c) + + self._accepted += 1 + wave_accepted += 1 + if is_sc_wave: + self._sc_accepted += 1 + + pub_date = work.get("publication_date", "") + + pdf_url = None + oa = work.get("open_access", {}) + if oa.get("is_oa") and oa.get("oa_url"): + pdf_url = oa["oa_url"] + + url = ( + (work.get("primary_location", {}) or {}).get("landing_page_url") + or doi + or "" + ) + if not url: + url = f"https://openalex.org/{work.get('id', '').replace('https://openalex.org/', '')}" + + # Extract SDG tags from concepts + sdg_tags = self._extract_sdg_from_concepts(concepts) + + yield { + "title": title, + "abstract": abstract, + "authors": authors, + "author_orcids": author_orcids, + "authors_full": authors_full, + "doi": doi, + "url": url, + "pdf_url": pdf_url, + "publication_date": pub_date, + "source_repository": "OpenAlex", + "is_unilag_author": True, # Legacy field + "raw_affiliation": raw_affiliation, + "institution": self.institution_name, + "institution_ror": self.ror_id, + "sdg_tags": sdg_tags, + "dc_subject": ", ".join( + c.get("display_name", "") for c in concepts[:5] if c + ), + } + + # Cursor-based pagination — keep paginating within the same wave until its + # cap is hit. Reuse the originating wave's filter (extracted from current URL) + # so SC waves don't degrade back into plain ROR queries. + meta = data.get("meta", {}) + next_cursor = meta.get("next_cursor") + if ( + next_cursor + and results + and wave_accepted < wave_cap + and self._accepted < self.target_limit + ): + from urllib.parse import parse_qs, urlparse + + qs = parse_qs(urlparse(response.url).query) + current_filters = ( + qs.get("filter") or [f"institutions.ror:{self.ror_short}"] + )[0] + next_url = self._build_url(filters=current_filters, cursor=next_cursor) + yield scrapy.Request( + url=next_url, + callback=self.parse, + meta={ + "source": response.meta.get("source", "ror"), + "wave": wave, + "wave_accepted": wave_accepted, + }, + ) + + def _reconstruct_abstract(self, inverted_index: dict) -> str: + """OpenAlex stores abstracts as word→[position] inverted index.""" + if not inverted_index: + return "" + word_positions = [] + for word, positions in inverted_index.items(): + for pos in positions: + word_positions.append((pos, word)) + word_positions.sort() + return " ".join(w for _, w in word_positions) + + def _extract_sdg_from_concepts(self, concepts: list) -> str: + """Map OpenAlex concepts to SDG numbers (rough heuristic).""" + sdg_concept_map = { + "Poverty": 1, + "Food security": 2, + "Health": 3, + "Medicine": 3, + "Education": 4, + "Gender studies": 5, + "Water resources": 6, + "Renewable energy": 7, + "Economic growth": 8, + "Engineering": 9, + "Inequality": 10, + "Urban planning": 11, + "Sustainability": 12, + "Climate change": 13, + "Marine biology": 14, + "Ecology": 15, + "Political science": 16, + "International development": 17, + } + matched_sdgs = set() + for concept in concepts: + name = concept.get("display_name", "") + for key, sdg_num in sdg_concept_map.items(): + if key.lower() in name.lower(): + matched_sdgs.add(str(sdg_num)) + return ",".join(sorted(matched_sdgs)) + + def closed(self, reason): + self.logger.info( + f"Spider closed: {self.institution_name} | " + f"Accepted: {self._accepted} (SC-wave: {self._sc_accepted}) | " + f"Rejected (gate2/ROR): {self._rejected_gate2} | " + f"Rejected (gate3/pattern): {self._rejected_gate3}" + ) + total_seen = self._accepted + self._rejected_gate2 + self._rejected_gate3 + if total_seen > 0: + precision = round(self._accepted / total_seen * 100, 1) + self.logger.info(f"Precision: {precision}%") diff --git a/uraas/spiders/sources/orcid_spider.py b/uraas/spiders/sources/orcid_spider.py index d5f627cf5711ddcdb0519fad0a3b8188a0b60928..086adcd26886ac6aa13eded9b18e4d74ad20b37d 100644 --- a/uraas/spiders/sources/orcid_spider.py +++ b/uraas/spiders/sources/orcid_spider.py @@ -1,134 +1,134 @@ -""" -ORCID Spider - Harvests papers using ORCID IDs from rich staff data. -Loads staff records with ORCID from {inst}_staff.json. -No arbitrary limits — crawls all staff with ORCIDs. -""" - -import json -import os -import sys - -import scrapy - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) -from uraas.config.institutions import get_registry - - -class ORCIDSpider(scrapy.Spider): - """Harvests papers from ORCID for all staff members with ORCID IDs.""" - - name = "orcid_multi" - custom_settings = { - "DOWNLOAD_DELAY": 2.0, - "RETRY_ENABLED": True, - "RETRY_TIMES": 3, - "CONCURRENT_REQUESTS": 2, - } - - def __init__(self, institution="unilag", *args, **kwargs): - super().__init__(*args, **kwargs) - registry = get_registry() - self.institution_config = registry.get(institution) - if not self.institution_config: - raise ValueError(f"Institution '{institution}' not found in registry") - self.institution_name = self.institution_config.name - self.ror_id = self.institution_config.ror - self.logger.info( - f"ORCID spider for {self.institution_name} | " - f"{len(self.institution_config.staff_with_orcid)} staff with ORCIDs" - ) - - async def start(self): - """Query ORCID API for each staff member that has an ORCID.""" - staff_with_orcid = self.institution_config.staff_with_orcid - if not staff_with_orcid: - self.logger.warning( - f"No staff with ORCID IDs found for {self.institution_name}. " - f"Run scripts/harvest_staff_openalex.py first." - ) - return - - self.logger.info(f"Querying ORCID API for {len(staff_with_orcid)} researchers") - for staff_member in staff_with_orcid: - orcid_id = staff_member["orcid"] - if not orcid_id: - continue - url = f"https://pub.orcid.org/v3.0/{orcid_id}/works" - yield scrapy.Request( - url=url, - callback=self.parse_works, - headers={"Accept": "application/json"}, - meta={ - "orcid": orcid_id, - "name": staff_member["name"], - "department": staff_member.get("department", ""), - "faculty": staff_member.get("faculty", ""), - }, - errback=self.errback_handler, - ) - - def errback_handler(self, failure): - self.logger.error(f"Request failed: {failure.request.url}") - - def parse_works(self, response): - """Parse works from ORCID API response.""" - orcid = response.meta["orcid"] - name = response.meta["name"] - department = response.meta.get("department", "") - faculty = response.meta.get("faculty", "") - - try: - data = response.json() - works = data.get("group", []) - self.logger.info(f"Found {len(works)} works for {name} (ORCID: {orcid})") - - for work_group in works: - work_summary_list = work_group.get("work-summary", []) - if not work_summary_list: - continue - work = work_summary_list[0] - - title_data = work.get("title", {}) - title = (title_data.get("title", {}) or {}).get("value", "").strip() - if not title: - continue - - # Get DOI from external IDs - doi = None - url = None - for ext_id in (work.get("external-ids", {}) or {}).get( - "external-id", [] - ): - if ext_id.get("external-id-type") == "doi": - doi = ext_id.get("external-id-value", "").strip() - if doi: - url = f"https://doi.org/{doi}" - break - - # Publication date - pub_date_obj = work.get("publication-date") or {} - pub_year = (pub_date_obj.get("year", {}) or {}).get("value") - pub_date = f"{pub_year}-01-01" if pub_year else None - - journal = (work.get("journal-title", {}) or {}).get("value", "") - - yield { - "title": title, - "authors": [name], - "author_orcids": [orcid], - "doi": doi, - "url": url or f"https://orcid.org/{orcid}", - "source_repository": "ORCID", - "is_unilag_author": True, # Legacy field - "raw_affiliation": self.institution_name, - "orcid": orcid, - "publication_date": pub_date, - "journal": journal, - "abstract": "", - "institution": self.institution_name, - "institution_ror": self.ror_id, - "department": department, - "faculty": faculty, - } - except Exception as e: - self.logger.error(f"Error parsing works for {name}: {e}") +""" +ORCID Spider - Harvests papers using ORCID IDs from rich staff data. +Loads staff records with ORCID from {inst}_staff.json. +No arbitrary limits — crawls all staff with ORCIDs. +""" + +import json +import os +import sys + +import scrapy + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) +from uraas.config.institutions import get_registry + + +class ORCIDSpider(scrapy.Spider): + """Harvests papers from ORCID for all staff members with ORCID IDs.""" + + name = "orcid_multi" + custom_settings = { + "DOWNLOAD_DELAY": 2.0, + "RETRY_ENABLED": True, + "RETRY_TIMES": 3, + "CONCURRENT_REQUESTS": 2, + } + + def __init__(self, institution="unilag", *args, **kwargs): + super().__init__(*args, **kwargs) + registry = get_registry() + self.institution_config = registry.get(institution) + if not self.institution_config: + raise ValueError(f"Institution '{institution}' not found in registry") + self.institution_name = self.institution_config.name + self.ror_id = self.institution_config.ror + self.logger.info( + f"ORCID spider for {self.institution_name} | " + f"{len(self.institution_config.staff_with_orcid)} staff with ORCIDs" + ) + + async def start(self): + """Query ORCID API for each staff member that has an ORCID.""" + staff_with_orcid = self.institution_config.staff_with_orcid + if not staff_with_orcid: + self.logger.warning( + f"No staff with ORCID IDs found for {self.institution_name}. " + f"Run scripts/harvest_staff_openalex.py first." + ) + return + + self.logger.info(f"Querying ORCID API for {len(staff_with_orcid)} researchers") + for staff_member in staff_with_orcid: + orcid_id = staff_member["orcid"] + if not orcid_id: + continue + url = f"https://pub.orcid.org/v3.0/{orcid_id}/works" + yield scrapy.Request( + url=url, + callback=self.parse_works, + headers={"Accept": "application/json"}, + meta={ + "orcid": orcid_id, + "name": staff_member["name"], + "department": staff_member.get("department", ""), + "faculty": staff_member.get("faculty", ""), + }, + errback=self.errback_handler, + ) + + def errback_handler(self, failure): + self.logger.error(f"Request failed: {failure.request.url}") + + def parse_works(self, response): + """Parse works from ORCID API response.""" + orcid = response.meta["orcid"] + name = response.meta["name"] + department = response.meta.get("department", "") + faculty = response.meta.get("faculty", "") + + try: + data = response.json() + works = data.get("group", []) + self.logger.info(f"Found {len(works)} works for {name} (ORCID: {orcid})") + + for work_group in works: + work_summary_list = work_group.get("work-summary", []) + if not work_summary_list: + continue + work = work_summary_list[0] + + title_data = work.get("title", {}) + title = (title_data.get("title", {}) or {}).get("value", "").strip() + if not title: + continue + + # Get DOI from external IDs + doi = None + url = None + for ext_id in (work.get("external-ids", {}) or {}).get( + "external-id", [] + ): + if ext_id.get("external-id-type") == "doi": + doi = ext_id.get("external-id-value", "").strip() + if doi: + url = f"https://doi.org/{doi}" + break + + # Publication date + pub_date_obj = work.get("publication-date") or {} + pub_year = (pub_date_obj.get("year", {}) or {}).get("value") + pub_date = f"{pub_year}-01-01" if pub_year else None + + journal = (work.get("journal-title", {}) or {}).get("value", "") + + yield { + "title": title, + "authors": [name], + "author_orcids": [orcid], + "doi": doi, + "url": url or f"https://orcid.org/{orcid}", + "source_repository": "ORCID", + "is_unilag_author": True, # Legacy field + "raw_affiliation": self.institution_name, + "orcid": orcid, + "publication_date": pub_date, + "journal": journal, + "abstract": "", + "institution": self.institution_name, + "institution_ror": self.ror_id, + "department": department, + "faculty": faculty, + } + except Exception as e: + self.logger.error(f"Error parsing works for {name}: {e}") diff --git a/uraas/spiders/sources/pubmed_spider.py b/uraas/spiders/sources/pubmed_spider.py index 4befb9f482fa5e87a0836b05869a9ca03d6babe8..ba9c4284ee0b08bfa2ed23e9d6837556690404da 100644 --- a/uraas/spiders/sources/pubmed_spider.py +++ b/uraas/spiders/sources/pubmed_spider.py @@ -1,149 +1,149 @@ -""" -PubMed/NCBI spider — queries the NCBI E-utilities API. - -PubMed indexes 37M+ biomedical papers. For URAAS Special Collections it is the -single best source for: - • Ethnobotany & traditional plant medicine (Indigenous Knowledge category) - • Traditional healing practices - • Ethno-pharmacology - • Community health using indigenous methods - -No API key required (1 req/3s). With a free NCBI API key (ncbi.nlm.nih.gov/account/) -the rate limit rises to 10 req/s. Set NCBI_API_KEY in .env. -""" - -import os -import sys -from urllib.parse import urlencode - -import scrapy - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) - -from uraas.config import config -from uraas.config.institutions import get_registry - -_ESEARCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi" -_EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" -_BATCH = 100 - -# Seeds specifically relevant to ethnobotany / traditional medicine SC category -_PUBMED_SEEDS = [ - "ethnobotany", "traditional medicine", "medicinal plants", - "indigenous knowledge", "traditional healing", "folk medicine", - "ethnopharmacology", "phytomedicine", "herbal medicine", - "traditional ecological knowledge", -] - - -class PubMedSpider(scrapy.Spider): - name = "pubmed" - custom_settings = { - "DOWNLOAD_DELAY": 0.4, # NCBI etiquette: max 3 req/s without key, 10 with - "CONCURRENT_REQUESTS": 1, - "USER_AGENT": f"URAAS/1.0 (+SC discovery; mailto:{config.OPENALEX_MAILTO})", - } - - def __init__(self, institution="unilag", target=50, *args, **kwargs): - super().__init__(*args, **kwargs) - self.target_limit = int(target) - self.api_key = getattr(config, "NCBI_API_KEY", "") or os.environ.get("NCBI_API_KEY", "") - - registry = get_registry() - self.institution_config = registry.get(institution) - if not self.institution_config: - raise ValueError(f"Institution '{institution}' not found") - self.institution_name = self.institution_config.name - self.ror_id = self.institution_config.ror - self._accepted = 0 - - def _affil_term(self) -> str: - return f'"{self.institution_name}"[Affiliation]' - - def _esearch_url(self, seed: str, retstart: int = 0) -> str: - term = f'{self._affil_term()} AND "{seed}"[Title/Abstract]' - params = { - "db": "pubmed", "term": term, "retmax": _BATCH, - "retstart": retstart, "retmode": "json", - "tool": "URAAS", "email": config.OPENALEX_MAILTO, - } - if self.api_key: - params["api_key"] = self.api_key - return f"{_ESEARCH}?{urlencode(params)}" - - def _efetch_url(self, ids: list) -> str: - params = { - "db": "pubmed", "id": ",".join(ids), "rettype": "abstract", - "retmode": "xml", "tool": "URAAS", "email": config.OPENALEX_MAILTO, - } - if self.api_key: - params["api_key"] = self.api_key - return f"{_EFETCH}?{urlencode(params)}" - - async def start(self): - for seed in _PUBMED_SEEDS: - if self._accepted >= self.target_limit: - break - url = self._esearch_url(seed) - yield scrapy.Request(url=url, callback=self.parse_search, - meta={"seed": seed, "retstart": 0}) - - def parse_search(self, response): - if self._accepted >= self.target_limit: - return - data = response.json() - ids = data.get("esearchresult", {}).get("idlist", []) - if not ids: - return - seed = response.meta["seed"] - fetch_url = self._efetch_url(ids) - yield scrapy.Request(url=fetch_url, callback=self.parse_fetch, meta={"seed": seed}) - - total = int(data.get("esearchresult", {}).get("count", 0)) - retstart = response.meta["retstart"] + _BATCH - if retstart < min(total, 500) and self._accepted < self.target_limit: - next_url = self._esearch_url(seed, retstart) - yield scrapy.Request(url=next_url, callback=self.parse_search, - meta={"seed": seed, "retstart": retstart}) - - def parse_fetch(self, response): - if self._accepted >= self.target_limit: - return - # PubMed efetch returns XML; parse with Scrapy's Selector - from scrapy import Selector - sel = Selector(text=response.text, type="xml") - - for art in sel.xpath("//PubmedArticle"): - if self._accepted >= self.target_limit: - return - title = (art.xpath(".//ArticleTitle//text()").getall() or [""]) - title = " ".join(title).strip() - if not title: - continue - - abstract = " ".join(art.xpath(".//AbstractText//text()").getall()).strip() - - authors = [] - for auth in art.xpath(".//Author"): - last = auth.xpath("LastName/text()").get("") - fore = auth.xpath("ForeName/text()").get("") - if last: - authors.append(f"{fore} {last}".strip()) - - pmid = art.xpath(".//PMID/text()").get("") - doi = art.xpath(".//ELocationID[@EIdType='doi']/text()").get("") - - year = ( - art.xpath(".//PubDate/Year/text()").get("") - or art.xpath(".//PubDate/MedlineDate/text()").get("")[:4] - ) - - self._accepted += 1 - yield { - "title": title, "abstract": abstract, "authors": authors, "doi": doi, - "url": f"https://doi.org/{doi}" if doi else f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/", - "pdf_url": None, "publication_date": str(year), - "source_repository": "PubMed", "is_unilag_author": True, - "raw_affiliation": self.institution_name, "institution": self.institution_name, - "institution_ror": self.ror_id, - } +""" +PubMed/NCBI spider — queries the NCBI E-utilities API. + +PubMed indexes 37M+ biomedical papers. For URAAS Special Collections it is the +single best source for: + • Ethnobotany & traditional plant medicine (Indigenous Knowledge category) + • Traditional healing practices + • Ethno-pharmacology + • Community health using indigenous methods + +No API key required (1 req/3s). With a free NCBI API key (ncbi.nlm.nih.gov/account/) +the rate limit rises to 10 req/s. Set NCBI_API_KEY in .env. +""" + +import os +import sys +from urllib.parse import urlencode + +import scrapy + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) + +from uraas.config import config +from uraas.config.institutions import get_registry + +_ESEARCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi" +_EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" +_BATCH = 100 + +# Seeds specifically relevant to ethnobotany / traditional medicine SC category +_PUBMED_SEEDS = [ + "ethnobotany", "traditional medicine", "medicinal plants", + "indigenous knowledge", "traditional healing", "folk medicine", + "ethnopharmacology", "phytomedicine", "herbal medicine", + "traditional ecological knowledge", +] + + +class PubMedSpider(scrapy.Spider): + name = "pubmed" + custom_settings = { + "DOWNLOAD_DELAY": 0.4, # NCBI etiquette: max 3 req/s without key, 10 with + "CONCURRENT_REQUESTS": 1, + "USER_AGENT": f"URAAS/1.0 (+SC discovery; mailto:{config.OPENALEX_MAILTO})", + } + + def __init__(self, institution="unilag", target=50, *args, **kwargs): + super().__init__(*args, **kwargs) + self.target_limit = int(target) + self.api_key = getattr(config, "NCBI_API_KEY", "") or os.environ.get("NCBI_API_KEY", "") + + registry = get_registry() + self.institution_config = registry.get(institution) + if not self.institution_config: + raise ValueError(f"Institution '{institution}' not found") + self.institution_name = self.institution_config.name + self.ror_id = self.institution_config.ror + self._accepted = 0 + + def _affil_term(self) -> str: + return f'"{self.institution_name}"[Affiliation]' + + def _esearch_url(self, seed: str, retstart: int = 0) -> str: + term = f'{self._affil_term()} AND "{seed}"[Title/Abstract]' + params = { + "db": "pubmed", "term": term, "retmax": _BATCH, + "retstart": retstart, "retmode": "json", + "tool": "URAAS", "email": config.OPENALEX_MAILTO, + } + if self.api_key: + params["api_key"] = self.api_key + return f"{_ESEARCH}?{urlencode(params)}" + + def _efetch_url(self, ids: list) -> str: + params = { + "db": "pubmed", "id": ",".join(ids), "rettype": "abstract", + "retmode": "xml", "tool": "URAAS", "email": config.OPENALEX_MAILTO, + } + if self.api_key: + params["api_key"] = self.api_key + return f"{_EFETCH}?{urlencode(params)}" + + async def start(self): + for seed in _PUBMED_SEEDS: + if self._accepted >= self.target_limit: + break + url = self._esearch_url(seed) + yield scrapy.Request(url=url, callback=self.parse_search, + meta={"seed": seed, "retstart": 0}) + + def parse_search(self, response): + if self._accepted >= self.target_limit: + return + data = response.json() + ids = data.get("esearchresult", {}).get("idlist", []) + if not ids: + return + seed = response.meta["seed"] + fetch_url = self._efetch_url(ids) + yield scrapy.Request(url=fetch_url, callback=self.parse_fetch, meta={"seed": seed}) + + total = int(data.get("esearchresult", {}).get("count", 0)) + retstart = response.meta["retstart"] + _BATCH + if retstart < min(total, 500) and self._accepted < self.target_limit: + next_url = self._esearch_url(seed, retstart) + yield scrapy.Request(url=next_url, callback=self.parse_search, + meta={"seed": seed, "retstart": retstart}) + + def parse_fetch(self, response): + if self._accepted >= self.target_limit: + return + # PubMed efetch returns XML; parse with Scrapy's Selector + from scrapy import Selector + sel = Selector(text=response.text, type="xml") + + for art in sel.xpath("//PubmedArticle"): + if self._accepted >= self.target_limit: + return + title = (art.xpath(".//ArticleTitle//text()").getall() or [""]) + title = " ".join(title).strip() + if not title: + continue + + abstract = " ".join(art.xpath(".//AbstractText//text()").getall()).strip() + + authors = [] + for auth in art.xpath(".//Author"): + last = auth.xpath("LastName/text()").get("") + fore = auth.xpath("ForeName/text()").get("") + if last: + authors.append(f"{fore} {last}".strip()) + + pmid = art.xpath(".//PMID/text()").get("") + doi = art.xpath(".//ELocationID[@EIdType='doi']/text()").get("") + + year = ( + art.xpath(".//PubDate/Year/text()").get("") + or art.xpath(".//PubDate/MedlineDate/text()").get("")[:4] + ) + + self._accepted += 1 + yield { + "title": title, "abstract": abstract, "authors": authors, "doi": doi, + "url": f"https://doi.org/{doi}" if doi else f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/", + "pdf_url": None, "publication_date": str(year), + "source_repository": "PubMed", "is_unilag_author": True, + "raw_affiliation": self.institution_name, "institution": self.institution_name, + "institution_ror": self.ror_id, + } diff --git a/uraas/spiders/sources/semantic_scholar_spider.py b/uraas/spiders/sources/semantic_scholar_spider.py index 9f0d8774d4690736c5225adb3ffbcf6ee8d66e39..852451f79541cabf3fa003e52e8fecd2d2dc9beb 100644 --- a/uraas/spiders/sources/semantic_scholar_spider.py +++ b/uraas/spiders/sources/semantic_scholar_spider.py @@ -1,238 +1,238 @@ -""" -Semantic Scholar spider — queries the free S2 Graph API. - -Semantic Scholar (semanticscholar.org) has broader humanities and social-science -coverage than arXiv, including African studies, philosophy, cultural heritage, and -postcolonial literature — exactly the SC categories URAAS cares about. The API is -free (no key required for basic use; 100 reqs/5 min per IP). - -The spider runs two types of waves: - • SC seed waves — institution + each SC seed phrase (e.g. "indigenous knowledge") - • General wave — institution name alone (catches SC hits missed by seeds) - -The SC classifier in the pipeline gates what actually gets saved. -""" - -import os -import sys - -import scrapy - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) - -from uraas.config import config -from uraas.config.institutions import get_registry -from uraas.config.special_collections import SC_SEED_KEYWORDS - -_S2_BASE = "https://api.semanticscholar.org/graph/v1/paper/search" -_FIELDS = "title,abstract,authors,year,externalIds,openAccessPdf,fieldsOfStudy,venue" -_LIMIT = 100 - - -class SemanticScholarSpider(scrapy.Spider): - name = "semantic_scholar" - custom_settings = { - # S2 free tier: 100 req / 5 min per IP (~1 req/3 sec sustained). - # When running alongside other spiders the shared IP exhausts the - # budget quickly, so we use a conservative 3 s delay + autothrottle - # with a generous max to absorb 429 bursts. - "DOWNLOAD_DELAY": 3.0, - "CONCURRENT_REQUESTS": 1, - "AUTOTHROTTLE_ENABLED": True, - "AUTOTHROTTLE_START_DELAY": 3.0, - "AUTOTHROTTLE_MAX_DELAY": 30.0, - "AUTOTHROTTLE_TARGET_CONCURRENCY": 1, - "RETRY_ENABLED": True, - "RETRY_TIMES": 3, - "RETRY_HTTP_CODES": [429, 500, 502, 503, 504], - "USER_AGENT": ( - f"URAAS/1.0 (+read-only SC discovery; mailto:{config.OPENALEX_MAILTO})" - ), - "HTTPERROR_ALLOWED_CODES": [429], - } - - def __init__( - self, - institution="unilag", - target=50, - boost_special=True, - sc_only=False, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - self.target_limit = int(target) - _truthy = {"1", "true", "yes", "on"} - self.boost_special = ( - boost_special.lower() in _truthy - if isinstance(boost_special, str) - else bool(boost_special) - ) - self.sc_only = ( - sc_only.lower() in _truthy - if isinstance(sc_only, str) - else bool(sc_only) - ) - registry = get_registry() - self.institution_config = registry.get(institution) - if not self.institution_config: - raise ValueError(f"Institution '{institution}' not found in registry") - self.institution_name = self.institution_config.name - self.ror_id = self.institution_config.ror - self._accepted = 0 - - self.logger.info( - "S2 spider | %s | boost_special=%s | sc_only=%s | target=%d", - self.institution_name, - self.boost_special, - self.sc_only, - self.target_limit, - ) - - def _build_url(self, query: str, offset: int = 0) -> str: - import urllib.parse - q = urllib.parse.quote(query) - return ( - f"{_S2_BASE}?query={q}" - f"&fields={_FIELDS}" - f"&limit={_LIMIT}" - f"&offset={offset}" - ) - - def _institution_match(self, title: str, abstract: str, venue: str) -> bool: - """ - S2 basic search doesn't return author affiliations, so we verify the - institution appears anywhere in the available text fields. This is - intentionally lenient — a false negative (dropping a valid paper) - is preferable to a false positive (storing a paper from the wrong - university). - """ - combined = f"{title} {abstract} {venue}".lower() - return any( - pat.lower() in combined - for pat in self.institution_config.affiliation_patterns - ) - - async def start(self): - if not self.sc_only: - url = self._build_url(self.institution_name) - yield scrapy.Request( - url=url, - callback=self.parse, - meta={"wave": "general", "query": self.institution_name, "offset": 0}, - ) - - if self.boost_special: - # Cap at 8 highest-signal SC seeds to avoid exhausting the S2 - # 100 req/5 min budget before we get any results. - _TOP_SEEDS = [ - "indigenous knowledge", "oral tradition", "african literature", - "cultural heritage", "postcolonial", "ethnomusicology", - "pan-african", "traditional medicine", - ] - for seed in _TOP_SEEDS: - query = f"{self.institution_name} {seed}" - url = self._build_url(query) - yield scrapy.Request( - url=url, - callback=self.parse, - meta={"wave": f"sc:{seed}", "query": query, "offset": 0}, - priority=10, - ) - - def parse(self, response): - if response.status == 429: - self.logger.warning("S2 rate-limited (429) — request will be retried by Scrapy") - return - if self._accepted >= self.target_limit: - return - - data = response.json() - papers = data.get("data", []) - wave = response.meta.get("wave", "general") - self.logger.info("[S2:%s] received %d papers", wave, len(papers)) - - rejected_aff = 0 - for paper in papers: - if self._accepted >= self.target_limit: - return - - title = (paper.get("title") or "").strip() - if not title: - continue - - abstract = (paper.get("abstract") or "").strip() - venue = (paper.get("venue") or "").strip() - year = paper.get("year") - pub_date = f"{year}-01-01" if year else "" - - ext_ids = paper.get("externalIds") or {} - doi = ext_ids.get("DOI") or ext_ids.get("doi") or "" - arxiv_id = ext_ids.get("ArXiv") or "" - - oa = paper.get("openAccessPdf") or {} - pdf_url = oa.get("url") if oa else None - - url_val = ( - f"https://doi.org/{doi}" if doi - else (f"https://arxiv.org/abs/{arxiv_id}" if arxiv_id else "") - ) - - authors = [ - a.get("name", "") for a in (paper.get("authors") or []) if a.get("name") - ] - - fields = [ - f.get("category", "") for f in (paper.get("fieldsOfStudy") or []) if f.get("category") - ] - dc_subject = ", ".join(fields[:6]) - - # S2 doesn't return author affiliations in basic search — verify - # that the institution name appears somewhere in the paper data. - if not self._institution_match(title, abstract, venue): - rejected_aff += 1 - self.logger.debug(f"S2 aff FAIL: {title[:60]}") - continue - - self._accepted += 1 - yield { - "title": title, - "abstract": abstract, - "authors": authors, - "doi": doi, - "url": url_val, - "pdf_url": pdf_url, - "publication_date": pub_date, - "source_repository": "Semantic Scholar", - "is_unilag_author": True, - "raw_affiliation": self.institution_name, - "institution": self.institution_name, - "institution_ror": self.ror_id, - "dc_subject": dc_subject, - } - - # Offset-based pagination - total = data.get("total", 0) - offset = response.meta.get("offset", 0) + _LIMIT - if offset < min(total, 500) and self._accepted < self.target_limit: - query = response.meta["query"] - wave = response.meta["wave"] - next_url = self._build_url(query, offset) - yield scrapy.Request( - url=next_url, - callback=self.parse, - meta={"wave": wave, "query": query, "offset": offset}, - ) - - def closed(self, reason): - self.logger.info( - "S2 spider closed | %s | accepted=%d | reason=%s", - self.institution_name, - self._accepted, - reason, - ) - if self._accepted == 0: - self.logger.warning( - "S2: 0 papers accepted for %s — check institution affiliation_patterns", - self.institution_name, - ) +""" +Semantic Scholar spider — queries the free S2 Graph API. + +Semantic Scholar (semanticscholar.org) has broader humanities and social-science +coverage than arXiv, including African studies, philosophy, cultural heritage, and +postcolonial literature — exactly the SC categories URAAS cares about. The API is +free (no key required for basic use; 100 reqs/5 min per IP). + +The spider runs two types of waves: + • SC seed waves — institution + each SC seed phrase (e.g. "indigenous knowledge") + • General wave — institution name alone (catches SC hits missed by seeds) + +The SC classifier in the pipeline gates what actually gets saved. +""" + +import os +import sys + +import scrapy + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..")) + +from uraas.config import config +from uraas.config.institutions import get_registry +from uraas.config.special_collections import SC_SEED_KEYWORDS + +_S2_BASE = "https://api.semanticscholar.org/graph/v1/paper/search" +_FIELDS = "title,abstract,authors,year,externalIds,openAccessPdf,fieldsOfStudy,venue" +_LIMIT = 100 + + +class SemanticScholarSpider(scrapy.Spider): + name = "semantic_scholar" + custom_settings = { + # S2 free tier: 100 req / 5 min per IP (~1 req/3 sec sustained). + # When running alongside other spiders the shared IP exhausts the + # budget quickly, so we use a conservative 3 s delay + autothrottle + # with a generous max to absorb 429 bursts. + "DOWNLOAD_DELAY": 3.0, + "CONCURRENT_REQUESTS": 1, + "AUTOTHROTTLE_ENABLED": True, + "AUTOTHROTTLE_START_DELAY": 3.0, + "AUTOTHROTTLE_MAX_DELAY": 30.0, + "AUTOTHROTTLE_TARGET_CONCURRENCY": 1, + "RETRY_ENABLED": True, + "RETRY_TIMES": 3, + "RETRY_HTTP_CODES": [429, 500, 502, 503, 504], + "USER_AGENT": ( + f"URAAS/1.0 (+read-only SC discovery; mailto:{config.OPENALEX_MAILTO})" + ), + "HTTPERROR_ALLOWED_CODES": [429], + } + + def __init__( + self, + institution="unilag", + target=50, + boost_special=True, + sc_only=False, + *args, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.target_limit = int(target) + _truthy = {"1", "true", "yes", "on"} + self.boost_special = ( + boost_special.lower() in _truthy + if isinstance(boost_special, str) + else bool(boost_special) + ) + self.sc_only = ( + sc_only.lower() in _truthy + if isinstance(sc_only, str) + else bool(sc_only) + ) + registry = get_registry() + self.institution_config = registry.get(institution) + if not self.institution_config: + raise ValueError(f"Institution '{institution}' not found in registry") + self.institution_name = self.institution_config.name + self.ror_id = self.institution_config.ror + self._accepted = 0 + + self.logger.info( + "S2 spider | %s | boost_special=%s | sc_only=%s | target=%d", + self.institution_name, + self.boost_special, + self.sc_only, + self.target_limit, + ) + + def _build_url(self, query: str, offset: int = 0) -> str: + import urllib.parse + q = urllib.parse.quote(query) + return ( + f"{_S2_BASE}?query={q}" + f"&fields={_FIELDS}" + f"&limit={_LIMIT}" + f"&offset={offset}" + ) + + def _institution_match(self, title: str, abstract: str, venue: str) -> bool: + """ + S2 basic search doesn't return author affiliations, so we verify the + institution appears anywhere in the available text fields. This is + intentionally lenient — a false negative (dropping a valid paper) + is preferable to a false positive (storing a paper from the wrong + university). + """ + combined = f"{title} {abstract} {venue}".lower() + return any( + pat.lower() in combined + for pat in self.institution_config.affiliation_patterns + ) + + async def start(self): + if not self.sc_only: + url = self._build_url(self.institution_name) + yield scrapy.Request( + url=url, + callback=self.parse, + meta={"wave": "general", "query": self.institution_name, "offset": 0}, + ) + + if self.boost_special: + # Cap at 8 highest-signal SC seeds to avoid exhausting the S2 + # 100 req/5 min budget before we get any results. + _TOP_SEEDS = [ + "indigenous knowledge", "oral tradition", "african literature", + "cultural heritage", "postcolonial", "ethnomusicology", + "pan-african", "traditional medicine", + ] + for seed in _TOP_SEEDS: + query = f"{self.institution_name} {seed}" + url = self._build_url(query) + yield scrapy.Request( + url=url, + callback=self.parse, + meta={"wave": f"sc:{seed}", "query": query, "offset": 0}, + priority=10, + ) + + def parse(self, response): + if response.status == 429: + self.logger.warning("S2 rate-limited (429) — request will be retried by Scrapy") + return + if self._accepted >= self.target_limit: + return + + data = response.json() + papers = data.get("data", []) + wave = response.meta.get("wave", "general") + self.logger.info("[S2:%s] received %d papers", wave, len(papers)) + + rejected_aff = 0 + for paper in papers: + if self._accepted >= self.target_limit: + return + + title = (paper.get("title") or "").strip() + if not title: + continue + + abstract = (paper.get("abstract") or "").strip() + venue = (paper.get("venue") or "").strip() + year = paper.get("year") + pub_date = f"{year}-01-01" if year else "" + + ext_ids = paper.get("externalIds") or {} + doi = ext_ids.get("DOI") or ext_ids.get("doi") or "" + arxiv_id = ext_ids.get("ArXiv") or "" + + oa = paper.get("openAccessPdf") or {} + pdf_url = oa.get("url") if oa else None + + url_val = ( + f"https://doi.org/{doi}" if doi + else (f"https://arxiv.org/abs/{arxiv_id}" if arxiv_id else "") + ) + + authors = [ + a.get("name", "") for a in (paper.get("authors") or []) if a.get("name") + ] + + fields = [ + f.get("category", "") for f in (paper.get("fieldsOfStudy") or []) if f.get("category") + ] + dc_subject = ", ".join(fields[:6]) + + # S2 doesn't return author affiliations in basic search — verify + # that the institution name appears somewhere in the paper data. + if not self._institution_match(title, abstract, venue): + rejected_aff += 1 + self.logger.debug(f"S2 aff FAIL: {title[:60]}") + continue + + self._accepted += 1 + yield { + "title": title, + "abstract": abstract, + "authors": authors, + "doi": doi, + "url": url_val, + "pdf_url": pdf_url, + "publication_date": pub_date, + "source_repository": "Semantic Scholar", + "is_unilag_author": True, + "raw_affiliation": self.institution_name, + "institution": self.institution_name, + "institution_ror": self.ror_id, + "dc_subject": dc_subject, + } + + # Offset-based pagination + total = data.get("total", 0) + offset = response.meta.get("offset", 0) + _LIMIT + if offset < min(total, 500) and self._accepted < self.target_limit: + query = response.meta["query"] + wave = response.meta["wave"] + next_url = self._build_url(query, offset) + yield scrapy.Request( + url=next_url, + callback=self.parse, + meta={"wave": wave, "query": query, "offset": offset}, + ) + + def closed(self, reason): + self.logger.info( + "S2 spider closed | %s | accepted=%d | reason=%s", + self.institution_name, + self._accepted, + reason, + ) + if self._accepted == 0: + self.logger.warning( + "S2: 0 papers accepted for %s — check institution affiliation_patterns", + self.institution_name, + ) diff --git a/uraas/utils/ai_classifier.py b/uraas/utils/ai_classifier.py index 6421405893ac0dfbf1568afb0ec6a039e4359c9d..5d41a083085014d05babd1c86afb8727762efe63 100644 --- a/uraas/utils/ai_classifier.py +++ b/uraas/utils/ai_classifier.py @@ -1,1462 +1,1462 @@ -""" -URAAS AI Classifier v2.0 -Uses spaCy (en_core_web_sm) for NLP-based SDG classification and keyword extraction. -Falls back to enhanced TF-IDF keyword matching if spaCy is unavailable. - -Key improvements over v1: -- Aggressive HTML/XML/JATS artifact stripping -- Bigram and trigram phrase extraction -- Corpus-level TF-IDF (not single-doc frequency) -- Expanded stop word list removing academic filler words -- Named entity filtering (no city/country names as keywords) -""" - -import html -import logging -import math -import re -from typing import Dict, List, Optional, Tuple - -log = logging.getLogger(__name__) - -# ── SDG Definitions ─────────────────────────────────────────────────────────── -SDG_DEFINITIONS: Dict[int, Dict] = { - 1: { - "name": "No Poverty", - "core": [ - "poverty", - "economic inequality", - "social protection", - "income", - "destitution", - "livelihood", - "microcredit", - "social safety net", - "extreme poverty", - "basic needs", - ], - }, - 2: { - "name": "Zero Hunger", - "core": [ - "food security", - "malnutrition", - "hunger", - "food systems", - "agriculture", - "crop production", - "famine", - "nutrition", - "food access", - "smallholder farmers", - ], - }, - 3: { - "name": "Good Health", - "core": [ - "health", - "disease", - "medicine", - "clinical", - "mortality", - "morbidity", - "vaccine", - "immunization", - "malaria", - "hiv", - "tuberculosis", - "cancer", - "mental health", - "maternal health", - "child mortality", - "public health", - "epidemiology", - "infectious disease", - "non-communicable disease", - ], - }, - 4: { - "name": "Quality Education", - "core": [ - "education", - "learning outcomes", - "school", - "university", - "literacy", - "numeracy", - "curriculum", - "pedagogy", - "teacher training", - "educational access", - "early childhood", - "higher education", - "vocational training", - ], - }, - 5: { - "name": "Gender Equality", - "core": [ - "gender equality", - "women empowerment", - "female participation", - "gender-based violence", - "feminism", - "gender gap", - "reproductive rights", - "sexual harassment", - "discrimination against women", - "gender mainstreaming", - ], - }, - 6: { - "name": "Clean Water", - "core": [ - "water supply", - "sanitation", - "wastewater treatment", - "drinking water quality", - "water scarcity", - "water access", - "hygiene", - "groundwater", - "water pollution", - "watershed", - ], - }, - 7: { - "name": "Affordable Energy", - "core": [ - "renewable energy", - "solar power", - "wind energy", - "energy access", - "photovoltaic", - "energy poverty", - "electricity grid", - "energy efficiency", - "hydropower", - "biomass energy", - "off-grid", - ], - }, - 8: { - "name": "Decent Work", - "core": [ - "employment", - "labour market", - "economic growth", - "entrepreneurship", - "gdp growth", - "decent work", - "youth employment", - "informal economy", - "productivity", - "workers rights", - "job creation", - ], - }, - 9: { - "name": "Industry and Innovation", - "core": [ - "innovation", - "infrastructure", - "industrial development", - "manufacturing", - "technology transfer", - "research and development", - "patent", - "startup", - "digitalization", - "industrialization", - ], - }, - 10: { - "name": "Reduced Inequalities", - "core": [ - "inequality", - "income distribution", - "social inclusion", - "discrimination", - "marginalization", - "affirmative action", - "wealth gap", - "racial inequality", - "ethnic inequality", - ], - }, - 11: { - "name": "Sustainable Cities", - "core": [ - "urban planning", - "smart city", - "housing", - "transport", - "urbanization", - "slum", - "public space", - "urban resilience", - "waste management", - "urban governance", - ], - }, - 12: { - "name": "Responsible Consumption", - "core": [ - "sustainable consumption", - "circular economy", - "waste reduction", - "recycling", - "sustainable production", - "resource efficiency", - "plastic pollution", - "food waste", - "lifecycle assessment", - ], - }, - 13: { - "name": "Climate Action", - "core": [ - "climate change", - "global warming", - "carbon emissions", - "greenhouse gas", - "climate adaptation", - "climate mitigation", - "sea level rise", - "carbon footprint", - "climate policy", - "net zero", - ], - }, - 14: { - "name": "Life Below Water", - "core": [ - "ocean", - "marine ecosystem", - "fisheries", - "coastal management", - "aquatic biodiversity", - "coral reef", - "sea pollution", - "overfishing", - "marine conservation", - "lagoon", - ], - }, - 15: { - "name": "Life on Land", - "core": [ - "biodiversity", - "ecosystem", - "deforestation", - "land degradation", - "wildlife conservation", - "endangered species", - "forest management", - "land use change", - "wetland", - "desertification", - ], - }, - 16: { - "name": "Peace and Justice", - "core": [ - "governance", - "rule of law", - "corruption", - "institutional capacity", - "peace", - "conflict", - "human rights", - "access to justice", - "transparency", - "democracy", - "peacebuilding", - ], - }, - 17: { - "name": "Partnerships", - "core": [ - "international cooperation", - "development aid", - "public-private partnership", - "technology transfer", - "south-south cooperation", - "global governance", - "multilateralism", - "financing for development", - ], - }, -} - -# ── Special Collections ─────────────────────────────────────────────────────── -SPECIAL_COLLECTIONS: Dict[str, List[str]] = { - "Indigenous Knowledge": [ - "indigenous knowledge", - "traditional knowledge", - "indigenous epistemology", - "ethnobotany", - "ethnobotanical", - "traditional ecological knowledge", - "indigenous medicine", - "traditional healing", - "ancestral wisdom", - "precolonial knowledge", - "traditional practices", - "indigenous technology", - "folk medicine", - "oral traditions", - "folklore", - "cultural transmission", - "indigenous cosmology", - "traditional farming", - "indigenous peoples", - "traditional religion", - "traditional medicine", - "ethno-medicine", - "indigenous farming", - "ethnoveterinary", - "indigenous architecture", - "traditional weather forecasting", - "local ecological knowledge", - "indigenous forestry", - "indigenous land management", - "traditional food systems", - "indigenous soil conservation", - "indigenous metallurgy", - "traditional pottery", - ], - "African Literature": [ - "postcolonial literature", - "african literature", - "negritude", - "afrocentrism", - "african novel", - "african drama", - "oral literature", - "oral poetry", - "african aesthetics", - "indigenous poetry", - "pan-africanism", - "decolonizing the mind", - "colonial literature", - "nigerian literature", - "kenyan literature", - "african narrative", - "wole soyinka", - "chinua achebe", - "ngugi wa thiongo", - "african writers", - "african storytelling", - "griots", - "afrofuturism", - "african literary criticism", - "oral narrative", - "indigenous drama", - "decolonial literature", - "black aesthetics", - "african theatre", - "contemporary african writing", - ], - "Cultural Heritage": [ - "cultural heritage", - "intangible heritage", - "cultural identity", - "heritage preservation", - "oral history", - "material culture", - "museum studies", - "cultural memory", - "sacred sites", - "cultural artifacts", - "postcolonial heritage", - "traditional customs", - "cultural continuity", - "ethnography", - "cultural landscape", - "cultural practices", - "heritage conservation", - "world heritage", - "cultural diversity", - "indigenous heritage", - "living heritage", - "ancestral heritage", - "cultural preservation", - "traditional ceremonies", - "indigenous art", - "monuments preservation", - "archaeological heritage", - "sacred groves", - "rock art preservation", - "indigenous textiles", - ], - "Ethnic Languages & Groups": [ - "ethnic group", - "ethnic language", - "indigenous language", - "yoruba", - "igbo", - "hausa", - "swahili", - "kiswahili", - "amharic", - "zulu", - "xhosa", - "shona", - "somali", - "kinyarwanda", - "oromo", - "twi", - "fante", - "ewe", - "wolof", - "luganda", - "lingala", - "bambara", - "tigrinya", - "chewa", - "ndebele", - "sotho", - "sesotho", - "setswana", - "tsonga", - "ss", - "venda", - "fulani", - "maasai", - "igboland", - "yorubaland", - "hausaland", - "kikuyu", - "oromoland", - "luo", - "akan", - "ganda", - "shona culture", - "zulu kingdom", - "ashanti", - "yoruba cosmology", - "igbo metaphysics", - "swahili coast", - ], - "Postcolonial Studies": [ - "postcolonialism", - "decolonization", - "colonialism", - "imperialism", - "subaltern", - "hybridity", - "mimicry", - "diaspora studies", - "postcolonial theory", - "colonial legacy", - "neo-colonialism", - "independence movements", - "african nationalism", - "resistance literature", - "colonial history", - "settler colonialism", - "decolonial thought", - "decoloniality", - "epistemic decolonization", - "postcolonial identity", - "colonial violence", - "anti-colonial resistance", - "decolonial turn", - ], - "Pan-African Studies": [ - "pan-africanism", - "african unity", - "african union", - "african identity", - "african renaissance", - "afrocentricity", - "african development", - "african continental", - "afro-optimism", - "black consciousness", - "african solidarity", - "african geopolitics", - "ecowas", - "sadc", - "east african community", - "african integration", - "black diaspora", - "panafrican", - "african economic community", - "agenda 2063", - ], - "African Philosophy": [ - "ubuntu", - "african philosophy", - "african ethics", - "communalism", - "african metaphysics", - "african ontology", - "african logic", - "african epistemology", - "african humanism", - "african thought", - "african worldview", - "indigenous philosophy", - "sage philosophy", - "negritude philosophy", - "ubuntu ethics", - "african communitarianism", - ], - "Ethnomusicology": [ - "ethnomusicology", - "african music", - "traditional music", - "folk music", - "african drumming", - "musical heritage", - "afrobeats", - "highlife", - "african rhythm", - "musical traditions", - "indigenous music", - "oral musical tradition", - "african instruments", - "kora music", - "mbira music", - "djembe drumming", - "traditional chants", - ], -} - -# ── African Union Charter Targets ───────────────────────────────────────────── -AU_CHARTER_TARGETS: Dict[int, Dict] = { - 1: { - "name": "Tangible & Intangible Cultural Heritage Preservation", - "keywords": [ - "cultural heritage", - "intangible heritage", - "heritage preservation", - "material culture", - "museum studies", - "cultural artifacts", - "heritage conservation", - "world heritage", - "sacred sites", - "living heritage", - "archaeological", - "rock art", - "sacred groves", - ], - }, - 2: { - "name": "Development of African Languages & Decolonization of Science", - "keywords": [ - "indigenous language", - "yoruba", - "igbo", - "hausa", - "swahili", - "amharic", - "zulu", - "xhosa", - "shona", - "somali", - "kinyarwanda", - "oromo", - "twi", - "fante", - "ewe", - "wolof", - "luganda", - "lingala", - "bambara", - "tigrinya", - "chewa", - "ndebele", - "sotho", - "african language", - "ethnic language", - "decolonizing science", - "linguistic diversity", - ], - }, - 3: { - "name": "Integration of Cultural Values & Indigenous Knowledge Systems", - "keywords": [ - "indigenous knowledge", - "traditional knowledge", - "indigenous epistemology", - "ethnobotany", - "traditional ecological knowledge", - "indigenous medicine", - "traditional healing", - "ancestral wisdom", - "traditional practices", - "precolonial knowledge", - "indigenous cosmology", - "traditional religion", - ], - }, - 4: { - "name": "Inter-Institutional Cultural Exchange & Regional Integration", - "keywords": [ - "cultural exchange", - "regional integration", - "pan-africanism", - "african solidarity", - "african union", - "african continental", - "cross-border", - "ecowas", - "sadc", - "east african community", - ], - }, - 5: { - "name": "Support for Creative and Cultural Industries", - "keywords": [ - "creative industry", - "cultural industry", - "african literature", - "african music", - "african novel", - "african drama", - "oral literature", - "traditional music", - "african drumming", - "highlife", - "afrobeats", - "performing arts", - "african cinema", - "storytelling", - "folklore", - ], - }, - 6: { - "name": "Scientific Innovation & Traditional Technology Integration", - "keywords": [ - "traditional technology", - "indigenous technology", - "traditional farming", - "traditional agriculture", - "ethnoveterinary", - "indigenous agriculture", - "traditional metallurgy", - "traditional medicine production", - ], - }, - 7: { - "name": "Youth Engagement & Cultural Education", - "keywords": [ - "cultural transmission", - "cultural education", - "pedagogy", - "oral history", - "folklore", - "oral tradition", - "youth engagement", - "cultural values", - ], - }, - 8: { - "name": "Intellectual Property, Open Access, and Copyright Protection", - "keywords": [ - "intellectual property", - "open access", - "copyright", - "traditional knowledge rights", - "biopiracy", - "patent protection", - "indigenous rights", - "open science", - ], - }, - 9: { - "name": "Decolonial Philosophy & African Thought Systems (Ubuntu)", - "keywords": [ - "ubuntu", - "african philosophy", - "african thought", - "african ethics", - "communalism", - "decolonial", - "decolonization", - "postcolonialism", - "negritude", - "afrocentricity", - "decolonial thought", - "subaltern", - "african worldview", - ], - }, -} - - -def classify_au_targets(title: str, abstract: str, dc_subject: str = "") -> List[Dict]: - """ - Classify a paper against the 9 African Union Charter Targets. - Returns list of {target_number, target_name, score, matched_keywords}. - """ - text = _clean_text(f"{title or ''} {abstract or ''} {dc_subject or ''}").lower() - if not text.strip(): - return [] - - results = [] - for num, defn in AU_CHARTER_TARGETS.items(): - score, matched = _keyword_score(text, defn["keywords"]) - if score >= 1: - results.append( - { - "target_number": num, - "target_name": defn["name"], - "score": score, - "matched_keywords": matched[:6], - } - ) - - results.sort(key=lambda x: -x["score"]) - return results - - -# ── Comprehensive Stop Words ────────────────────────────────────────────────── -# Covers: common English, academic filler, metadata artifacts, XML/JATS tags, -# geographic terms that are too broad, formatting remnants -STOP_WORDS = { - # Common English - "the", - "and", - "for", - "with", - "this", - "that", - "from", - "have", - "been", - "were", - "their", - "which", - "these", - "about", - "other", - "into", - "than", - "more", - "such", - "some", - "what", - "when", - "where", - "there", - "also", - "using", - "used", - "show", - "both", - "each", - "only", - "very", - "well", - "high", - "low", - "new", - "large", - "small", - "significant", - "different", - "similar", - "total", - "however", - "therefore", - "thus", - "hence", - "although", - "despite", - "while", - "after", - "before", - "through", - "across", - "during", - "within", - "among", - "between", - "either", - "neither", - "whether", - "since", - "upon", - "against", - "without", - "under", - "over", - "above", - "below", - "around", - "towards", - "onto", - "itself", - "itself", - "itself", - "they", - "them", - "their", - "those", - "these", - "here", - "then", - "just", - "like", - "make", - "many", - "most", - "much", - "even", - "back", - "still", - "need", - "could", - "would", - "should", - "shall", - "will", - "might", - "must", - "been", - "have", - "does", - "done", - "made", - "said", - "take", - "come", - "find", - "give", - "know", - "look", - "seem", - "feel", - "become", - "include", - "provide", - "require", - "remain", - "suggest", - "indicate", - "demonstrate", - "show", - "reveal", - "confirm", - "report", - "find", - "identify", - "examine", - "determine", - "evaluate", - "assess", - "compare", - "describe", - "present", - "discuss", - "explore", - "investigate", - "conduct", - "perform", - "apply", - "observe", - "measure", - "calculate", - "estimate", - "predict", - "test", - # Academic filler - "study", - "paper", - "research", - "analysis", - "findings", - "results", - "method", - "approach", - "model", - "system", - "review", - "case", - "report", - "effect", - "impact", - "based", - "data", - "conclusion", - "objective", - "background", - "introduction", - "abstract", - "methods", - "discussion", - "purpose", - "aim", - "goal", - "hypothesis", - "evidence", - "sample", - "group", - "population", - "intervention", - "outcome", - "variable", - "factor", - "relationship", - "association", - "correlation", - "significance", - "difference", - "increase", - "decrease", - "higher", - "lower", - "greater", - "lower", - "compared", - "respectively", - "overall", - "showed", - "found", - "reported", - "observed", - "noted", - "significantly", - "p-value", - "confidence", - "interval", - "mean", - "median", - "standard", - "deviation", - "percent", - "percentage", - "proportion", - "ratio", - "rate", - "number", - "approximately", - "moreover", - "furthermore", - "additionally", - "however", - "nevertheless", - "consequently", - "therefore", - "conclusion", - "finally", - "firstly", - "secondly", - "lastly", - "currently", - "recently", - "previously", - "generally", - "specifically", - "particularly", - "mainly", - "primarily", - "mostly", - "largely", - "significantly", - "approximately", - "relatively", - # Metadata / XML / JATS artifacts - "jats", - "jats-inline", - "journal", - "abstract", - "article", - "title", - "italic", - "bold", - "sup", - "sub", - "break", - "para", - "section", - "body", - "front", - "back", - "meta", - "keyword", - "keywords", - "author", - "authors", - "affiliation", - "institution", - "corresponding", - "mailto", - "email", - "grant", - "funding", - "acknowledgment", - "acknowledgements", - "references", - "bibliography", - "footnote", - "table", - "figure", - "appendix", - "supplement", - # Geographic terms too broad to be meaningful keywords - "nigeria", - "nigerian", - "lagos", - "africa", - "african", - "world", - "global", - "international", - "national", - "regional", - "local", - "country", - "countries", - "continent", - "west", - "east", - "south", - "north", - "central", - # Numbers and fragments that slip through - "2266", - "2015", - "2016", - "2017", - "2018", - "2019", - "2020", - "2021", - "2022", - "2023", - "2024", - "2025", - # Short meaningless words (caught by length filter but listed for clarity) - "also", - "into", - "onto", - "upon", - "with", - "from", - "have", - "that", -} - -# Regex patterns for artifacts to strip before processing -_HTML_ENTITY_RE = re.compile(r"&(?:#\d+|#x[0-9a-fA-F]+|[a-zA-Z]+);") -_HTML_TAG_RE = re.compile(r"<[^>]+>") -_JATS_TAG_RE = re.compile(r"\bjats:[a-z\-]+\b", re.IGNORECASE) -_NON_ALPHA_RE = re.compile(r"[^a-zA-Z\s\-]") -_MULTI_SPACE_RE = re.compile(r"\s+") - - -def _clean_text(text: str) -> str: - """Strip HTML entities, XML/JATS tags, and non-alphabetic artifacts.""" - if not text: - return "" - # Decode HTML entities (e.g., &lt; → <) - text = html.unescape(text) - text = html.unescape(text) # Double-decode for double-encoded entities - # Strip remaining HTML/XML tags - text = _HTML_TAG_RE.sub(" ", text) - # Strip JATS namespace tokens - text = _JATS_TAG_RE.sub(" ", text) - # Strip remaining HTML entities - text = _HTML_ENTITY_RE.sub(" ", text) - # Remove non-alphabetic characters (keep hyphens for compound terms) - text = _NON_ALPHA_RE.sub(" ", text) - # Normalise whitespace - text = _MULTI_SPACE_RE.sub(" ", text).strip() - return text - - -def _is_valid_word(word: str) -> bool: - """Return True if the word is a meaningful academic term.""" - if len(word) < 4: - return False - if word in STOP_WORDS: - return False - # Reject pure number strings - if word.isdigit(): - return False - # Reject words that are mostly digits - digit_ratio = sum(c.isdigit() for c in word) / len(word) - if digit_ratio > 0.3: - return False - # Reject very short all-caps (likely acronyms of metadata) - if len(word) <= 4 and word.isupper(): - return False - return True - - -# ── NLP loading ─────────────────────────────────────────────────────────────── -_nlp = None -_spacy_available = False - - -def _load_nlp(): - global _nlp, _spacy_available - if _nlp is not None: - return _nlp - try: - import spacy - - try: - _nlp = spacy.load("en_core_web_sm") - except OSError: - import subprocess - import sys - - subprocess.run( - [sys.executable, "-m", "spacy", "download", "en_core_web_sm"], - capture_output=True, - ) - _nlp = spacy.load("en_core_web_sm") - _spacy_available = True - log.info("spaCy model loaded: en_core_web_sm") - except Exception as e: - log.warning(f"spaCy unavailable ({e}), using TF-IDF fallback") - _spacy_available = False - return _nlp - - -# ── Core classification functions ───────────────────────────────────────────── - -import re - - -def _keyword_score(text: str, keywords: List[str]) -> Tuple[int, List[str]]: - """Score text against a keyword list, return (score, matched_keywords).""" - text_lower = text.lower() - matched = [] - for kw in keywords: - kw_lower = kw.lower() - if re.search(rf"\b{re.escape(kw_lower)}\b", text_lower): - matched.append(kw) - return len(matched), matched - - -def classify_sdgs(title: str, abstract: str, threshold: int = 1) -> List[Dict]: - """ - Classify a paper against all 17 SDGs using AI + keyword matching. - Returns list of {sdg_number, sdg_name, score, matched_keywords} sorted by score desc. - """ - text = _clean_text(f"{title or ''} {abstract or ''}").strip() - if not text: - return [] - - nlp = _load_nlp() - enriched = text.lower() - if _spacy_available and nlp: - try: - doc = nlp(text[:5000]) - noun_chunks = [chunk.text.lower() for chunk in doc.noun_chunks] - enriched = text.lower() + " " + " ".join(noun_chunks) - except Exception: - pass - - results = [] - for sdg_num, defn in SDG_DEFINITIONS.items(): - score, matched = _keyword_score(enriched, defn["core"]) - if score >= threshold: - results.append( - { - "sdg_number": sdg_num, - "sdg_name": defn["name"], - "score": score, - "matched_keywords": matched[:8], - } - ) - - results.sort(key=lambda x: -x["score"]) - return results - - -def classify_special_collections( - title: str, abstract: str, dc_subject: str = "" -) -> List[Dict]: - """ - Classify a paper into special collections categories. - Returns list of {category, score, matched_keywords}. - """ - text = _clean_text(f"{title or ''} {abstract or ''} {dc_subject or ''}").lower() - if not text.strip(): - return [] - - results = [] - for category, keywords in SPECIAL_COLLECTIONS.items(): - score, matched = _keyword_score(text, keywords) - if score >= 1: - results.append( - { - "category": category, - "score": score * 3, - "matched_keywords": matched[:6], - } - ) - - results.sort(key=lambda x: -x["score"]) - return results - - -def extract_keywords( - title: str, abstract: str, top_n: int = 60, all_texts: Optional[List[str]] = None -) -> List[Dict]: - """ - Extract key academic terms using TF-IDF + spaCy NER + bigrams. - - Args: - title: Paper title - abstract: Paper abstract - top_n: Number of keywords to return - all_texts: Optional list of all abstracts/titles for IDF calculation. - If provided, uses corpus-level TF-IDF for better ranking. - - Returns: - [{word, score, count, type}] sorted by relevance score. - """ - raw = f"{title or ''} {abstract or ''}".strip() - text = _clean_text(raw) - if not text: - return [] - - text_lower = text.lower() - - # ── Unigrams ────────────────────────────────────────────────────────────── - words = re.findall(r"\b[a-zA-Z][a-zA-Z\-]{3,}\b", text_lower) - unigrams = [w for w in words if _is_valid_word(w)] - - # ── Bigrams (two-word phrases) ──────────────────────────────────────────── - bigrams = [] - for i in range(len(words) - 1): - w1, w2 = words[i], words[i + 1] - if _is_valid_word(w1) and _is_valid_word(w2): - bigram = f"{w1} {w2}" - bigrams.append(bigram) - - # ── Trigrams (three-word phrases for compound terms) ───────────────────── - trigrams = [] - for i in range(len(words) - 2): - w1, w2, w3 = words[i], words[i + 1], words[i + 2] - if _is_valid_word(w1) and _is_valid_word(w2) and _is_valid_word(w3): - trigrams.append(f"{w1} {w2} {w3}") - - # ── Frequency counts ────────────────────────────────────────────────────── - term_freq: Dict[str, int] = {} - for t in unigrams + bigrams + trigrams: - term_freq[t] = term_freq.get(t, 0) + 1 - - # ── IDF computation (if corpus provided) ───────────────────────────────── - idf_scores: Dict[str, float] = {} - if all_texts and len(all_texts) > 1: - N = len(all_texts) - all_texts_lower = [doc.lower() for doc in all_texts] - candidate_terms = sorted(term_freq.keys(), key=lambda t: -term_freq[t])[:1500] - for term in candidate_terms: - doc_count = sum(1 for doc in all_texts_lower if term in doc) - idf_scores[term] = math.log((N + 1) / (doc_count + 1)) + 1 - for term in term_freq: - if term not in idf_scores: - idf_scores[term] = 1.0 - else: - # Without corpus, use log(freq+1) as a proxy - for term in term_freq: - idf_scores[term] = math.log(term_freq[term] + 2) - - # ── Score = TF * IDF ────────────────────────────────────────────────────── - total = sum(term_freq.values()) or 1 - scored = [] - for term, freq in term_freq.items(): - tf = freq / total - idf = idf_scores.get(term, 1.0) - score = round(tf * idf * 100, 3) - # Boost multi-word phrases (they're more meaningful) - if " " in term: - score *= 1.5 if term.count(" ") == 1 else 2.0 - scored.append( - { - "word": term, - "score": score, - "count": freq, - "type": "phrase" if " " in term else "term", - } - ) - - # ── Add spaCy named entities (boost recognized entities) ───────────────── - nlp = _load_nlp() - if _spacy_available and nlp: - try: - doc = nlp(text[:3000]) - for ent in doc.ents: - # Only include meaningful entity types, skip countries/cities as keywords - if ent.label_ in ( - "ORG", - "PRODUCT", - "WORK_OF_ART", - "EVENT", - "LAW", - "NORP", - ): - ent_text = ent.text.lower() - ent_clean = _clean_text(ent_text) - if len(ent_clean) >= 4 and _is_valid_word(ent_clean.split()[0]): - scored.append( - { - "word": ent_clean, - "score": 6.0, - "count": 1, - "type": "entity", - } - ) - except Exception: - pass - - # ── Sort, deduplicate, return top N ────────────────────────────────────── - scored.sort(key=lambda x: -x["score"]) - seen: set = set() - unique = [] - for item in scored: - key = item["word"].lower().strip() - # Skip if a longer phrase containing this term already present - if key not in seen: - seen.add(key) - # Also mark sub-terms of present phrases to avoid redundancy - unique.append(item) - - return unique[:top_n] - - -def extract_trends_from_corpus(papers: List[Dict], top_n: int = 12) -> List[Dict]: - """ - AI-powered trend extraction from a corpus of papers. - Groups papers into semantic topic clusters using TF-IDF similarity. - - Args: - papers: List of {id, title, abstract, year} dicts - top_n: Number of trends to discover - - Returns: - List of {topic, keywords, paper_count, years, papers} dicts - """ - if not papers: - return [] - - # Build per-year keyword frequency - year_keywords: Dict[int, Dict[str, int]] = {} - all_texts = [f"{p.get('title','')} {p.get('abstract','')}" for p in papers] - - for p in papers: - year = p.get("year") - if not year: - continue - text = _clean_text(f"{p.get('title','')} {p.get('abstract','')}") - words = re.findall(r"\b[a-zA-Z][a-zA-Z\-]{3,}\b", text.lower()) - words = [w for w in words if _is_valid_word(w)] - - # Also extract bigrams - for i in range(len(words) - 1): - w1, w2 = words[i], words[i + 1] - if _is_valid_word(w1) and _is_valid_word(w2): - words.append(f"{w1} {w2}") - - if year not in year_keywords: - year_keywords[year] = {} - for w in words: - year_keywords[year][w] = year_keywords[year].get(w, 0) + 1 - - # Find terms that appear across multiple years (persistent trends) - global_freq: Dict[str, int] = {} - global_year_presence: Dict[str, set] = {} - for year, freq_map in year_keywords.items(): - for term, count in freq_map.items(): - global_freq[term] = global_freq.get(term, 0) + count - if term not in global_year_presence: - global_year_presence[term] = set() - global_year_presence[term].add(year) - - # Score terms: frequency × year spread - N_docs = len(all_texts) - all_texts_lower = [t.lower() for t in all_texts] - candidate_terms = sorted(global_freq.keys(), key=lambda t: -global_freq[t])[:1500] - trend_scores: Dict[str, float] = {} - for term, freq in global_freq.items(): - if term in candidate_terms: - year_spread = len(global_year_presence.get(term, set())) - doc_count = sum(1 for t in all_texts_lower if term in t) - idf = math.log((N_docs + 1) / (doc_count + 1)) + 1 - trend_scores[term] = freq * year_spread * idf - else: - trend_scores[term] = 0.0 - - # Get top terms as seed topics - top_terms = sorted(trend_scores.items(), key=lambda x: -x[1])[: top_n * 3] - - # Group into trends by co-occurrence (simple greedy clustering) - trends = [] - used_terms: set = set() - - for seed_term, seed_score in top_terms: - if seed_term in used_terms: - continue - if len(trends) >= top_n: - break - - # Find related terms (share at least one year and high frequency) - related = [seed_term] - used_terms.add(seed_term) - seed_years = global_year_presence.get(seed_term, set()) - - for term, score in top_terms: - if term in used_terms: - continue - term_years = global_year_presence.get(term, set()) - overlap = len(seed_years & term_years) - if overlap >= 1: - # Check if they appear near each other in text (simple proxy: both in same abstract) - co_occur = sum( - 1 for t in all_texts if seed_term in t.lower() and term in t.lower() - ) - if co_occur >= 1: - related.append(term) - used_terms.add(term) - if len(related) >= 5: - break - - # Find papers matching this trend cluster - trend_papers = [] - for p in papers: - text = f"{p.get('title','')} {p.get('abstract','')}".lower() - if any(t in text for t in related[:3]): - trend_papers.append(p) - - if len(trend_papers) < 2: - continue - - # Build topic label from seed (capitalise key phrase) - label_parts = seed_term.split() - label = " ".join(w.capitalize() for w in label_parts[:3]) - - # Year-by-year paper counts - by_year = {} - for p in trend_papers: - yr = p.get("year") - if yr: - by_year[str(yr)] = by_year.get(str(yr), 0) + 1 - - trends.append( - { - "topic": label, - "keywords": related[:6], - "paper_count": len(trend_papers), - "total": len(trend_papers), - "by_year": by_year, - "papers": [ - {"id": p.get("id"), "title": p.get("title"), "year": p.get("year")} - for p in trend_papers[:20] - ], - } - ) - - trends.sort(key=lambda x: -x["paper_count"]) - return trends[:top_n] +""" +URAAS AI Classifier v2.0 +Uses spaCy (en_core_web_sm) for NLP-based SDG classification and keyword extraction. +Falls back to enhanced TF-IDF keyword matching if spaCy is unavailable. + +Key improvements over v1: +- Aggressive HTML/XML/JATS artifact stripping +- Bigram and trigram phrase extraction +- Corpus-level TF-IDF (not single-doc frequency) +- Expanded stop word list removing academic filler words +- Named entity filtering (no city/country names as keywords) +""" + +import html +import logging +import math +import re +from typing import Dict, List, Optional, Tuple + +log = logging.getLogger(__name__) + +# ── SDG Definitions ─────────────────────────────────────────────────────────── +SDG_DEFINITIONS: Dict[int, Dict] = { + 1: { + "name": "No Poverty", + "core": [ + "poverty", + "economic inequality", + "social protection", + "income", + "destitution", + "livelihood", + "microcredit", + "social safety net", + "extreme poverty", + "basic needs", + ], + }, + 2: { + "name": "Zero Hunger", + "core": [ + "food security", + "malnutrition", + "hunger", + "food systems", + "agriculture", + "crop production", + "famine", + "nutrition", + "food access", + "smallholder farmers", + ], + }, + 3: { + "name": "Good Health", + "core": [ + "health", + "disease", + "medicine", + "clinical", + "mortality", + "morbidity", + "vaccine", + "immunization", + "malaria", + "hiv", + "tuberculosis", + "cancer", + "mental health", + "maternal health", + "child mortality", + "public health", + "epidemiology", + "infectious disease", + "non-communicable disease", + ], + }, + 4: { + "name": "Quality Education", + "core": [ + "education", + "learning outcomes", + "school", + "university", + "literacy", + "numeracy", + "curriculum", + "pedagogy", + "teacher training", + "educational access", + "early childhood", + "higher education", + "vocational training", + ], + }, + 5: { + "name": "Gender Equality", + "core": [ + "gender equality", + "women empowerment", + "female participation", + "gender-based violence", + "feminism", + "gender gap", + "reproductive rights", + "sexual harassment", + "discrimination against women", + "gender mainstreaming", + ], + }, + 6: { + "name": "Clean Water", + "core": [ + "water supply", + "sanitation", + "wastewater treatment", + "drinking water quality", + "water scarcity", + "water access", + "hygiene", + "groundwater", + "water pollution", + "watershed", + ], + }, + 7: { + "name": "Affordable Energy", + "core": [ + "renewable energy", + "solar power", + "wind energy", + "energy access", + "photovoltaic", + "energy poverty", + "electricity grid", + "energy efficiency", + "hydropower", + "biomass energy", + "off-grid", + ], + }, + 8: { + "name": "Decent Work", + "core": [ + "employment", + "labour market", + "economic growth", + "entrepreneurship", + "gdp growth", + "decent work", + "youth employment", + "informal economy", + "productivity", + "workers rights", + "job creation", + ], + }, + 9: { + "name": "Industry and Innovation", + "core": [ + "innovation", + "infrastructure", + "industrial development", + "manufacturing", + "technology transfer", + "research and development", + "patent", + "startup", + "digitalization", + "industrialization", + ], + }, + 10: { + "name": "Reduced Inequalities", + "core": [ + "inequality", + "income distribution", + "social inclusion", + "discrimination", + "marginalization", + "affirmative action", + "wealth gap", + "racial inequality", + "ethnic inequality", + ], + }, + 11: { + "name": "Sustainable Cities", + "core": [ + "urban planning", + "smart city", + "housing", + "transport", + "urbanization", + "slum", + "public space", + "urban resilience", + "waste management", + "urban governance", + ], + }, + 12: { + "name": "Responsible Consumption", + "core": [ + "sustainable consumption", + "circular economy", + "waste reduction", + "recycling", + "sustainable production", + "resource efficiency", + "plastic pollution", + "food waste", + "lifecycle assessment", + ], + }, + 13: { + "name": "Climate Action", + "core": [ + "climate change", + "global warming", + "carbon emissions", + "greenhouse gas", + "climate adaptation", + "climate mitigation", + "sea level rise", + "carbon footprint", + "climate policy", + "net zero", + ], + }, + 14: { + "name": "Life Below Water", + "core": [ + "ocean", + "marine ecosystem", + "fisheries", + "coastal management", + "aquatic biodiversity", + "coral reef", + "sea pollution", + "overfishing", + "marine conservation", + "lagoon", + ], + }, + 15: { + "name": "Life on Land", + "core": [ + "biodiversity", + "ecosystem", + "deforestation", + "land degradation", + "wildlife conservation", + "endangered species", + "forest management", + "land use change", + "wetland", + "desertification", + ], + }, + 16: { + "name": "Peace and Justice", + "core": [ + "governance", + "rule of law", + "corruption", + "institutional capacity", + "peace", + "conflict", + "human rights", + "access to justice", + "transparency", + "democracy", + "peacebuilding", + ], + }, + 17: { + "name": "Partnerships", + "core": [ + "international cooperation", + "development aid", + "public-private partnership", + "technology transfer", + "south-south cooperation", + "global governance", + "multilateralism", + "financing for development", + ], + }, +} + +# ── Special Collections ─────────────────────────────────────────────────────── +SPECIAL_COLLECTIONS: Dict[str, List[str]] = { + "Indigenous Knowledge": [ + "indigenous knowledge", + "traditional knowledge", + "indigenous epistemology", + "ethnobotany", + "ethnobotanical", + "traditional ecological knowledge", + "indigenous medicine", + "traditional healing", + "ancestral wisdom", + "precolonial knowledge", + "traditional practices", + "indigenous technology", + "folk medicine", + "oral traditions", + "folklore", + "cultural transmission", + "indigenous cosmology", + "traditional farming", + "indigenous peoples", + "traditional religion", + "traditional medicine", + "ethno-medicine", + "indigenous farming", + "ethnoveterinary", + "indigenous architecture", + "traditional weather forecasting", + "local ecological knowledge", + "indigenous forestry", + "indigenous land management", + "traditional food systems", + "indigenous soil conservation", + "indigenous metallurgy", + "traditional pottery", + ], + "African Literature": [ + "postcolonial literature", + "african literature", + "negritude", + "afrocentrism", + "african novel", + "african drama", + "oral literature", + "oral poetry", + "african aesthetics", + "indigenous poetry", + "pan-africanism", + "decolonizing the mind", + "colonial literature", + "nigerian literature", + "kenyan literature", + "african narrative", + "wole soyinka", + "chinua achebe", + "ngugi wa thiongo", + "african writers", + "african storytelling", + "griots", + "afrofuturism", + "african literary criticism", + "oral narrative", + "indigenous drama", + "decolonial literature", + "black aesthetics", + "african theatre", + "contemporary african writing", + ], + "Cultural Heritage": [ + "cultural heritage", + "intangible heritage", + "cultural identity", + "heritage preservation", + "oral history", + "material culture", + "museum studies", + "cultural memory", + "sacred sites", + "cultural artifacts", + "postcolonial heritage", + "traditional customs", + "cultural continuity", + "ethnography", + "cultural landscape", + "cultural practices", + "heritage conservation", + "world heritage", + "cultural diversity", + "indigenous heritage", + "living heritage", + "ancestral heritage", + "cultural preservation", + "traditional ceremonies", + "indigenous art", + "monuments preservation", + "archaeological heritage", + "sacred groves", + "rock art preservation", + "indigenous textiles", + ], + "Ethnic Languages & Groups": [ + "ethnic group", + "ethnic language", + "indigenous language", + "yoruba", + "igbo", + "hausa", + "swahili", + "kiswahili", + "amharic", + "zulu", + "xhosa", + "shona", + "somali", + "kinyarwanda", + "oromo", + "twi", + "fante", + "ewe", + "wolof", + "luganda", + "lingala", + "bambara", + "tigrinya", + "chewa", + "ndebele", + "sotho", + "sesotho", + "setswana", + "tsonga", + "ss", + "venda", + "fulani", + "maasai", + "igboland", + "yorubaland", + "hausaland", + "kikuyu", + "oromoland", + "luo", + "akan", + "ganda", + "shona culture", + "zulu kingdom", + "ashanti", + "yoruba cosmology", + "igbo metaphysics", + "swahili coast", + ], + "Postcolonial Studies": [ + "postcolonialism", + "decolonization", + "colonialism", + "imperialism", + "subaltern", + "hybridity", + "mimicry", + "diaspora studies", + "postcolonial theory", + "colonial legacy", + "neo-colonialism", + "independence movements", + "african nationalism", + "resistance literature", + "colonial history", + "settler colonialism", + "decolonial thought", + "decoloniality", + "epistemic decolonization", + "postcolonial identity", + "colonial violence", + "anti-colonial resistance", + "decolonial turn", + ], + "Pan-African Studies": [ + "pan-africanism", + "african unity", + "african union", + "african identity", + "african renaissance", + "afrocentricity", + "african development", + "african continental", + "afro-optimism", + "black consciousness", + "african solidarity", + "african geopolitics", + "ecowas", + "sadc", + "east african community", + "african integration", + "black diaspora", + "panafrican", + "african economic community", + "agenda 2063", + ], + "African Philosophy": [ + "ubuntu", + "african philosophy", + "african ethics", + "communalism", + "african metaphysics", + "african ontology", + "african logic", + "african epistemology", + "african humanism", + "african thought", + "african worldview", + "indigenous philosophy", + "sage philosophy", + "negritude philosophy", + "ubuntu ethics", + "african communitarianism", + ], + "Ethnomusicology": [ + "ethnomusicology", + "african music", + "traditional music", + "folk music", + "african drumming", + "musical heritage", + "afrobeats", + "highlife", + "african rhythm", + "musical traditions", + "indigenous music", + "oral musical tradition", + "african instruments", + "kora music", + "mbira music", + "djembe drumming", + "traditional chants", + ], +} + +# ── African Union Charter Targets ───────────────────────────────────────────── +AU_CHARTER_TARGETS: Dict[int, Dict] = { + 1: { + "name": "Tangible & Intangible Cultural Heritage Preservation", + "keywords": [ + "cultural heritage", + "intangible heritage", + "heritage preservation", + "material culture", + "museum studies", + "cultural artifacts", + "heritage conservation", + "world heritage", + "sacred sites", + "living heritage", + "archaeological", + "rock art", + "sacred groves", + ], + }, + 2: { + "name": "Development of African Languages & Decolonization of Science", + "keywords": [ + "indigenous language", + "yoruba", + "igbo", + "hausa", + "swahili", + "amharic", + "zulu", + "xhosa", + "shona", + "somali", + "kinyarwanda", + "oromo", + "twi", + "fante", + "ewe", + "wolof", + "luganda", + "lingala", + "bambara", + "tigrinya", + "chewa", + "ndebele", + "sotho", + "african language", + "ethnic language", + "decolonizing science", + "linguistic diversity", + ], + }, + 3: { + "name": "Integration of Cultural Values & Indigenous Knowledge Systems", + "keywords": [ + "indigenous knowledge", + "traditional knowledge", + "indigenous epistemology", + "ethnobotany", + "traditional ecological knowledge", + "indigenous medicine", + "traditional healing", + "ancestral wisdom", + "traditional practices", + "precolonial knowledge", + "indigenous cosmology", + "traditional religion", + ], + }, + 4: { + "name": "Inter-Institutional Cultural Exchange & Regional Integration", + "keywords": [ + "cultural exchange", + "regional integration", + "pan-africanism", + "african solidarity", + "african union", + "african continental", + "cross-border", + "ecowas", + "sadc", + "east african community", + ], + }, + 5: { + "name": "Support for Creative and Cultural Industries", + "keywords": [ + "creative industry", + "cultural industry", + "african literature", + "african music", + "african novel", + "african drama", + "oral literature", + "traditional music", + "african drumming", + "highlife", + "afrobeats", + "performing arts", + "african cinema", + "storytelling", + "folklore", + ], + }, + 6: { + "name": "Scientific Innovation & Traditional Technology Integration", + "keywords": [ + "traditional technology", + "indigenous technology", + "traditional farming", + "traditional agriculture", + "ethnoveterinary", + "indigenous agriculture", + "traditional metallurgy", + "traditional medicine production", + ], + }, + 7: { + "name": "Youth Engagement & Cultural Education", + "keywords": [ + "cultural transmission", + "cultural education", + "pedagogy", + "oral history", + "folklore", + "oral tradition", + "youth engagement", + "cultural values", + ], + }, + 8: { + "name": "Intellectual Property, Open Access, and Copyright Protection", + "keywords": [ + "intellectual property", + "open access", + "copyright", + "traditional knowledge rights", + "biopiracy", + "patent protection", + "indigenous rights", + "open science", + ], + }, + 9: { + "name": "Decolonial Philosophy & African Thought Systems (Ubuntu)", + "keywords": [ + "ubuntu", + "african philosophy", + "african thought", + "african ethics", + "communalism", + "decolonial", + "decolonization", + "postcolonialism", + "negritude", + "afrocentricity", + "decolonial thought", + "subaltern", + "african worldview", + ], + }, +} + + +def classify_au_targets(title: str, abstract: str, dc_subject: str = "") -> List[Dict]: + """ + Classify a paper against the 9 African Union Charter Targets. + Returns list of {target_number, target_name, score, matched_keywords}. + """ + text = _clean_text(f"{title or ''} {abstract or ''} {dc_subject or ''}").lower() + if not text.strip(): + return [] + + results = [] + for num, defn in AU_CHARTER_TARGETS.items(): + score, matched = _keyword_score(text, defn["keywords"]) + if score >= 1: + results.append( + { + "target_number": num, + "target_name": defn["name"], + "score": score, + "matched_keywords": matched[:6], + } + ) + + results.sort(key=lambda x: -x["score"]) + return results + + +# ── Comprehensive Stop Words ────────────────────────────────────────────────── +# Covers: common English, academic filler, metadata artifacts, XML/JATS tags, +# geographic terms that are too broad, formatting remnants +STOP_WORDS = { + # Common English + "the", + "and", + "for", + "with", + "this", + "that", + "from", + "have", + "been", + "were", + "their", + "which", + "these", + "about", + "other", + "into", + "than", + "more", + "such", + "some", + "what", + "when", + "where", + "there", + "also", + "using", + "used", + "show", + "both", + "each", + "only", + "very", + "well", + "high", + "low", + "new", + "large", + "small", + "significant", + "different", + "similar", + "total", + "however", + "therefore", + "thus", + "hence", + "although", + "despite", + "while", + "after", + "before", + "through", + "across", + "during", + "within", + "among", + "between", + "either", + "neither", + "whether", + "since", + "upon", + "against", + "without", + "under", + "over", + "above", + "below", + "around", + "towards", + "onto", + "itself", + "itself", + "itself", + "they", + "them", + "their", + "those", + "these", + "here", + "then", + "just", + "like", + "make", + "many", + "most", + "much", + "even", + "back", + "still", + "need", + "could", + "would", + "should", + "shall", + "will", + "might", + "must", + "been", + "have", + "does", + "done", + "made", + "said", + "take", + "come", + "find", + "give", + "know", + "look", + "seem", + "feel", + "become", + "include", + "provide", + "require", + "remain", + "suggest", + "indicate", + "demonstrate", + "show", + "reveal", + "confirm", + "report", + "find", + "identify", + "examine", + "determine", + "evaluate", + "assess", + "compare", + "describe", + "present", + "discuss", + "explore", + "investigate", + "conduct", + "perform", + "apply", + "observe", + "measure", + "calculate", + "estimate", + "predict", + "test", + # Academic filler + "study", + "paper", + "research", + "analysis", + "findings", + "results", + "method", + "approach", + "model", + "system", + "review", + "case", + "report", + "effect", + "impact", + "based", + "data", + "conclusion", + "objective", + "background", + "introduction", + "abstract", + "methods", + "discussion", + "purpose", + "aim", + "goal", + "hypothesis", + "evidence", + "sample", + "group", + "population", + "intervention", + "outcome", + "variable", + "factor", + "relationship", + "association", + "correlation", + "significance", + "difference", + "increase", + "decrease", + "higher", + "lower", + "greater", + "lower", + "compared", + "respectively", + "overall", + "showed", + "found", + "reported", + "observed", + "noted", + "significantly", + "p-value", + "confidence", + "interval", + "mean", + "median", + "standard", + "deviation", + "percent", + "percentage", + "proportion", + "ratio", + "rate", + "number", + "approximately", + "moreover", + "furthermore", + "additionally", + "however", + "nevertheless", + "consequently", + "therefore", + "conclusion", + "finally", + "firstly", + "secondly", + "lastly", + "currently", + "recently", + "previously", + "generally", + "specifically", + "particularly", + "mainly", + "primarily", + "mostly", + "largely", + "significantly", + "approximately", + "relatively", + # Metadata / XML / JATS artifacts + "jats", + "jats-inline", + "journal", + "abstract", + "article", + "title", + "italic", + "bold", + "sup", + "sub", + "break", + "para", + "section", + "body", + "front", + "back", + "meta", + "keyword", + "keywords", + "author", + "authors", + "affiliation", + "institution", + "corresponding", + "mailto", + "email", + "grant", + "funding", + "acknowledgment", + "acknowledgements", + "references", + "bibliography", + "footnote", + "table", + "figure", + "appendix", + "supplement", + # Geographic terms too broad to be meaningful keywords + "nigeria", + "nigerian", + "lagos", + "africa", + "african", + "world", + "global", + "international", + "national", + "regional", + "local", + "country", + "countries", + "continent", + "west", + "east", + "south", + "north", + "central", + # Numbers and fragments that slip through + "2266", + "2015", + "2016", + "2017", + "2018", + "2019", + "2020", + "2021", + "2022", + "2023", + "2024", + "2025", + # Short meaningless words (caught by length filter but listed for clarity) + "also", + "into", + "onto", + "upon", + "with", + "from", + "have", + "that", +} + +# Regex patterns for artifacts to strip before processing +_HTML_ENTITY_RE = re.compile(r"&(?:#\d+|#x[0-9a-fA-F]+|[a-zA-Z]+);") +_HTML_TAG_RE = re.compile(r"<[^>]+>") +_JATS_TAG_RE = re.compile(r"\bjats:[a-z\-]+\b", re.IGNORECASE) +_NON_ALPHA_RE = re.compile(r"[^a-zA-Z\s\-]") +_MULTI_SPACE_RE = re.compile(r"\s+") + + +def _clean_text(text: str) -> str: + """Strip HTML entities, XML/JATS tags, and non-alphabetic artifacts.""" + if not text: + return "" + # Decode HTML entities (e.g., &lt; → <) + text = html.unescape(text) + text = html.unescape(text) # Double-decode for double-encoded entities + # Strip remaining HTML/XML tags + text = _HTML_TAG_RE.sub(" ", text) + # Strip JATS namespace tokens + text = _JATS_TAG_RE.sub(" ", text) + # Strip remaining HTML entities + text = _HTML_ENTITY_RE.sub(" ", text) + # Remove non-alphabetic characters (keep hyphens for compound terms) + text = _NON_ALPHA_RE.sub(" ", text) + # Normalise whitespace + text = _MULTI_SPACE_RE.sub(" ", text).strip() + return text + + +def _is_valid_word(word: str) -> bool: + """Return True if the word is a meaningful academic term.""" + if len(word) < 4: + return False + if word in STOP_WORDS: + return False + # Reject pure number strings + if word.isdigit(): + return False + # Reject words that are mostly digits + digit_ratio = sum(c.isdigit() for c in word) / len(word) + if digit_ratio > 0.3: + return False + # Reject very short all-caps (likely acronyms of metadata) + if len(word) <= 4 and word.isupper(): + return False + return True + + +# ── NLP loading ─────────────────────────────────────────────────────────────── +_nlp = None +_spacy_available = False + + +def _load_nlp(): + global _nlp, _spacy_available + if _nlp is not None: + return _nlp + try: + import spacy + + try: + _nlp = spacy.load("en_core_web_sm") + except OSError: + import subprocess + import sys + + subprocess.run( + [sys.executable, "-m", "spacy", "download", "en_core_web_sm"], + capture_output=True, + ) + _nlp = spacy.load("en_core_web_sm") + _spacy_available = True + log.info("spaCy model loaded: en_core_web_sm") + except Exception as e: + log.warning(f"spaCy unavailable ({e}), using TF-IDF fallback") + _spacy_available = False + return _nlp + + +# ── Core classification functions ───────────────────────────────────────────── + +import re + + +def _keyword_score(text: str, keywords: List[str]) -> Tuple[int, List[str]]: + """Score text against a keyword list, return (score, matched_keywords).""" + text_lower = text.lower() + matched = [] + for kw in keywords: + kw_lower = kw.lower() + if re.search(rf"\b{re.escape(kw_lower)}\b", text_lower): + matched.append(kw) + return len(matched), matched + + +def classify_sdgs(title: str, abstract: str, threshold: int = 1) -> List[Dict]: + """ + Classify a paper against all 17 SDGs using AI + keyword matching. + Returns list of {sdg_number, sdg_name, score, matched_keywords} sorted by score desc. + """ + text = _clean_text(f"{title or ''} {abstract or ''}").strip() + if not text: + return [] + + nlp = _load_nlp() + enriched = text.lower() + if _spacy_available and nlp: + try: + doc = nlp(text[:5000]) + noun_chunks = [chunk.text.lower() for chunk in doc.noun_chunks] + enriched = text.lower() + " " + " ".join(noun_chunks) + except Exception: + pass + + results = [] + for sdg_num, defn in SDG_DEFINITIONS.items(): + score, matched = _keyword_score(enriched, defn["core"]) + if score >= threshold: + results.append( + { + "sdg_number": sdg_num, + "sdg_name": defn["name"], + "score": score, + "matched_keywords": matched[:8], + } + ) + + results.sort(key=lambda x: -x["score"]) + return results + + +def classify_special_collections( + title: str, abstract: str, dc_subject: str = "" +) -> List[Dict]: + """ + Classify a paper into special collections categories. + Returns list of {category, score, matched_keywords}. + """ + text = _clean_text(f"{title or ''} {abstract or ''} {dc_subject or ''}").lower() + if not text.strip(): + return [] + + results = [] + for category, keywords in SPECIAL_COLLECTIONS.items(): + score, matched = _keyword_score(text, keywords) + if score >= 1: + results.append( + { + "category": category, + "score": score * 3, + "matched_keywords": matched[:6], + } + ) + + results.sort(key=lambda x: -x["score"]) + return results + + +def extract_keywords( + title: str, abstract: str, top_n: int = 60, all_texts: Optional[List[str]] = None +) -> List[Dict]: + """ + Extract key academic terms using TF-IDF + spaCy NER + bigrams. + + Args: + title: Paper title + abstract: Paper abstract + top_n: Number of keywords to return + all_texts: Optional list of all abstracts/titles for IDF calculation. + If provided, uses corpus-level TF-IDF for better ranking. + + Returns: + [{word, score, count, type}] sorted by relevance score. + """ + raw = f"{title or ''} {abstract or ''}".strip() + text = _clean_text(raw) + if not text: + return [] + + text_lower = text.lower() + + # ── Unigrams ────────────────────────────────────────────────────────────── + words = re.findall(r"\b[a-zA-Z][a-zA-Z\-]{3,}\b", text_lower) + unigrams = [w for w in words if _is_valid_word(w)] + + # ── Bigrams (two-word phrases) ──────────────────────────────────────────── + bigrams = [] + for i in range(len(words) - 1): + w1, w2 = words[i], words[i + 1] + if _is_valid_word(w1) and _is_valid_word(w2): + bigram = f"{w1} {w2}" + bigrams.append(bigram) + + # ── Trigrams (three-word phrases for compound terms) ───────────────────── + trigrams = [] + for i in range(len(words) - 2): + w1, w2, w3 = words[i], words[i + 1], words[i + 2] + if _is_valid_word(w1) and _is_valid_word(w2) and _is_valid_word(w3): + trigrams.append(f"{w1} {w2} {w3}") + + # ── Frequency counts ────────────────────────────────────────────────────── + term_freq: Dict[str, int] = {} + for t in unigrams + bigrams + trigrams: + term_freq[t] = term_freq.get(t, 0) + 1 + + # ── IDF computation (if corpus provided) ───────────────────────────────── + idf_scores: Dict[str, float] = {} + if all_texts and len(all_texts) > 1: + N = len(all_texts) + all_texts_lower = [doc.lower() for doc in all_texts] + candidate_terms = sorted(term_freq.keys(), key=lambda t: -term_freq[t])[:1500] + for term in candidate_terms: + doc_count = sum(1 for doc in all_texts_lower if term in doc) + idf_scores[term] = math.log((N + 1) / (doc_count + 1)) + 1 + for term in term_freq: + if term not in idf_scores: + idf_scores[term] = 1.0 + else: + # Without corpus, use log(freq+1) as a proxy + for term in term_freq: + idf_scores[term] = math.log(term_freq[term] + 2) + + # ── Score = TF * IDF ────────────────────────────────────────────────────── + total = sum(term_freq.values()) or 1 + scored = [] + for term, freq in term_freq.items(): + tf = freq / total + idf = idf_scores.get(term, 1.0) + score = round(tf * idf * 100, 3) + # Boost multi-word phrases (they're more meaningful) + if " " in term: + score *= 1.5 if term.count(" ") == 1 else 2.0 + scored.append( + { + "word": term, + "score": score, + "count": freq, + "type": "phrase" if " " in term else "term", + } + ) + + # ── Add spaCy named entities (boost recognized entities) ───────────────── + nlp = _load_nlp() + if _spacy_available and nlp: + try: + doc = nlp(text[:3000]) + for ent in doc.ents: + # Only include meaningful entity types, skip countries/cities as keywords + if ent.label_ in ( + "ORG", + "PRODUCT", + "WORK_OF_ART", + "EVENT", + "LAW", + "NORP", + ): + ent_text = ent.text.lower() + ent_clean = _clean_text(ent_text) + if len(ent_clean) >= 4 and _is_valid_word(ent_clean.split()[0]): + scored.append( + { + "word": ent_clean, + "score": 6.0, + "count": 1, + "type": "entity", + } + ) + except Exception: + pass + + # ── Sort, deduplicate, return top N ────────────────────────────────────── + scored.sort(key=lambda x: -x["score"]) + seen: set = set() + unique = [] + for item in scored: + key = item["word"].lower().strip() + # Skip if a longer phrase containing this term already present + if key not in seen: + seen.add(key) + # Also mark sub-terms of present phrases to avoid redundancy + unique.append(item) + + return unique[:top_n] + + +def extract_trends_from_corpus(papers: List[Dict], top_n: int = 12) -> List[Dict]: + """ + AI-powered trend extraction from a corpus of papers. + Groups papers into semantic topic clusters using TF-IDF similarity. + + Args: + papers: List of {id, title, abstract, year} dicts + top_n: Number of trends to discover + + Returns: + List of {topic, keywords, paper_count, years, papers} dicts + """ + if not papers: + return [] + + # Build per-year keyword frequency + year_keywords: Dict[int, Dict[str, int]] = {} + all_texts = [f"{p.get('title','')} {p.get('abstract','')}" for p in papers] + + for p in papers: + year = p.get("year") + if not year: + continue + text = _clean_text(f"{p.get('title','')} {p.get('abstract','')}") + words = re.findall(r"\b[a-zA-Z][a-zA-Z\-]{3,}\b", text.lower()) + words = [w for w in words if _is_valid_word(w)] + + # Also extract bigrams + for i in range(len(words) - 1): + w1, w2 = words[i], words[i + 1] + if _is_valid_word(w1) and _is_valid_word(w2): + words.append(f"{w1} {w2}") + + if year not in year_keywords: + year_keywords[year] = {} + for w in words: + year_keywords[year][w] = year_keywords[year].get(w, 0) + 1 + + # Find terms that appear across multiple years (persistent trends) + global_freq: Dict[str, int] = {} + global_year_presence: Dict[str, set] = {} + for year, freq_map in year_keywords.items(): + for term, count in freq_map.items(): + global_freq[term] = global_freq.get(term, 0) + count + if term not in global_year_presence: + global_year_presence[term] = set() + global_year_presence[term].add(year) + + # Score terms: frequency × year spread + N_docs = len(all_texts) + all_texts_lower = [t.lower() for t in all_texts] + candidate_terms = sorted(global_freq.keys(), key=lambda t: -global_freq[t])[:1500] + trend_scores: Dict[str, float] = {} + for term, freq in global_freq.items(): + if term in candidate_terms: + year_spread = len(global_year_presence.get(term, set())) + doc_count = sum(1 for t in all_texts_lower if term in t) + idf = math.log((N_docs + 1) / (doc_count + 1)) + 1 + trend_scores[term] = freq * year_spread * idf + else: + trend_scores[term] = 0.0 + + # Get top terms as seed topics + top_terms = sorted(trend_scores.items(), key=lambda x: -x[1])[: top_n * 3] + + # Group into trends by co-occurrence (simple greedy clustering) + trends = [] + used_terms: set = set() + + for seed_term, seed_score in top_terms: + if seed_term in used_terms: + continue + if len(trends) >= top_n: + break + + # Find related terms (share at least one year and high frequency) + related = [seed_term] + used_terms.add(seed_term) + seed_years = global_year_presence.get(seed_term, set()) + + for term, score in top_terms: + if term in used_terms: + continue + term_years = global_year_presence.get(term, set()) + overlap = len(seed_years & term_years) + if overlap >= 1: + # Check if they appear near each other in text (simple proxy: both in same abstract) + co_occur = sum( + 1 for t in all_texts if seed_term in t.lower() and term in t.lower() + ) + if co_occur >= 1: + related.append(term) + used_terms.add(term) + if len(related) >= 5: + break + + # Find papers matching this trend cluster + trend_papers = [] + for p in papers: + text = f"{p.get('title','')} {p.get('abstract','')}".lower() + if any(t in text for t in related[:3]): + trend_papers.append(p) + + if len(trend_papers) < 2: + continue + + # Build topic label from seed (capitalise key phrase) + label_parts = seed_term.split() + label = " ".join(w.capitalize() for w in label_parts[:3]) + + # Year-by-year paper counts + by_year = {} + for p in trend_papers: + yr = p.get("year") + if yr: + by_year[str(yr)] = by_year.get(str(yr), 0) + 1 + + trends.append( + { + "topic": label, + "keywords": related[:6], + "paper_count": len(trend_papers), + "total": len(trend_papers), + "by_year": by_year, + "papers": [ + {"id": p.get("id"), "title": p.get("title"), "year": p.get("year")} + for p in trend_papers[:20] + ], + } + ) + + trends.sort(key=lambda x: -x["paper_count"]) + return trends[:top_n] diff --git a/uraas/utils/ai_keyword_extractor.py b/uraas/utils/ai_keyword_extractor.py index c9b1e12b7ea9332c8166d88a6751f65d5afbe30e..b06fbf3df0b1abb44c7def539a2f3652a9de2752 100644 --- a/uraas/utils/ai_keyword_extractor.py +++ b/uraas/utils/ai_keyword_extractor.py @@ -1,621 +1,621 @@ -""" -AI-Powered Keyword Extraction System -Uses local LLM (Ollama) for robust keyword extraction and classification -""" - -import json -import logging -import re -from collections import Counter -from typing import Dict, List, Tuple - -logger = logging.getLogger(__name__) - - -class AIKeywordExtractor: - """Extract keywords using AI and semantic analysis""" - - def __init__(self): - self.academic_keywords = self._load_academic_keywords() - self.stop_words = self._load_stop_words() - self.domain_keywords = self._load_domain_keywords() - - def _load_academic_keywords(self) -> Dict[str, List[str]]: - """Load comprehensive academic keyword database""" - return { - "computer_science": [ - "algorithm", - "data structure", - "machine learning", - "deep learning", - "neural network", - "artificial intelligence", - "computer vision", - "natural language processing", - "database", - "software engineering", - "cloud computing", - "cybersecurity", - "blockchain", - "distributed systems", - "programming", - "code", - "software", - "application", - "system", - "network", - "server", - "client", - "protocol", - "encryption", - ], - "medicine": [ - "disease", - "treatment", - "diagnosis", - "patient", - "clinical", - "therapy", - "medication", - "surgery", - "infection", - "cancer", - "cardiovascular", - "neurological", - "respiratory", - "gastrointestinal", - "endocrine", - "immune", - "metabolic", - "genetic", - "pathology", - "pharmacology", - "epidemiology", - "public health", - "vaccine", - ], - "engineering": [ - "design", - "structure", - "material", - "construction", - "mechanical", - "electrical", - "civil", - "chemical", - "thermal", - "fluid", - "stress", - "strain", - "force", - "energy", - "power", - "efficiency", - "optimization", - "simulation", - "prototype", - "manufacturing", - "production", - "quality", - "testing", - ], - "biology": [ - "cell", - "protein", - "gene", - "dna", - "rna", - "enzyme", - "organism", - "species", - "evolution", - "ecology", - "ecosystem", - "photosynthesis", - "metabolism", - "reproduction", - "development", - "behavior", - "adaptation", - "mutation", - "biodiversity", - "conservation", - "microbiology", - "botany", - "zoology", - ], - "chemistry": [ - "molecule", - "atom", - "compound", - "reaction", - "catalyst", - "oxidation", - "reduction", - "acid", - "base", - "salt", - "polymer", - "organic", - "inorganic", - "analytical", - "synthetic", - "spectroscopy", - "chromatography", - "crystallography", - "electrochemistry", - "thermochemistry", - "kinetics", - ], - "physics": [ - "force", - "energy", - "momentum", - "wave", - "particle", - "quantum", - "relativity", - "thermodynamics", - "electromagnetism", - "optics", - "mechanics", - "dynamics", - "kinematics", - "acceleration", - "velocity", - "gravity", - "radiation", - "photon", - "electron", - "nucleus", - "atom", - ], - "mathematics": [ - "theorem", - "proof", - "equation", - "function", - "variable", - "calculus", - "algebra", - "geometry", - "topology", - "analysis", - "probability", - "statistics", - "matrix", - "vector", - "integral", - "derivative", - "limit", - "series", - "number theory", - "combinatorics", - "graph theory", - ], - "social_sciences": [ - "society", - "culture", - "economy", - "politics", - "history", - "psychology", - "sociology", - "anthropology", - "education", - "behavior", - "development", - "research", - "analysis", - "theory", - "method", - "data", - "survey", - "interview", - "qualitative", - "quantitative", - "ethnography", - ], - "business": [ - "market", - "business", - "finance", - "investment", - "profit", - "revenue", - "cost", - "management", - "strategy", - "planning", - "organization", - "leadership", - "team", - "performance", - "customer", - "product", - "service", - "sales", - "marketing", - "supply chain", - "logistics", - "quality", - ], - "environmental": [ - "climate", - "environment", - "sustainability", - "pollution", - "carbon", - "greenhouse", - "renewable", - "energy", - "water", - "soil", - "air", - "ecosystem", - "biodiversity", - "conservation", - "restoration", - "mitigation", - "adaptation", - "green", - "sustainable", - "ecological", - "environmental", - ], - } - - def _load_stop_words(self) -> set: - """Load common stop words""" - return { - "the", - "a", - "an", - "and", - "or", - "but", - "in", - "on", - "at", - "to", - "for", - "of", - "with", - "by", - "from", - "as", - "is", - "was", - "are", - "be", - "been", - "have", - "has", - "had", - "do", - "does", - "did", - "will", - "would", - "could", - "should", - "may", - "might", - "must", - "can", - "this", - "that", - "these", - "those", - "i", - "you", - "he", - "she", - "it", - "we", - "they", - "what", - "which", - "who", - "when", - "where", - "why", - "how", - "all", - "each", - "every", - "both", - "few", - "more", - "most", - "other", - "some", - "such", - "no", - "nor", - "not", - "only", - "own", - "same", - "so", - "than", - "too", - "very", - "just", - "also", - "about", - "above", - "after", - "again", - "against", - "before", - "between", - "during", - "into", - "through", - "under", - "up", - "out", - "over", - "down", - } - - def _load_domain_keywords(self) -> Dict[str, List[str]]: - """Load domain-specific keywords for better classification""" - return { - "research_methods": [ - "study", - "research", - "analysis", - "experiment", - "investigation", - "methodology", - "method", - "approach", - "technique", - "procedure", - "protocol", - "framework", - "model", - "theory", - "hypothesis", - "validation", - "verification", - "testing", - "evaluation", - ], - "statistical": [ - "statistical", - "statistics", - "data", - "analysis", - "correlation", - "regression", - "distribution", - "probability", - "significance", - "hypothesis test", - "confidence interval", - "sample", - "population", - "variance", - "mean", - "median", - "standard deviation", - ], - "publication": [ - "paper", - "article", - "journal", - "conference", - "publication", - "abstract", - "introduction", - "conclusion", - "reference", - "citation", - "author", - "peer review", - "manuscript", - ], - } - - def extract_keywords(self, text: str, top_n: int = 10) -> List[Tuple[str, float]]: - """ - Extract keywords from text using TF-IDF and semantic analysis - - Args: - text: Input text to extract keywords from - top_n: Number of top keywords to return - - Returns: - List of (keyword, score) tuples - """ - if not text or len(text.strip()) < 10: - return [] - - # Preprocess text - text_lower = text.lower() - words = self._tokenize(text_lower) - - # Filter stop words and short words - filtered_words = [w for w in words if w not in self.stop_words and len(w) > 2] - - if not filtered_words: - return [] - - # Calculate TF-IDF scores - word_freq = Counter(filtered_words) - total_words = len(filtered_words) - - # Calculate scores with domain boost - scores = {} - for word, freq in word_freq.items(): - tf = freq / total_words - - # Boost score for domain keywords - domain_boost = 1.0 - for domain, keywords in self.domain_keywords.items(): - if word in keywords: - domain_boost = 1.5 - break - - # Boost score for academic keywords - academic_boost = 1.0 - for field, keywords in self.academic_keywords.items(): - if word in keywords: - academic_boost = 2.0 - break - - # Calculate IDF (inverse document frequency) - # Simpler version: penalize very common words - idf = 1.0 if freq < total_words * 0.3 else 0.5 - - scores[word] = tf * idf * domain_boost * academic_boost - - # Sort and return top keywords - sorted_keywords = sorted(scores.items(), key=lambda x: x[1], reverse=True) - return sorted_keywords[:top_n] - - def classify_domain(self, text: str) -> List[Tuple[str, float]]: - """ - Classify text into academic domains - - Args: - text: Input text - - Returns: - List of (domain, confidence) tuples - """ - text_lower = text.lower() - domain_scores = {} - - for domain, keywords in self.academic_keywords.items(): - matches = sum(1 for kw in keywords if kw in text_lower) - if matches > 0: - confidence = min(matches / len(keywords), 1.0) - domain_scores[domain] = confidence - - # Sort by confidence - sorted_domains = sorted(domain_scores.items(), key=lambda x: x[1], reverse=True) - - return sorted_domains - - def extract_entities(self, text: str) -> Dict[str, List[str]]: - """ - Extract named entities and important phrases - - Args: - text: Input text - - Returns: - Dictionary of entity types and values - """ - entities = { - "organizations": [], - "locations": [], - "people": [], - "methods": [], - "measurements": [], - } - - # Extract organizations (capitalized words followed by Inc, Ltd, University, etc.) - org_pattern = r"\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\s+(?:Inc|Ltd|University|College|Institute|Lab|Center)\b" - entities["organizations"] = re.findall(org_pattern, text) - - # Extract measurements (numbers with units) - measurement_pattern = ( - r"\b\d+(?:\.\d+)?\s*(?:mg|g|kg|ml|l|m|cm|mm|°C|°F|%|ppm|ppb)\b" - ) - entities["measurements"] = re.findall(measurement_pattern, text, re.IGNORECASE) - - # Extract methods (words ending in -tion, -sis, -graphy) - method_pattern = r"\b\w+(?:tion|sis|graphy|metry|scopy|logy)\b" - entities["methods"] = list( - set(re.findall(method_pattern, text, re.IGNORECASE)) - )[:10] - - return entities - - def _tokenize(self, text: str) -> List[str]: - """Tokenize text into words""" - # Remove special characters but keep hyphens and apostrophes - text = re.sub(r"[^\w\s\-\']", " ", text) - # Split on whitespace - words = text.split() - return words - - def score_paper(self, title: str, abstract: str) -> Dict: - """ - Score a paper for quality and relevance - - Args: - title: Paper title - abstract: Paper abstract - - Returns: - Dictionary with scores - """ - combined_text = f"{title} {abstract}" - - # Extract keywords - keywords = self.extract_keywords(combined_text, top_n=15) - - # Classify domain - domains = self.classify_domain(combined_text) - - # Extract entities - entities = self.extract_entities(combined_text) - - # Calculate quality score - quality_score = 0.0 - - # Title quality (should be 5-15 words) - title_words = len(title.split()) - if 5 <= title_words <= 15: - quality_score += 0.2 - - # Abstract quality (should be 100-300 words) - abstract_words = len(abstract.split()) - if 100 <= abstract_words <= 300: - quality_score += 0.2 - - # Keyword diversity - if len(keywords) >= 10: - quality_score += 0.2 - - # Domain classification - if domains and domains[0][1] > 0.5: - quality_score += 0.2 - - # Entity extraction - if entities["methods"] or entities["measurements"]: - quality_score += 0.2 - - return { - "quality_score": min(quality_score, 1.0), - "keywords": keywords, - "domains": domains, - "entities": entities, - "title_length": title_words, - "abstract_length": abstract_words, - } - - -# Global instance -ai_extractor = AIKeywordExtractor() - - -def extract_keywords(text: str, top_n: int = 10) -> List[Tuple[str, float]]: - """Convenience function to extract keywords""" - return ai_extractor.extract_keywords(text, top_n) - - -def classify_domain(text: str) -> List[Tuple[str, float]]: - """Convenience function to classify domain""" - return ai_extractor.classify_domain(text) - - -def score_paper(title: str, abstract: str) -> Dict: - """Convenience function to score paper""" - return ai_extractor.score_paper(title, abstract) +""" +AI-Powered Keyword Extraction System +Uses local LLM (Ollama) for robust keyword extraction and classification +""" + +import json +import logging +import re +from collections import Counter +from typing import Dict, List, Tuple + +logger = logging.getLogger(__name__) + + +class AIKeywordExtractor: + """Extract keywords using AI and semantic analysis""" + + def __init__(self): + self.academic_keywords = self._load_academic_keywords() + self.stop_words = self._load_stop_words() + self.domain_keywords = self._load_domain_keywords() + + def _load_academic_keywords(self) -> Dict[str, List[str]]: + """Load comprehensive academic keyword database""" + return { + "computer_science": [ + "algorithm", + "data structure", + "machine learning", + "deep learning", + "neural network", + "artificial intelligence", + "computer vision", + "natural language processing", + "database", + "software engineering", + "cloud computing", + "cybersecurity", + "blockchain", + "distributed systems", + "programming", + "code", + "software", + "application", + "system", + "network", + "server", + "client", + "protocol", + "encryption", + ], + "medicine": [ + "disease", + "treatment", + "diagnosis", + "patient", + "clinical", + "therapy", + "medication", + "surgery", + "infection", + "cancer", + "cardiovascular", + "neurological", + "respiratory", + "gastrointestinal", + "endocrine", + "immune", + "metabolic", + "genetic", + "pathology", + "pharmacology", + "epidemiology", + "public health", + "vaccine", + ], + "engineering": [ + "design", + "structure", + "material", + "construction", + "mechanical", + "electrical", + "civil", + "chemical", + "thermal", + "fluid", + "stress", + "strain", + "force", + "energy", + "power", + "efficiency", + "optimization", + "simulation", + "prototype", + "manufacturing", + "production", + "quality", + "testing", + ], + "biology": [ + "cell", + "protein", + "gene", + "dna", + "rna", + "enzyme", + "organism", + "species", + "evolution", + "ecology", + "ecosystem", + "photosynthesis", + "metabolism", + "reproduction", + "development", + "behavior", + "adaptation", + "mutation", + "biodiversity", + "conservation", + "microbiology", + "botany", + "zoology", + ], + "chemistry": [ + "molecule", + "atom", + "compound", + "reaction", + "catalyst", + "oxidation", + "reduction", + "acid", + "base", + "salt", + "polymer", + "organic", + "inorganic", + "analytical", + "synthetic", + "spectroscopy", + "chromatography", + "crystallography", + "electrochemistry", + "thermochemistry", + "kinetics", + ], + "physics": [ + "force", + "energy", + "momentum", + "wave", + "particle", + "quantum", + "relativity", + "thermodynamics", + "electromagnetism", + "optics", + "mechanics", + "dynamics", + "kinematics", + "acceleration", + "velocity", + "gravity", + "radiation", + "photon", + "electron", + "nucleus", + "atom", + ], + "mathematics": [ + "theorem", + "proof", + "equation", + "function", + "variable", + "calculus", + "algebra", + "geometry", + "topology", + "analysis", + "probability", + "statistics", + "matrix", + "vector", + "integral", + "derivative", + "limit", + "series", + "number theory", + "combinatorics", + "graph theory", + ], + "social_sciences": [ + "society", + "culture", + "economy", + "politics", + "history", + "psychology", + "sociology", + "anthropology", + "education", + "behavior", + "development", + "research", + "analysis", + "theory", + "method", + "data", + "survey", + "interview", + "qualitative", + "quantitative", + "ethnography", + ], + "business": [ + "market", + "business", + "finance", + "investment", + "profit", + "revenue", + "cost", + "management", + "strategy", + "planning", + "organization", + "leadership", + "team", + "performance", + "customer", + "product", + "service", + "sales", + "marketing", + "supply chain", + "logistics", + "quality", + ], + "environmental": [ + "climate", + "environment", + "sustainability", + "pollution", + "carbon", + "greenhouse", + "renewable", + "energy", + "water", + "soil", + "air", + "ecosystem", + "biodiversity", + "conservation", + "restoration", + "mitigation", + "adaptation", + "green", + "sustainable", + "ecological", + "environmental", + ], + } + + def _load_stop_words(self) -> set: + """Load common stop words""" + return { + "the", + "a", + "an", + "and", + "or", + "but", + "in", + "on", + "at", + "to", + "for", + "of", + "with", + "by", + "from", + "as", + "is", + "was", + "are", + "be", + "been", + "have", + "has", + "had", + "do", + "does", + "did", + "will", + "would", + "could", + "should", + "may", + "might", + "must", + "can", + "this", + "that", + "these", + "those", + "i", + "you", + "he", + "she", + "it", + "we", + "they", + "what", + "which", + "who", + "when", + "where", + "why", + "how", + "all", + "each", + "every", + "both", + "few", + "more", + "most", + "other", + "some", + "such", + "no", + "nor", + "not", + "only", + "own", + "same", + "so", + "than", + "too", + "very", + "just", + "also", + "about", + "above", + "after", + "again", + "against", + "before", + "between", + "during", + "into", + "through", + "under", + "up", + "out", + "over", + "down", + } + + def _load_domain_keywords(self) -> Dict[str, List[str]]: + """Load domain-specific keywords for better classification""" + return { + "research_methods": [ + "study", + "research", + "analysis", + "experiment", + "investigation", + "methodology", + "method", + "approach", + "technique", + "procedure", + "protocol", + "framework", + "model", + "theory", + "hypothesis", + "validation", + "verification", + "testing", + "evaluation", + ], + "statistical": [ + "statistical", + "statistics", + "data", + "analysis", + "correlation", + "regression", + "distribution", + "probability", + "significance", + "hypothesis test", + "confidence interval", + "sample", + "population", + "variance", + "mean", + "median", + "standard deviation", + ], + "publication": [ + "paper", + "article", + "journal", + "conference", + "publication", + "abstract", + "introduction", + "conclusion", + "reference", + "citation", + "author", + "peer review", + "manuscript", + ], + } + + def extract_keywords(self, text: str, top_n: int = 10) -> List[Tuple[str, float]]: + """ + Extract keywords from text using TF-IDF and semantic analysis + + Args: + text: Input text to extract keywords from + top_n: Number of top keywords to return + + Returns: + List of (keyword, score) tuples + """ + if not text or len(text.strip()) < 10: + return [] + + # Preprocess text + text_lower = text.lower() + words = self._tokenize(text_lower) + + # Filter stop words and short words + filtered_words = [w for w in words if w not in self.stop_words and len(w) > 2] + + if not filtered_words: + return [] + + # Calculate TF-IDF scores + word_freq = Counter(filtered_words) + total_words = len(filtered_words) + + # Calculate scores with domain boost + scores = {} + for word, freq in word_freq.items(): + tf = freq / total_words + + # Boost score for domain keywords + domain_boost = 1.0 + for domain, keywords in self.domain_keywords.items(): + if word in keywords: + domain_boost = 1.5 + break + + # Boost score for academic keywords + academic_boost = 1.0 + for field, keywords in self.academic_keywords.items(): + if word in keywords: + academic_boost = 2.0 + break + + # Calculate IDF (inverse document frequency) + # Simpler version: penalize very common words + idf = 1.0 if freq < total_words * 0.3 else 0.5 + + scores[word] = tf * idf * domain_boost * academic_boost + + # Sort and return top keywords + sorted_keywords = sorted(scores.items(), key=lambda x: x[1], reverse=True) + return sorted_keywords[:top_n] + + def classify_domain(self, text: str) -> List[Tuple[str, float]]: + """ + Classify text into academic domains + + Args: + text: Input text + + Returns: + List of (domain, confidence) tuples + """ + text_lower = text.lower() + domain_scores = {} + + for domain, keywords in self.academic_keywords.items(): + matches = sum(1 for kw in keywords if kw in text_lower) + if matches > 0: + confidence = min(matches / len(keywords), 1.0) + domain_scores[domain] = confidence + + # Sort by confidence + sorted_domains = sorted(domain_scores.items(), key=lambda x: x[1], reverse=True) + + return sorted_domains + + def extract_entities(self, text: str) -> Dict[str, List[str]]: + """ + Extract named entities and important phrases + + Args: + text: Input text + + Returns: + Dictionary of entity types and values + """ + entities = { + "organizations": [], + "locations": [], + "people": [], + "methods": [], + "measurements": [], + } + + # Extract organizations (capitalized words followed by Inc, Ltd, University, etc.) + org_pattern = r"\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\s+(?:Inc|Ltd|University|College|Institute|Lab|Center)\b" + entities["organizations"] = re.findall(org_pattern, text) + + # Extract measurements (numbers with units) + measurement_pattern = ( + r"\b\d+(?:\.\d+)?\s*(?:mg|g|kg|ml|l|m|cm|mm|°C|°F|%|ppm|ppb)\b" + ) + entities["measurements"] = re.findall(measurement_pattern, text, re.IGNORECASE) + + # Extract methods (words ending in -tion, -sis, -graphy) + method_pattern = r"\b\w+(?:tion|sis|graphy|metry|scopy|logy)\b" + entities["methods"] = list( + set(re.findall(method_pattern, text, re.IGNORECASE)) + )[:10] + + return entities + + def _tokenize(self, text: str) -> List[str]: + """Tokenize text into words""" + # Remove special characters but keep hyphens and apostrophes + text = re.sub(r"[^\w\s\-\']", " ", text) + # Split on whitespace + words = text.split() + return words + + def score_paper(self, title: str, abstract: str) -> Dict: + """ + Score a paper for quality and relevance + + Args: + title: Paper title + abstract: Paper abstract + + Returns: + Dictionary with scores + """ + combined_text = f"{title} {abstract}" + + # Extract keywords + keywords = self.extract_keywords(combined_text, top_n=15) + + # Classify domain + domains = self.classify_domain(combined_text) + + # Extract entities + entities = self.extract_entities(combined_text) + + # Calculate quality score + quality_score = 0.0 + + # Title quality (should be 5-15 words) + title_words = len(title.split()) + if 5 <= title_words <= 15: + quality_score += 0.2 + + # Abstract quality (should be 100-300 words) + abstract_words = len(abstract.split()) + if 100 <= abstract_words <= 300: + quality_score += 0.2 + + # Keyword diversity + if len(keywords) >= 10: + quality_score += 0.2 + + # Domain classification + if domains and domains[0][1] > 0.5: + quality_score += 0.2 + + # Entity extraction + if entities["methods"] or entities["measurements"]: + quality_score += 0.2 + + return { + "quality_score": min(quality_score, 1.0), + "keywords": keywords, + "domains": domains, + "entities": entities, + "title_length": title_words, + "abstract_length": abstract_words, + } + + +# Global instance +ai_extractor = AIKeywordExtractor() + + +def extract_keywords(text: str, top_n: int = 10) -> List[Tuple[str, float]]: + """Convenience function to extract keywords""" + return ai_extractor.extract_keywords(text, top_n) + + +def classify_domain(text: str) -> List[Tuple[str, float]]: + """Convenience function to classify domain""" + return ai_extractor.classify_domain(text) + + +def score_paper(title: str, abstract: str) -> Dict: + """Convenience function to score paper""" + return ai_extractor.score_paper(title, abstract) diff --git a/uraas/utils/analytics_cache.py b/uraas/utils/analytics_cache.py index 4f18d5d34a05afafc8340b676b9f5dabb3a81b54..8f5ee6deb42e917ee37ea9c833ee18682bd72ac7 100644 --- a/uraas/utils/analytics_cache.py +++ b/uraas/utils/analytics_cache.py @@ -1,78 +1,78 @@ -""" -URAAS Analytics Cache -Thread-safe in-memory cache with TTL for expensive analytics computations. -Invalidated on crawl completion. -""" - -import logging -import threading -import time -from typing import Any, Dict, Optional - -log = logging.getLogger(__name__) - -_DEFAULT_TTL = 1800 # 30 minutes - - -class _CacheEntry: - __slots__ = ("value", "expires_at") - - def __init__(self, value: Any, ttl: int): - self.value = value - self.expires_at = time.monotonic() + ttl - - -class AnalyticsCache: - """Thread-safe in-memory key-value cache with TTL expiry.""" - - def __init__(self, default_ttl: int = _DEFAULT_TTL): - self._store: Dict[str, _CacheEntry] = {} - self._lock = threading.Lock() - self._default_ttl = default_ttl - - def get(self, key: str) -> Optional[Any]: - with self._lock: - entry = self._store.get(key) - if entry is None: - return None - if time.monotonic() > entry.expires_at: - del self._store[key] - log.debug("Cache MISS (expired): %s", key) - return None - log.debug("Cache HIT: %s", key) - return entry.value - - def set(self, key: str, value: Any, ttl: Optional[int] = None) -> None: - with self._lock: - self._store[key] = _CacheEntry(value, ttl or self._default_ttl) - log.debug("Cache SET: %s (ttl=%ss)", key, ttl or self._default_ttl) - - def invalidate(self, key: str) -> None: - with self._lock: - self._store.pop(key, None) - - def invalidate_all(self) -> None: - """Call this when new papers are crawled to flush stale analytics.""" - with self._lock: - count = len(self._store) - self._store.clear() - log.info("Analytics cache flushed (%d entries cleared)", count) - - def invalidate_prefix(self, prefix: str) -> None: - """Invalidate all keys starting with a given prefix.""" - with self._lock: - keys = [k for k in self._store if k.startswith(prefix)] - for k in keys: - del self._store[k] - log.debug( - "Invalidated %d cache entries with prefix '%s'", len(keys), prefix - ) - - @property - def size(self) -> int: - with self._lock: - return len(self._store) - - -# Singleton instance shared across the app -analytics_cache = AnalyticsCache(default_ttl=_DEFAULT_TTL) +""" +URAAS Analytics Cache +Thread-safe in-memory cache with TTL for expensive analytics computations. +Invalidated on crawl completion. +""" + +import logging +import threading +import time +from typing import Any, Dict, Optional + +log = logging.getLogger(__name__) + +_DEFAULT_TTL = 1800 # 30 minutes + + +class _CacheEntry: + __slots__ = ("value", "expires_at") + + def __init__(self, value: Any, ttl: int): + self.value = value + self.expires_at = time.monotonic() + ttl + + +class AnalyticsCache: + """Thread-safe in-memory key-value cache with TTL expiry.""" + + def __init__(self, default_ttl: int = _DEFAULT_TTL): + self._store: Dict[str, _CacheEntry] = {} + self._lock = threading.Lock() + self._default_ttl = default_ttl + + def get(self, key: str) -> Optional[Any]: + with self._lock: + entry = self._store.get(key) + if entry is None: + return None + if time.monotonic() > entry.expires_at: + del self._store[key] + log.debug("Cache MISS (expired): %s", key) + return None + log.debug("Cache HIT: %s", key) + return entry.value + + def set(self, key: str, value: Any, ttl: Optional[int] = None) -> None: + with self._lock: + self._store[key] = _CacheEntry(value, ttl or self._default_ttl) + log.debug("Cache SET: %s (ttl=%ss)", key, ttl or self._default_ttl) + + def invalidate(self, key: str) -> None: + with self._lock: + self._store.pop(key, None) + + def invalidate_all(self) -> None: + """Call this when new papers are crawled to flush stale analytics.""" + with self._lock: + count = len(self._store) + self._store.clear() + log.info("Analytics cache flushed (%d entries cleared)", count) + + def invalidate_prefix(self, prefix: str) -> None: + """Invalidate all keys starting with a given prefix.""" + with self._lock: + keys = [k for k in self._store if k.startswith(prefix)] + for k in keys: + del self._store[k] + log.debug( + "Invalidated %d cache entries with prefix '%s'", len(keys), prefix + ) + + @property + def size(self) -> int: + with self._lock: + return len(self._store) + + +# Singleton instance shared across the app +analytics_cache = AnalyticsCache(default_ttl=_DEFAULT_TTL) diff --git a/uraas/utils/ark_generator.py b/uraas/utils/ark_generator.py index 2a630345586ddd7eb3636bbd4c5fc3019a6bc62c..d76934dad6c9a1a6a7b032cc7dede310760d6c20 100644 --- a/uraas/utils/ark_generator.py +++ b/uraas/utils/ark_generator.py @@ -1,81 +1,81 @@ -""" -ARK (Archival Resource Key) generator for URAAS. - -ARKs are free, decentralised persistent identifiers (arks.org). The Africa -PID Alliance partnered with the ARK Alliance (March 2025) to strengthen PID -infrastructure in Africa — URAAS mints an ARK for every item alongside its -DocID™ so outputs without DOIs still carry a resolvable persistent ID. - -Format: ark:// -- NAAN: Name Assigning Authority Number (env ARK_NAAN; 99999 = official - test NAAN until the production registration completes). -- shoulder: sub-namespace (env ARK_SHOULDER, default "u1"). -- name: 12 betanumeric chars derived deterministically from the seed - (the item's DocID hash), so re-minting is idempotent. -- check: NCDA (Noid Check Digit Algorithm) character. -""" - -import hashlib -import re - -from uraas.config import config - -# Betanumeric alphabet (NOID's "extended digits"): digits + consonants, -# excluding vowels and 'l' to avoid accidental words and misreads. -BETANUMERIC = "0123456789bcdfghjkmnpqrstvwxz" - -_ARK_RE = re.compile(r"^ark:/(\d{5,9})/([" + BETANUMERIC + r"]+)$") - - -class ARKGenerator: - """Mint and validate ARK identifiers (deterministic from a seed).""" - - @property - def naan(self) -> str: - return config.ARK_NAAN - - @property - def shoulder(self) -> str: - return config.ARK_SHOULDER - - @classmethod - def _check_char(cls, naan: str, name: str) -> str: - """NCDA check character over the NAAN + name portion. - - Each char maps to its betanumeric ordinal (non-betanumeric chars - count as 0), weighted by 1-based position; sum mod 29 indexes the - alphabet (the standard NOID checkchar algorithm).""" - s = f"{naan}/{name}" - total = 0 - for pos, ch in enumerate(s, start=1): - total += BETANUMERIC.find(ch) * pos if ch in BETANUMERIC else 0 - return BETANUMERIC[total % len(BETANUMERIC)] - - def mint(self, seed: str) -> str: - """Deterministically mint an ARK from a seed string (the item DocID). - - Same seed → same ARK, so backfills are idempotent.""" - digest = hashlib.sha256((seed or "").encode("utf-8")).digest() - # Map hash bytes onto the betanumeric alphabet for a 12-char name. - body = "".join(BETANUMERIC[b % len(BETANUMERIC)] for b in digest[:12]) - name = f"{self.shoulder}{body}" - return f"ark:/{self.naan}/{name}{self._check_char(self.naan, name)}" - - @classmethod - def validate(cls, ark: str) -> bool: - m = _ARK_RE.match(ark or "") - if not m: - return False - naan, full_name = m.groups() - if len(full_name) < 4: - return False - name, check = full_name[:-1], full_name[-1] - return cls._check_char(naan, name) == check - - @staticmethod - def to_url(ark: str) -> str: - """Local resolver path (ARK spec: resolver base + '/' + ark).""" - return f"/{ark}" - - -ark_generator = ARKGenerator() +""" +ARK (Archival Resource Key) generator for URAAS. + +ARKs are free, decentralised persistent identifiers (arks.org). The Africa +PID Alliance partnered with the ARK Alliance (March 2025) to strengthen PID +infrastructure in Africa — URAAS mints an ARK for every item alongside its +DocID™ so outputs without DOIs still carry a resolvable persistent ID. + +Format: ark:// +- NAAN: Name Assigning Authority Number (env ARK_NAAN; 99999 = official + test NAAN until the production registration completes). +- shoulder: sub-namespace (env ARK_SHOULDER, default "u1"). +- name: 12 betanumeric chars derived deterministically from the seed + (the item's DocID hash), so re-minting is idempotent. +- check: NCDA (Noid Check Digit Algorithm) character. +""" + +import hashlib +import re + +from uraas.config import config + +# Betanumeric alphabet (NOID's "extended digits"): digits + consonants, +# excluding vowels and 'l' to avoid accidental words and misreads. +BETANUMERIC = "0123456789bcdfghjkmnpqrstvwxz" + +_ARK_RE = re.compile(r"^ark:/(\d{5,9})/([" + BETANUMERIC + r"]+)$") + + +class ARKGenerator: + """Mint and validate ARK identifiers (deterministic from a seed).""" + + @property + def naan(self) -> str: + return config.ARK_NAAN + + @property + def shoulder(self) -> str: + return config.ARK_SHOULDER + + @classmethod + def _check_char(cls, naan: str, name: str) -> str: + """NCDA check character over the NAAN + name portion. + + Each char maps to its betanumeric ordinal (non-betanumeric chars + count as 0), weighted by 1-based position; sum mod 29 indexes the + alphabet (the standard NOID checkchar algorithm).""" + s = f"{naan}/{name}" + total = 0 + for pos, ch in enumerate(s, start=1): + total += BETANUMERIC.find(ch) * pos if ch in BETANUMERIC else 0 + return BETANUMERIC[total % len(BETANUMERIC)] + + def mint(self, seed: str) -> str: + """Deterministically mint an ARK from a seed string (the item DocID). + + Same seed → same ARK, so backfills are idempotent.""" + digest = hashlib.sha256((seed or "").encode("utf-8")).digest() + # Map hash bytes onto the betanumeric alphabet for a 12-char name. + body = "".join(BETANUMERIC[b % len(BETANUMERIC)] for b in digest[:12]) + name = f"{self.shoulder}{body}" + return f"ark:/{self.naan}/{name}{self._check_char(self.naan, name)}" + + @classmethod + def validate(cls, ark: str) -> bool: + m = _ARK_RE.match(ark or "") + if not m: + return False + naan, full_name = m.groups() + if len(full_name) < 4: + return False + name, check = full_name[:-1], full_name[-1] + return cls._check_char(naan, name) == check + + @staticmethod + def to_url(ark: str) -> str: + """Local resolver path (ARK spec: resolver base + '/' + ark).""" + return f"/{ark}" + + +ark_generator = ARKGenerator() diff --git a/uraas/utils/docid_generator.py b/uraas/utils/docid_generator.py index ddebc8278a12a003ffee3f7280affc58384e49bc..91a46f71881a4f7fff334f4f826737a0df0ad1cd 100644 --- a/uraas/utils/docid_generator.py +++ b/uraas/utils/docid_generator.py @@ -1,202 +1,202 @@ -""" -DocID Generator for URAAS -Implements the Africa PID Alliance DocID™ persistent identifier system. - -DocID Format: 20.500.14351/[unique-hash] -- 20.500.14351 is the APA Handle prefix -- Unique hash ensures global uniqueness and long-term accessibility -""" - -import hashlib -import uuid -from datetime import datetime -from typing import Dict, Optional - - -class DocIDGenerator: - """ - Generate DocID™ persistent identifiers following Africa PID Alliance standards. - - DocID™ is a persistent identifier (PID) system developed by TCC Africa and - the Africa PID Alliance to secure African research outputs, indigenous knowledge, - and cultural heritage. - """ - - # APA Handle prefix for Africa PID Alliance - APA_HANDLE_PREFIX = "20.500.14351" - - # Supported identifier types - IDENTIFIER_TYPES = [ - "DOCiD", # Internal identifier format - "APA Handle ID", # African PID Alliance Handle Service - "DOI", # Digital Object Identifier - "Handle", # Handle System identifier - "ARK", # Archival Resource Key - "URN", # Uniform Resource Name - "ORCID", # Creator identification - "ROR", # Organization identification - ] - - @classmethod - def generate_docid( - cls, - title: str, - doi: Optional[str] = None, - institution: str = "University of Lagos", - timestamp: Optional[datetime] = None, - ) -> str: - """ - Generate a unique DocID™ identifier. - - Args: - title: Publication title - doi: Existing DOI (if available) - institution: Institution name - timestamp: Publication timestamp (defaults to now) - - Returns: - DocID in format: 20.500.14351/[unique-hash] - """ - if timestamp is None: - timestamp = datetime.utcnow() - - # Create unique string from multiple sources - unique_string = ( - f"{title}|{doi or ''}|{institution}|{timestamp.isoformat()}|{uuid.uuid4()}" - ) - - # Generate SHA-256 hash and take first 20 characters for readability - hash_object = hashlib.sha256(unique_string.encode("utf-8")) - unique_hash = hash_object.hexdigest()[:20] - - # Return DocID in APA Handle format - return f"{cls.APA_HANDLE_PREFIX}/{unique_hash}" - - @classmethod - def generate_docid_metadata(cls, paper_data: Dict) -> Dict: - """ - Generate complete DocID™ metadata package for a publication. - - Args: - paper_data: Dictionary containing paper information - - Returns: - Dictionary with DocID and associated metadata - """ - docid = cls.generate_docid( - title=paper_data.get("title", "Untitled"), - doi=paper_data.get("doi"), - institution=paper_data.get("institution", "University of Lagos"), - timestamp=paper_data.get("publication_date"), - ) - - metadata = { - # Primary identifier - "document_docid": docid, - "docid_assigned_date": datetime.utcnow().isoformat(), - # Identifier type - "identifier_type": "APA Handle ID", - "identifier_scheme": "Handle", - # Alternate identifiers - "alternate_identifiers": [], - # Metadata - "title": paper_data.get("title"), - "institution": paper_data.get("institution", "University of Lagos"), - "institution_ror": "https://ror.org/05rk03822", # UNILAG ROR ID - # Provenance - "source_repository": paper_data.get( - "source_repository", "UNILAG Institutional Repository" - ), - "source_url": paper_data.get("url"), - # Handle resolution URL - "handle_url": f"https://hdl.handle.net/{docid}", - "docid_url": f"https://docid.africapidalliance.org/resolve/{docid}", - } - - # Add DOI as alternate identifier if present - if paper_data.get("doi"): - metadata["alternate_identifiers"].append( - {"type": "DOI", "value": paper_data["doi"], "is_primary": False} - ) - - # Add ORCID for authors if available - if paper_data.get("authors"): - for author in paper_data["authors"]: - if author.get("orcid"): - metadata["alternate_identifiers"].append( - {"type": "ORCID", "value": author["orcid"], "role": "Creator"} - ) - - return metadata - - @classmethod - def validate_docid(cls, docid: str) -> bool: - """ - Validate a DocID™ identifier format. - - Args: - docid: DocID string to validate - - Returns: - True if valid, False otherwise - """ - if not docid: - return False - - parts = docid.split("/") - if len(parts) != 2: - return False - - prefix, hash_part = parts - - # Check prefix matches APA Handle - if prefix != cls.APA_HANDLE_PREFIX: - return False - - # Check hash part is alphanumeric and reasonable length - if not hash_part or not hash_part.isalnum(): - return False - - if len(hash_part) < 10 or len(hash_part) > 64: - return False - - return True - - @classmethod - def format_citation_with_docid(cls, paper_data: Dict, docid: str) -> str: - """ - Format a citation including the DocID™ identifier. - - Args: - paper_data: Paper metadata - docid: DocID identifier - - Returns: - Formatted citation string - """ - authors = paper_data.get("authors", []) - author_str = ", ".join([a.get("name", "") for a in authors[:3]]) - if len(authors) > 3: - author_str += " et al." - - title = paper_data.get("title", "Untitled") - - # Handle both string and datetime objects for publication_date - pub_date = paper_data.get("publication_date") - if isinstance(pub_date, datetime): - year = pub_date.year - elif isinstance(pub_date, str): - year = pub_date.split("-")[0] if pub_date else "n.d." - else: - year = "n.d." - - citation = f"{author_str} ({year}). {title}. " - citation += f"University of Lagos Institutional Repository. " - citation += f"DocID: {docid}. " - citation += f"Available at: https://hdl.handle.net/{docid}" - - return citation - - -# Singleton instance -docid_generator = DocIDGenerator() +""" +DocID Generator for URAAS +Implements the Africa PID Alliance DocID™ persistent identifier system. + +DocID Format: 20.500.14351/[unique-hash] +- 20.500.14351 is the APA Handle prefix +- Unique hash ensures global uniqueness and long-term accessibility +""" + +import hashlib +import uuid +from datetime import datetime +from typing import Dict, Optional + + +class DocIDGenerator: + """ + Generate DocID™ persistent identifiers following Africa PID Alliance standards. + + DocID™ is a persistent identifier (PID) system developed by TCC Africa and + the Africa PID Alliance to secure African research outputs, indigenous knowledge, + and cultural heritage. + """ + + # APA Handle prefix for Africa PID Alliance + APA_HANDLE_PREFIX = "20.500.14351" + + # Supported identifier types + IDENTIFIER_TYPES = [ + "DOCiD", # Internal identifier format + "APA Handle ID", # African PID Alliance Handle Service + "DOI", # Digital Object Identifier + "Handle", # Handle System identifier + "ARK", # Archival Resource Key + "URN", # Uniform Resource Name + "ORCID", # Creator identification + "ROR", # Organization identification + ] + + @classmethod + def generate_docid( + cls, + title: str, + doi: Optional[str] = None, + institution: str = "University of Lagos", + timestamp: Optional[datetime] = None, + ) -> str: + """ + Generate a unique DocID™ identifier. + + Args: + title: Publication title + doi: Existing DOI (if available) + institution: Institution name + timestamp: Publication timestamp (defaults to now) + + Returns: + DocID in format: 20.500.14351/[unique-hash] + """ + if timestamp is None: + timestamp = datetime.utcnow() + + # Create unique string from multiple sources + unique_string = ( + f"{title}|{doi or ''}|{institution}|{timestamp.isoformat()}|{uuid.uuid4()}" + ) + + # Generate SHA-256 hash and take first 20 characters for readability + hash_object = hashlib.sha256(unique_string.encode("utf-8")) + unique_hash = hash_object.hexdigest()[:20] + + # Return DocID in APA Handle format + return f"{cls.APA_HANDLE_PREFIX}/{unique_hash}" + + @classmethod + def generate_docid_metadata(cls, paper_data: Dict) -> Dict: + """ + Generate complete DocID™ metadata package for a publication. + + Args: + paper_data: Dictionary containing paper information + + Returns: + Dictionary with DocID and associated metadata + """ + docid = cls.generate_docid( + title=paper_data.get("title", "Untitled"), + doi=paper_data.get("doi"), + institution=paper_data.get("institution", "University of Lagos"), + timestamp=paper_data.get("publication_date"), + ) + + metadata = { + # Primary identifier + "document_docid": docid, + "docid_assigned_date": datetime.utcnow().isoformat(), + # Identifier type + "identifier_type": "APA Handle ID", + "identifier_scheme": "Handle", + # Alternate identifiers + "alternate_identifiers": [], + # Metadata + "title": paper_data.get("title"), + "institution": paper_data.get("institution", "University of Lagos"), + "institution_ror": "https://ror.org/05rk03822", # UNILAG ROR ID + # Provenance + "source_repository": paper_data.get( + "source_repository", "UNILAG Institutional Repository" + ), + "source_url": paper_data.get("url"), + # Handle resolution URL + "handle_url": f"https://hdl.handle.net/{docid}", + "docid_url": f"https://docid.africapidalliance.org/resolve/{docid}", + } + + # Add DOI as alternate identifier if present + if paper_data.get("doi"): + metadata["alternate_identifiers"].append( + {"type": "DOI", "value": paper_data["doi"], "is_primary": False} + ) + + # Add ORCID for authors if available + if paper_data.get("authors"): + for author in paper_data["authors"]: + if author.get("orcid"): + metadata["alternate_identifiers"].append( + {"type": "ORCID", "value": author["orcid"], "role": "Creator"} + ) + + return metadata + + @classmethod + def validate_docid(cls, docid: str) -> bool: + """ + Validate a DocID™ identifier format. + + Args: + docid: DocID string to validate + + Returns: + True if valid, False otherwise + """ + if not docid: + return False + + parts = docid.split("/") + if len(parts) != 2: + return False + + prefix, hash_part = parts + + # Check prefix matches APA Handle + if prefix != cls.APA_HANDLE_PREFIX: + return False + + # Check hash part is alphanumeric and reasonable length + if not hash_part or not hash_part.isalnum(): + return False + + if len(hash_part) < 10 or len(hash_part) > 64: + return False + + return True + + @classmethod + def format_citation_with_docid(cls, paper_data: Dict, docid: str) -> str: + """ + Format a citation including the DocID™ identifier. + + Args: + paper_data: Paper metadata + docid: DocID identifier + + Returns: + Formatted citation string + """ + authors = paper_data.get("authors", []) + author_str = ", ".join([a.get("name", "") for a in authors[:3]]) + if len(authors) > 3: + author_str += " et al." + + title = paper_data.get("title", "Untitled") + + # Handle both string and datetime objects for publication_date + pub_date = paper_data.get("publication_date") + if isinstance(pub_date, datetime): + year = pub_date.year + elif isinstance(pub_date, str): + year = pub_date.split("-")[0] if pub_date else "n.d." + else: + year = "n.d." + + citation = f"{author_str} ({year}). {title}. " + citation += f"University of Lagos Institutional Repository. " + citation += f"DocID: {docid}. " + citation += f"Available at: https://hdl.handle.net/{docid}" + + return citation + + +# Singleton instance +docid_generator = DocIDGenerator() diff --git a/uraas/utils/embedding_model.py b/uraas/utils/embedding_model.py index 5e5d0306b0a9d0fd937ae1f81df6512fb21bdeaa..e0c8095402b4b1d3993fcd017273b06f150c82a7 100644 --- a/uraas/utils/embedding_model.py +++ b/uraas/utils/embedding_model.py @@ -1,63 +1,63 @@ -""" -Lazy-loaded Model2Vec embedding singleton with graceful degradation. - -Uses minishlab/potion-base-8M (~30MB static embeddings, numpy-only) so the -semantic half of alignment scoring runs comfortably inside the Flask process -on a small Render instance. If the model can't load (no network on first -boot, missing dependency), the alignment engine silently falls back to -keyword-only scoring — the dashboard never 500s. -""" - -import logging -import os -import threading - -# Windows lacks symlink privileges for the HF cache by default; harmless on Linux. -os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS", "1") - -logger = logging.getLogger(__name__) - - -class EmbeddingModel: - """Thread-safe lazy singleton around model2vec.StaticModel.""" - - MODEL_NAME = os.getenv("URAAS_EMBED_MODEL", "minishlab/potion-base-8M") - - _model = None - _failed = False - _lock = threading.Lock() - - @classmethod - def get(cls): - """Return the StaticModel, or None if unavailable (keyword-only mode).""" - if cls._model is not None or cls._failed: - return cls._model - with cls._lock: - if cls._model is None and not cls._failed: - try: - from model2vec import StaticModel - - cls._model = StaticModel.from_pretrained(cls.MODEL_NAME) - logger.info("Embedding model loaded: %s", cls.MODEL_NAME) - except Exception as e: - logger.warning( - "Embedding model unavailable -> keyword-only alignment: %s", e - ) - cls._failed = True - return cls._model - - @classmethod - def encode(cls, texts): - """Encode a list of strings -> np.ndarray, or None when unavailable.""" - model = cls.get() - if model is None: - return None - try: - return model.encode(texts) - except Exception as e: - logger.warning("Embedding encode failed: %s", e) - return None - - @classmethod - def is_available(cls) -> bool: - return cls.get() is not None +""" +Lazy-loaded Model2Vec embedding singleton with graceful degradation. + +Uses minishlab/potion-base-8M (~30MB static embeddings, numpy-only) so the +semantic half of alignment scoring runs comfortably inside the Flask process +on a small Render instance. If the model can't load (no network on first +boot, missing dependency), the alignment engine silently falls back to +keyword-only scoring — the dashboard never 500s. +""" + +import logging +import os +import threading + +# Windows lacks symlink privileges for the HF cache by default; harmless on Linux. +os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS", "1") + +logger = logging.getLogger(__name__) + + +class EmbeddingModel: + """Thread-safe lazy singleton around model2vec.StaticModel.""" + + MODEL_NAME = os.getenv("URAAS_EMBED_MODEL", "minishlab/potion-base-8M") + + _model = None + _failed = False + _lock = threading.Lock() + + @classmethod + def get(cls): + """Return the StaticModel, or None if unavailable (keyword-only mode).""" + if cls._model is not None or cls._failed: + return cls._model + with cls._lock: + if cls._model is None and not cls._failed: + try: + from model2vec import StaticModel + + cls._model = StaticModel.from_pretrained(cls.MODEL_NAME) + logger.info("Embedding model loaded: %s", cls.MODEL_NAME) + except Exception as e: + logger.warning( + "Embedding model unavailable -> keyword-only alignment: %s", e + ) + cls._failed = True + return cls._model + + @classmethod + def encode(cls, texts): + """Encode a list of strings -> np.ndarray, or None when unavailable.""" + model = cls.get() + if model is None: + return None + try: + return model.encode(texts) + except Exception as e: + logger.warning("Embedding encode failed: %s", e) + return None + + @classmethod + def is_available(cls) -> bool: + return cls.get() is not None diff --git a/uraas/utils/geo.py b/uraas/utils/geo.py index 29b4b39326bb284e7e63d9fca6c1b14413c05952..051d93d069b0bdea971b99fc4ddcf06bd5ca1938 100644 --- a/uraas/utils/geo.py +++ b/uraas/utils/geo.py @@ -1,52 +1,52 @@ -""" -Lightweight geo helpers — great-circle arc generation for the collaboration -map. Plain spherical interpolation (slerp); no shapely/pyproj dependency. -""" - -import math -from typing import List - - -def great_circle_arc( - lat1: float, lon1: float, lat2: float, lon2: float, n_points: int = 32 -) -> List[List[float]]: - """Interpolate a great-circle path between two points. - - Returns [[lon, lat], ...] (GeoJSON coordinate order) with n_points - vertices inclusive of both endpoints.""" - phi1, lam1 = math.radians(lat1), math.radians(lon1) - phi2, lam2 = math.radians(lat2), math.radians(lon2) - - # Cartesian unit vectors - a = ( - math.cos(phi1) * math.cos(lam1), - math.cos(phi1) * math.sin(lam1), - math.sin(phi1), - ) - b = ( - math.cos(phi2) * math.cos(lam2), - math.cos(phi2) * math.sin(lam2), - math.sin(phi2), - ) - - dot = max(-1.0, min(1.0, a[0] * b[0] + a[1] * b[1] + a[2] * b[2])) - omega = math.acos(dot) - if omega < 1e-9: # coincident points - return [[lon1, lat1], [lon2, lat2]] - - sin_omega = math.sin(omega) - coords = [] - for i in range(n_points): - t = i / (n_points - 1) - s1 = math.sin((1 - t) * omega) / sin_omega - s2 = math.sin(t * omega) / sin_omega - x = s1 * a[0] + s2 * b[0] - y = s1 * a[1] + s2 * b[1] - z = s1 * a[2] + s2 * b[2] - coords.append( - [ - round(math.degrees(math.atan2(y, x)), 4), - round(math.degrees(math.atan2(z, math.hypot(x, y))), 4), - ] - ) - return coords +""" +Lightweight geo helpers — great-circle arc generation for the collaboration +map. Plain spherical interpolation (slerp); no shapely/pyproj dependency. +""" + +import math +from typing import List + + +def great_circle_arc( + lat1: float, lon1: float, lat2: float, lon2: float, n_points: int = 32 +) -> List[List[float]]: + """Interpolate a great-circle path between two points. + + Returns [[lon, lat], ...] (GeoJSON coordinate order) with n_points + vertices inclusive of both endpoints.""" + phi1, lam1 = math.radians(lat1), math.radians(lon1) + phi2, lam2 = math.radians(lat2), math.radians(lon2) + + # Cartesian unit vectors + a = ( + math.cos(phi1) * math.cos(lam1), + math.cos(phi1) * math.sin(lam1), + math.sin(phi1), + ) + b = ( + math.cos(phi2) * math.cos(lam2), + math.cos(phi2) * math.sin(lam2), + math.sin(phi2), + ) + + dot = max(-1.0, min(1.0, a[0] * b[0] + a[1] * b[1] + a[2] * b[2])) + omega = math.acos(dot) + if omega < 1e-9: # coincident points + return [[lon1, lat1], [lon2, lat2]] + + sin_omega = math.sin(omega) + coords = [] + for i in range(n_points): + t = i / (n_points - 1) + s1 = math.sin((1 - t) * omega) / sin_omega + s2 = math.sin(t * omega) / sin_omega + x = s1 * a[0] + s2 * b[0] + y = s1 * a[1] + s2 * b[1] + z = s1 * a[2] + s2 * b[2] + coords.append( + [ + round(math.degrees(math.atan2(y, x)), 4), + round(math.degrees(math.atan2(z, math.hypot(x, y))), 4), + ] + ) + return coords diff --git a/uraas/utils/normalizer.py b/uraas/utils/normalizer.py index af2b71b7b93a656c9fdcbb2d7abd1c459f0a01fc..648e1883d1fb4bf9be72a63583c040b8ab715c46 100644 --- a/uraas/utils/normalizer.py +++ b/uraas/utils/normalizer.py @@ -1,17 +1,17 @@ -"""Title normalization utilities for gap analysis deduplication.""" - -import re - - -def normalize_title(title: str) -> str: - """ - Lowercase, strip punctuation, collapse whitespace. - Used so 'Climate Change In Lagos.' and 'climate change in lagos' - score ≥95% similarity under Levenshtein distance. - """ - if not title: - return "" - t = title.lower() - t = re.sub(r"[^\w\s]", " ", t) # strip punctuation - t = re.sub(r"\s+", " ", t).strip() - return t +"""Title normalization utilities for gap analysis deduplication.""" + +import re + + +def normalize_title(title: str) -> str: + """ + Lowercase, strip punctuation, collapse whitespace. + Used so 'Climate Change In Lagos.' and 'climate change in lagos' + score ≥95% similarity under Levenshtein distance. + """ + if not title: + return "" + t = title.lower() + t = re.sub(r"[^\w\s]", " ", t) # strip punctuation + t = re.sub(r"\s+", " ", t).strip() + return t diff --git a/uraas/utils/openalex_client.py b/uraas/utils/openalex_client.py index 5dacbe6ba9de558e5cefab57a3a1566bd1fd0994..f3f21e248bc9236eb3e0e1d23c7d3a53023122f8 100644 --- a/uraas/utils/openalex_client.py +++ b/uraas/utils/openalex_client.py @@ -1,45 +1,45 @@ -""" -Shared OpenAlex HTTP helper — single place for the api_key / mailto params. - -Used by the citation tracker and the backfill scripts (the Scrapy spider -builds its own URLs but reads the same Config values). -""" - -import logging -from typing import Optional - -import requests - -from uraas.config import config - -OPENALEX_API = "https://api.openalex.org" - -logger = logging.getLogger(__name__) - - -def oa_params(extra: Optional[dict] = None) -> dict: - """Base query params: polite mailto + api_key when configured.""" - params = {"mailto": config.OPENALEX_MAILTO} - if config.OPENALEX_API_KEY: - params["api_key"] = config.OPENALEX_API_KEY - if extra: - params.update(extra) - return params - - -def oa_get(path: str, params: Optional[dict] = None, timeout: int = 30) -> Optional[dict]: - """GET an OpenAlex endpoint ('/works', '/works/W123', or a full URL). - - Returns parsed JSON, or None on any failure (callers treat missing data - as 'not yet enriched', never as fatal).""" - url = path if path.startswith("http") else f"{OPENALEX_API}{path}" - try: - resp = requests.get(url, params=oa_params(params), timeout=timeout) - if resp.status_code == 429: - logger.warning("OpenAlex rate limited (429): %s", url) - return None - resp.raise_for_status() - return resp.json() - except Exception as e: - logger.warning("OpenAlex request failed (%s): %s", url, e) - return None +""" +Shared OpenAlex HTTP helper — single place for the api_key / mailto params. + +Used by the citation tracker and the backfill scripts (the Scrapy spider +builds its own URLs but reads the same Config values). +""" + +import logging +from typing import Optional + +import requests + +from uraas.config import config + +OPENALEX_API = "https://api.openalex.org" + +logger = logging.getLogger(__name__) + + +def oa_params(extra: Optional[dict] = None) -> dict: + """Base query params: polite mailto + api_key when configured.""" + params = {"mailto": config.OPENALEX_MAILTO} + if config.OPENALEX_API_KEY: + params["api_key"] = config.OPENALEX_API_KEY + if extra: + params.update(extra) + return params + + +def oa_get(path: str, params: Optional[dict] = None, timeout: int = 30) -> Optional[dict]: + """GET an OpenAlex endpoint ('/works', '/works/W123', or a full URL). + + Returns parsed JSON, or None on any failure (callers treat missing data + as 'not yet enriched', never as fatal).""" + url = path if path.startswith("http") else f"{OPENALEX_API}{path}" + try: + resp = requests.get(url, params=oa_params(params), timeout=timeout) + if resp.status_code == 429: + logger.warning("OpenAlex rate limited (429): %s", url) + return None + resp.raise_for_status() + return resp.json() + except Exception as e: + logger.warning("OpenAlex request failed (%s): %s", url, e) + return None diff --git a/uraas/utils/pdf_downloader.py b/uraas/utils/pdf_downloader.py index 2e1dd39d0532c9d8db0ceb5f45c6e70d5628efa0..fdc27dfc1833f3b6858c86863c61ba76c6bbd600 100644 --- a/uraas/utils/pdf_downloader.py +++ b/uraas/utils/pdf_downloader.py @@ -1,126 +1,126 @@ -""" -PDF Downloader - Downloads and stores PDFs locally with metadata extraction. -Tries direct URL first, then falls back to Unpaywall open-access copy. -""" - -import hashlib -import os -from datetime import datetime -from io import BytesIO -from typing import Dict, Optional - -import PyPDF2 -import requests - -UNPAYWALL_EMAIL = "library@unilag.edu.ng" - - -class PDFDownloader: - """Handles PDF downloading, storage, and basic metadata extraction.""" - - def __init__(self, storage_path: str = "./storage/pdfs"): - # Convert to absolute path from project root - if not os.path.isabs(storage_path): - # Get project root (3 levels up from this file) - project_root = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - ) - storage_path = os.path.normpath(os.path.join(project_root, storage_path)) - - self.storage_path = storage_path - os.makedirs(storage_path, exist_ok=True) - - def _fetch_pdf_bytes(self, url: str, timeout: int) -> Optional[bytes]: - """Attempt to download PDF bytes from a URL.""" - headers = { - "User-Agent": "Mozilla/5.0 (compatible; URAAS/1.0; mailto:library@unilag.edu.ng)" - } - try: - resp = requests.get( - url, headers=headers, timeout=timeout, stream=True, allow_redirects=True - ) - resp.raise_for_status() - content_type = resp.headers.get("Content-Type", "") - content = resp.content - # Accept if content-type says PDF or file starts with PDF magic bytes - if "pdf" in content_type.lower() or content[:4] == b"%PDF": - return content - except Exception: - pass - return None - - def _unpaywall_url(self, doi: str) -> Optional[str]: - """Look up open-access PDF URL via Unpaywall API.""" - if not doi: - return None - # Strip prefix if present - clean_doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "") - try: - resp = requests.get( - f"https://api.unpaywall.org/v2/{clean_doi}", - params={"email": UNPAYWALL_EMAIL}, - timeout=10, - ) - if resp.status_code == 200: - data = resp.json() - best = data.get("best_oa_location") or {} - return best.get("url_for_pdf") or best.get("url") - except Exception: - pass - return None - - def download_pdf( - self, url: str, item_id: int, doi: str = None, timeout: int = 30 - ) -> Optional[Dict]: - """ - Download PDF from URL, falling back to Unpaywall if blocked. - Returns dict with file_path, sha256_hash, file_size, page_count or None. - """ - pdf_content = None - - # 1. Try the direct URL - if url: - pdf_content = self._fetch_pdf_bytes(url, timeout) - - # 2. Fallback: Unpaywall open-access copy - if pdf_content is None and doi: - oa_url = self._unpaywall_url(doi) - if oa_url and oa_url != url: - pdf_content = self._fetch_pdf_bytes(oa_url, timeout) - - if pdf_content is None: - return None - - sha256_hash = hashlib.sha256(pdf_content).hexdigest() - filename = f"{item_id}_{sha256_hash[:8]}.pdf" - file_path = os.path.join(self.storage_path, filename) - - with open(file_path, "wb") as f: - f.write(pdf_content) - - return { - "file_path": file_path, - "sha256_hash": sha256_hash, - "file_size": len(pdf_content), - "page_count": self._get_page_count(pdf_content), - "downloaded_at": datetime.utcnow(), - } - - def _get_page_count(self, pdf_content: bytes) -> Optional[int]: - try: - return len(PyPDF2.PdfReader(BytesIO(pdf_content)).pages) - except Exception: - return None - - def extract_first_page_text(self, file_path: str) -> Optional[str]: - try: - with open(file_path, "rb") as f: - reader = PyPDF2.PdfReader(f) - if reader.pages: - return reader.pages[0].extract_text() - except Exception: - pass - return None - - -pdf_downloader = PDFDownloader() +""" +PDF Downloader - Downloads and stores PDFs locally with metadata extraction. +Tries direct URL first, then falls back to Unpaywall open-access copy. +""" + +import hashlib +import os +from datetime import datetime +from io import BytesIO +from typing import Dict, Optional + +import PyPDF2 +import requests + +UNPAYWALL_EMAIL = "library@unilag.edu.ng" + + +class PDFDownloader: + """Handles PDF downloading, storage, and basic metadata extraction.""" + + def __init__(self, storage_path: str = "./storage/pdfs"): + # Convert to absolute path from project root + if not os.path.isabs(storage_path): + # Get project root (3 levels up from this file) + project_root = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ) + storage_path = os.path.normpath(os.path.join(project_root, storage_path)) + + self.storage_path = storage_path + os.makedirs(storage_path, exist_ok=True) + + def _fetch_pdf_bytes(self, url: str, timeout: int) -> Optional[bytes]: + """Attempt to download PDF bytes from a URL.""" + headers = { + "User-Agent": "Mozilla/5.0 (compatible; URAAS/1.0; mailto:library@unilag.edu.ng)" + } + try: + resp = requests.get( + url, headers=headers, timeout=timeout, stream=True, allow_redirects=True + ) + resp.raise_for_status() + content_type = resp.headers.get("Content-Type", "") + content = resp.content + # Accept if content-type says PDF or file starts with PDF magic bytes + if "pdf" in content_type.lower() or content[:4] == b"%PDF": + return content + except Exception: + pass + return None + + def _unpaywall_url(self, doi: str) -> Optional[str]: + """Look up open-access PDF URL via Unpaywall API.""" + if not doi: + return None + # Strip prefix if present + clean_doi = doi.replace("https://doi.org/", "").replace("http://doi.org/", "") + try: + resp = requests.get( + f"https://api.unpaywall.org/v2/{clean_doi}", + params={"email": UNPAYWALL_EMAIL}, + timeout=10, + ) + if resp.status_code == 200: + data = resp.json() + best = data.get("best_oa_location") or {} + return best.get("url_for_pdf") or best.get("url") + except Exception: + pass + return None + + def download_pdf( + self, url: str, item_id: int, doi: str = None, timeout: int = 30 + ) -> Optional[Dict]: + """ + Download PDF from URL, falling back to Unpaywall if blocked. + Returns dict with file_path, sha256_hash, file_size, page_count or None. + """ + pdf_content = None + + # 1. Try the direct URL + if url: + pdf_content = self._fetch_pdf_bytes(url, timeout) + + # 2. Fallback: Unpaywall open-access copy + if pdf_content is None and doi: + oa_url = self._unpaywall_url(doi) + if oa_url and oa_url != url: + pdf_content = self._fetch_pdf_bytes(oa_url, timeout) + + if pdf_content is None: + return None + + sha256_hash = hashlib.sha256(pdf_content).hexdigest() + filename = f"{item_id}_{sha256_hash[:8]}.pdf" + file_path = os.path.join(self.storage_path, filename) + + with open(file_path, "wb") as f: + f.write(pdf_content) + + return { + "file_path": file_path, + "sha256_hash": sha256_hash, + "file_size": len(pdf_content), + "page_count": self._get_page_count(pdf_content), + "downloaded_at": datetime.utcnow(), + } + + def _get_page_count(self, pdf_content: bytes) -> Optional[int]: + try: + return len(PyPDF2.PdfReader(BytesIO(pdf_content)).pages) + except Exception: + return None + + def extract_first_page_text(self, file_path: str) -> Optional[str]: + try: + with open(file_path, "rb") as f: + reader = PyPDF2.PdfReader(f) + if reader.pages: + return reader.pages[0].extract_text() + except Exception: + pass + return None + + +pdf_downloader = PDFDownloader() diff --git a/uraas/utils/staff_validator.py b/uraas/utils/staff_validator.py index 89ca370564145581b60827f944cd2c0a684c3502..56545286a57687306137c23ae8dfdbf21b960e22 100644 --- a/uraas/utils/staff_validator.py +++ b/uraas/utils/staff_validator.py @@ -1,318 +1,318 @@ -""" -Staff Validator - Validates authors against institution staff lists. -Supports multi-institution with ROR-based configuration. -""" - -import json -import logging -import os -import re -from typing import Dict, List, Optional, Set, Tuple - -from thefuzz import fuzz - -logger = logging.getLogger(__name__) - - -class StaffValidator: - """ - Validates authors against institution staff lists. - Now supports multiple institutions via InstitutionConfig. - """ - - def __init__(self, institution_config=None, staff_cache_path: str = None): - """ - Initialize validator with institution configuration. - - Args: - institution_config: InstitutionConfig object (new multi-institution support) - staff_cache_path: Legacy path to staff JSON (for backward compatibility) - """ - self.institution_config = institution_config - - # Determine staff file path - if institution_config: - self.staff_cache_path = institution_config.staff_file - self.institution_name = institution_config.name - self.ror = institution_config.ror - elif staff_cache_path: - self.staff_cache_path = staff_cache_path - self.institution_name = "Unknown" - self.ror = None - else: - # Default to UNILAG for backward compatibility - self.staff_cache_path = os.path.join( - os.path.dirname(__file__), "..", "..", "data", "unilag_staff.json" - ) - self.institution_name = "University of Lagos" - self.ror = "https://ror.org/05rk03822" - - self.staff_names: Set[str] = set() - self.normalized_staff: Set[str] = set() - self._faculty_map: Dict[str, List[str]] = {} - self._surname_to_faculty: Dict[str, str] = {} - self._surname_to_dept: Dict[str, str] = {} - self._fullname_to_faculty: Dict[str, str] = {} - self._fullname_to_dept: Dict[str, str] = {} - self._detailed_records: List = [] - - self.load_staff_cache() - self._load_faculty_map() - - def load_staff_cache(self): - """Load staff names from JSON file""" - if not os.path.exists(self.staff_cache_path): - logger.warning(f"Staff file not found: {self.staff_cache_path}") - return - - try: - with open(self.staff_cache_path, "r", encoding="utf-8") as f: - staff_data = json.load(f) - - # Handle different JSON structures - if isinstance(staff_data, list): - staff_list = staff_data - elif isinstance(staff_data, dict): - # Extract names from various possible structures - if "staff" in staff_data: - staff_list = staff_data["staff"] - elif "names" in staff_data: - staff_list = staff_data["names"] - else: - # Flatten all values that are lists - staff_list = [] - for value in staff_data.values(): - if isinstance(value, list): - staff_list.extend(value) - else: - staff_list = [] - - for entry in staff_list: - # Each entry may be a plain string or a dict with a 'name' key - if isinstance(entry, dict): - name = entry.get("name", "") - elif isinstance(entry, str): - name = entry - else: - continue - cleaned = self._clean_name(name) - if cleaned: - self.staff_names.add(cleaned) - self.normalized_staff.add(self._normalize_name(cleaned)) - - logger.info( - f"Loaded {len(self.staff_names)} staff members for {self.institution_name}" - ) - - except Exception as e: - logger.error(f"Error loading staff cache from {self.staff_cache_path}: {e}") - - def _load_faculty_map(self): - """Load faculty and department mappings""" - # Try to load detailed staff records (name → faculty/dept) - base_name = os.path.splitext(os.path.basename(self.staff_cache_path))[0] - detailed_path = os.path.join( - os.path.dirname(self.staff_cache_path), f"{base_name}_detailed.json" - ) - - if os.path.exists(detailed_path): - try: - with open(detailed_path, "r", encoding="utf-8") as f: - records = json.load(f) - self._detailed_records = records - self._surname_to_faculty: Dict[str, str] = {} - self._surname_to_dept: Dict[str, str] = {} - self._fullname_to_faculty: Dict[str, str] = {} - self._fullname_to_dept: Dict[str, str] = {} - - for r in records: - name = r.get("name", "") - faculty = r.get("faculty", "Unknown") - dept = r.get("department", "Unknown") - if not name or faculty == "Unknown": - continue - - # Store full normalized name lookup (most accurate) - norm = self._normalize_name(self._clean_name(name)) - if norm: - self._fullname_to_faculty[norm] = faculty - if dept != "Unknown": - self._fullname_to_dept[norm] = dept - - # Store surname (last word) lookup as fallback - parts = name.strip().split() - if parts: - surname = re.sub(r"[^\w]", "", parts[-1].lower()) - if len(surname) >= 4: - # Only set if not already set (first match wins) - if surname not in self._surname_to_faculty: - self._surname_to_faculty[surname] = faculty - if ( - dept != "Unknown" - and surname not in self._surname_to_dept - ): - self._surname_to_dept[surname] = dept - return - except Exception as e: - logger.warning(f"Could not load detailed staff records: {e}") - - # Fallback: keyword map (UNILAG-specific, may not exist for other institutions) - self._detailed_records = [] - self._surname_to_faculty = {} - self._surname_to_dept = {} - self._fullname_to_faculty = {} - self._fullname_to_dept = {} - - map_path = os.path.join( - os.path.dirname(self.staff_cache_path), "staff_department_map.json" - ) - if os.path.exists(map_path): - try: - with open(map_path, "r", encoding="utf-8") as f: - raw = json.load(f) - self._faculty_map = { - faculty: data.get("keywords", []) for faculty, data in raw.items() - } - except Exception: - pass - - def _clean_name(self, name: str) -> str: - """Remove titles and degrees from name""" - name = re.sub( - r"\b(Prof\.?|Dr\.?|Mr\.?|Mrs\.?|Miss\.?|Ms\.?|Engr\.?|Pharm\.?|Assoc\.?|Associate)\s*", - "", - name, - flags=re.IGNORECASE, - ) - name = re.sub( - r"\b(Ph\.?D\.?|M\.?Sc\.?|B\.?Sc\.?|M\.?B\.?,?\s*B\.?S\.?|M\.?Phil\.?)\s*", - "", - name, - flags=re.IGNORECASE, - ) - name = re.sub(r"\(Mrs\.?\)|(\(Mr\.?\))", "", name) - name = re.sub(r"[,\.]", "", name) - name = re.sub(r"\s+", " ", name).strip() - return name - - def _normalize_name(self, name: str) -> str: - """Normalize name for comparison""" - name = name.lower() - name = re.sub(r"[^\w\s]", "", name) - name = re.sub(r"\s+", " ", name).strip() - return name - - def is_staff_member(self, author_name: str, fuzzy_threshold: int = 75) -> bool: - """Check if author is a staff member of this institution""" - if not author_name: - return False - - # If no staff directory loaded, bypass and rely on ROR and affiliation gates - if not self.normalized_staff: - return True - - cleaned = self._clean_name(author_name) - normalized = self._normalize_name(cleaned) - - # Exact match - if normalized in self.normalized_staff: - return True - - # Fuzzy match - for staff_name in self.normalized_staff: - if fuzz.ratio(normalized, staff_name) >= fuzzy_threshold: - return True - - return False - - def get_faculty_hint(self, author_name: str) -> Optional[str]: - """Return most likely faculty — checks full name first, then surname.""" - if not author_name: - return None - - # 1. Full normalized name lookup (most accurate) - norm = self._normalize_name(self._clean_name(author_name)) - if norm in self._fullname_to_faculty: - return self._fullname_to_faculty[norm] - - # 2. Fuzzy match against full name lookup - from thefuzz import process - - if self._fullname_to_faculty: - match = process.extractOne( - norm, self._fullname_to_faculty.keys(), score_cutoff=80 - ) - if match: - return self._fullname_to_faculty[match[0]] - - # 3. Surname fallback - parts = author_name.strip().split() - if parts: - surname = re.sub(r"[^\w]", "", parts[-1].lower()) - if surname in self._surname_to_faculty: - return self._surname_to_faculty[surname] - - # 4. Keyword map fallback - for part in author_name.lower().split(): - part = re.sub(r"[^\w]", "", part) - if len(part) < 4: - continue - for faculty, keywords in self._faculty_map.items(): - if any(part in kw or kw in part for kw in keywords): - return faculty - - return None - - def get_department_hint(self, author_name: str) -> Optional[str]: - """Return most likely department.""" - if not author_name: - return None - - norm = self._normalize_name(self._clean_name(author_name)) - if norm in self._fullname_to_dept: - return self._fullname_to_dept[norm] - - from thefuzz import process - - if self._fullname_to_dept: - match = process.extractOne( - norm, self._fullname_to_dept.keys(), score_cutoff=80 - ) - if match: - return self._fullname_to_dept[match[0]] - - parts = author_name.strip().split() - if parts: - surname = re.sub(r"[^\w]", "", parts[-1].lower()) - if surname in self._surname_to_dept: - return self._surname_to_dept[surname] - - return None - - def get_all_faculty_hints(self, authors: List[str]) -> List[Tuple[str, str]]: - """ - Returns (author, faculty) pairs for all confirmed staff authors. - Handles multiple staff authors on the same paper. - """ - results = [] - for author in authors: - if self.is_staff_member(author, fuzzy_threshold=75): - hint = self.get_faculty_hint(author) - if hint: - results.append((author, hint)) - return results - - def validate_authors(self, authors: List[str], require_all: bool = False) -> bool: - """Validate if authors include staff members""" - if not authors: - return False - matches = [self.is_staff_member(a) for a in authors] - return all(matches) if require_all else any(matches) - - def get_matching_staff(self, authors: List[str]) -> List[str]: - """Get list of authors who are staff members""" - return [a for a in authors if self.is_staff_member(a)] - - -# Global instance (backward compatibility - defaults to UNILAG) -staff_validator = StaffValidator() +""" +Staff Validator - Validates authors against institution staff lists. +Supports multi-institution with ROR-based configuration. +""" + +import json +import logging +import os +import re +from typing import Dict, List, Optional, Set, Tuple + +from thefuzz import fuzz + +logger = logging.getLogger(__name__) + + +class StaffValidator: + """ + Validates authors against institution staff lists. + Now supports multiple institutions via InstitutionConfig. + """ + + def __init__(self, institution_config=None, staff_cache_path: str = None): + """ + Initialize validator with institution configuration. + + Args: + institution_config: InstitutionConfig object (new multi-institution support) + staff_cache_path: Legacy path to staff JSON (for backward compatibility) + """ + self.institution_config = institution_config + + # Determine staff file path + if institution_config: + self.staff_cache_path = institution_config.staff_file + self.institution_name = institution_config.name + self.ror = institution_config.ror + elif staff_cache_path: + self.staff_cache_path = staff_cache_path + self.institution_name = "Unknown" + self.ror = None + else: + # Default to UNILAG for backward compatibility + self.staff_cache_path = os.path.join( + os.path.dirname(__file__), "..", "..", "data", "unilag_staff.json" + ) + self.institution_name = "University of Lagos" + self.ror = "https://ror.org/05rk03822" + + self.staff_names: Set[str] = set() + self.normalized_staff: Set[str] = set() + self._faculty_map: Dict[str, List[str]] = {} + self._surname_to_faculty: Dict[str, str] = {} + self._surname_to_dept: Dict[str, str] = {} + self._fullname_to_faculty: Dict[str, str] = {} + self._fullname_to_dept: Dict[str, str] = {} + self._detailed_records: List = [] + + self.load_staff_cache() + self._load_faculty_map() + + def load_staff_cache(self): + """Load staff names from JSON file""" + if not os.path.exists(self.staff_cache_path): + logger.warning(f"Staff file not found: {self.staff_cache_path}") + return + + try: + with open(self.staff_cache_path, "r", encoding="utf-8") as f: + staff_data = json.load(f) + + # Handle different JSON structures + if isinstance(staff_data, list): + staff_list = staff_data + elif isinstance(staff_data, dict): + # Extract names from various possible structures + if "staff" in staff_data: + staff_list = staff_data["staff"] + elif "names" in staff_data: + staff_list = staff_data["names"] + else: + # Flatten all values that are lists + staff_list = [] + for value in staff_data.values(): + if isinstance(value, list): + staff_list.extend(value) + else: + staff_list = [] + + for entry in staff_list: + # Each entry may be a plain string or a dict with a 'name' key + if isinstance(entry, dict): + name = entry.get("name", "") + elif isinstance(entry, str): + name = entry + else: + continue + cleaned = self._clean_name(name) + if cleaned: + self.staff_names.add(cleaned) + self.normalized_staff.add(self._normalize_name(cleaned)) + + logger.info( + f"Loaded {len(self.staff_names)} staff members for {self.institution_name}" + ) + + except Exception as e: + logger.error(f"Error loading staff cache from {self.staff_cache_path}: {e}") + + def _load_faculty_map(self): + """Load faculty and department mappings""" + # Try to load detailed staff records (name → faculty/dept) + base_name = os.path.splitext(os.path.basename(self.staff_cache_path))[0] + detailed_path = os.path.join( + os.path.dirname(self.staff_cache_path), f"{base_name}_detailed.json" + ) + + if os.path.exists(detailed_path): + try: + with open(detailed_path, "r", encoding="utf-8") as f: + records = json.load(f) + self._detailed_records = records + self._surname_to_faculty: Dict[str, str] = {} + self._surname_to_dept: Dict[str, str] = {} + self._fullname_to_faculty: Dict[str, str] = {} + self._fullname_to_dept: Dict[str, str] = {} + + for r in records: + name = r.get("name", "") + faculty = r.get("faculty", "Unknown") + dept = r.get("department", "Unknown") + if not name or faculty == "Unknown": + continue + + # Store full normalized name lookup (most accurate) + norm = self._normalize_name(self._clean_name(name)) + if norm: + self._fullname_to_faculty[norm] = faculty + if dept != "Unknown": + self._fullname_to_dept[norm] = dept + + # Store surname (last word) lookup as fallback + parts = name.strip().split() + if parts: + surname = re.sub(r"[^\w]", "", parts[-1].lower()) + if len(surname) >= 4: + # Only set if not already set (first match wins) + if surname not in self._surname_to_faculty: + self._surname_to_faculty[surname] = faculty + if ( + dept != "Unknown" + and surname not in self._surname_to_dept + ): + self._surname_to_dept[surname] = dept + return + except Exception as e: + logger.warning(f"Could not load detailed staff records: {e}") + + # Fallback: keyword map (UNILAG-specific, may not exist for other institutions) + self._detailed_records = [] + self._surname_to_faculty = {} + self._surname_to_dept = {} + self._fullname_to_faculty = {} + self._fullname_to_dept = {} + + map_path = os.path.join( + os.path.dirname(self.staff_cache_path), "staff_department_map.json" + ) + if os.path.exists(map_path): + try: + with open(map_path, "r", encoding="utf-8") as f: + raw = json.load(f) + self._faculty_map = { + faculty: data.get("keywords", []) for faculty, data in raw.items() + } + except Exception: + pass + + def _clean_name(self, name: str) -> str: + """Remove titles and degrees from name""" + name = re.sub( + r"\b(Prof\.?|Dr\.?|Mr\.?|Mrs\.?|Miss\.?|Ms\.?|Engr\.?|Pharm\.?|Assoc\.?|Associate)\s*", + "", + name, + flags=re.IGNORECASE, + ) + name = re.sub( + r"\b(Ph\.?D\.?|M\.?Sc\.?|B\.?Sc\.?|M\.?B\.?,?\s*B\.?S\.?|M\.?Phil\.?)\s*", + "", + name, + flags=re.IGNORECASE, + ) + name = re.sub(r"\(Mrs\.?\)|(\(Mr\.?\))", "", name) + name = re.sub(r"[,\.]", "", name) + name = re.sub(r"\s+", " ", name).strip() + return name + + def _normalize_name(self, name: str) -> str: + """Normalize name for comparison""" + name = name.lower() + name = re.sub(r"[^\w\s]", "", name) + name = re.sub(r"\s+", " ", name).strip() + return name + + def is_staff_member(self, author_name: str, fuzzy_threshold: int = 75) -> bool: + """Check if author is a staff member of this institution""" + if not author_name: + return False + + # If no staff directory loaded, bypass and rely on ROR and affiliation gates + if not self.normalized_staff: + return True + + cleaned = self._clean_name(author_name) + normalized = self._normalize_name(cleaned) + + # Exact match + if normalized in self.normalized_staff: + return True + + # Fuzzy match + for staff_name in self.normalized_staff: + if fuzz.ratio(normalized, staff_name) >= fuzzy_threshold: + return True + + return False + + def get_faculty_hint(self, author_name: str) -> Optional[str]: + """Return most likely faculty — checks full name first, then surname.""" + if not author_name: + return None + + # 1. Full normalized name lookup (most accurate) + norm = self._normalize_name(self._clean_name(author_name)) + if norm in self._fullname_to_faculty: + return self._fullname_to_faculty[norm] + + # 2. Fuzzy match against full name lookup + from thefuzz import process + + if self._fullname_to_faculty: + match = process.extractOne( + norm, self._fullname_to_faculty.keys(), score_cutoff=80 + ) + if match: + return self._fullname_to_faculty[match[0]] + + # 3. Surname fallback + parts = author_name.strip().split() + if parts: + surname = re.sub(r"[^\w]", "", parts[-1].lower()) + if surname in self._surname_to_faculty: + return self._surname_to_faculty[surname] + + # 4. Keyword map fallback + for part in author_name.lower().split(): + part = re.sub(r"[^\w]", "", part) + if len(part) < 4: + continue + for faculty, keywords in self._faculty_map.items(): + if any(part in kw or kw in part for kw in keywords): + return faculty + + return None + + def get_department_hint(self, author_name: str) -> Optional[str]: + """Return most likely department.""" + if not author_name: + return None + + norm = self._normalize_name(self._clean_name(author_name)) + if norm in self._fullname_to_dept: + return self._fullname_to_dept[norm] + + from thefuzz import process + + if self._fullname_to_dept: + match = process.extractOne( + norm, self._fullname_to_dept.keys(), score_cutoff=80 + ) + if match: + return self._fullname_to_dept[match[0]] + + parts = author_name.strip().split() + if parts: + surname = re.sub(r"[^\w]", "", parts[-1].lower()) + if surname in self._surname_to_dept: + return self._surname_to_dept[surname] + + return None + + def get_all_faculty_hints(self, authors: List[str]) -> List[Tuple[str, str]]: + """ + Returns (author, faculty) pairs for all confirmed staff authors. + Handles multiple staff authors on the same paper. + """ + results = [] + for author in authors: + if self.is_staff_member(author, fuzzy_threshold=75): + hint = self.get_faculty_hint(author) + if hint: + results.append((author, hint)) + return results + + def validate_authors(self, authors: List[str], require_all: bool = False) -> bool: + """Validate if authors include staff members""" + if not authors: + return False + matches = [self.is_staff_member(a) for a in authors] + return all(matches) if require_all else any(matches) + + def get_matching_staff(self, authors: List[str]) -> List[str]: + """Get list of authors who are staff members""" + return [a for a in authors if self.is_staff_member(a)] + + +# Global instance (backward compatibility - defaults to UNILAG) +staff_validator = StaffValidator() diff --git a/uraas/utils/unilag_classifier.py b/uraas/utils/unilag_classifier.py index 48fa526162d75590555c313348b660c343361d96..4c9cb46177a65169210815254f13c0dc65ac48a0 100644 --- a/uraas/utils/unilag_classifier.py +++ b/uraas/utils/unilag_classifier.py @@ -1,1504 +1,1504 @@ -""" -URAAS UNILAG Academic Classifier — Production Grade -===================================================== -Features: - - TF-IDF–style weighted keyword scoring (rare/specific keywords score higher) - - Multi-word phrase detection with word boundary matching - - Inverse-document-frequency (IDF) weighting across departments - - SDG (UN Sustainable Development Goals 1–17) alignment detection - - classify_with_explanation() for transparency and debugging - - Full 2024 UNILAG Faculty & Department structure -""" - -import math -import re -from typing import Any, Dict, List, Optional, Tuple - -# ─── Complete UNILAG Faculty and Department Structure (2024) ─────────────────── -UNILAG_STRUCTURE = { - "Faculty of Arts": { - "Creative Arts": [ - "creative arts", - "theatre", - "drama", - "performance", - "visual arts", - "music", - "dance", - "cinematography", - "costume design", - "stage design", - "african theatre", - "yoruba drama", - "nollywood", - "film studies", - ], - "English": [ - "english literature", - "linguistics", - "language studies", - "literary criticism", - "phonetics", - "morphology", - "syntax", - "pragmatics", - "discourse analysis", - "stylistics", - "narrative theory", - "african literature in english", - "postcolonial literature", - ], - "History and Strategic Studies": [ - "history", - "strategic studies", - "military", - "warfare", - "historical analysis", - "colonialism", - "decolonization", - "lagos history", - "nigerian history", - "precolonial", - "empire", - "slave trade", - "nationalism", - ], - "Philosophy": [ - "philosophy", - "logic", - "ethics", - "metaphysics", - "epistemology", - "african philosophy", - "political philosophy", - "philosophy of mind", - "existentialism", - "ontology", - "moral philosophy", - ], - "Languages": [ - "french", - "german", - "russian", - "arabic", - "foreign language", - "translation", - "language pedagogy", - "yoruba", - "igbo", - "hausa", - "pidgin", - "nigerian languages", - "oral tradition", - "indigenous language", - "language policy", - "multilingualism", - "sociolinguistics", - ], - }, - "Faculty of Science": { - "Biochemistry": [ - "biochemistry", - "molecular biology", - "enzymology", - "metabolism", - "protein", - "lipid", - "carbohydrate", - "enzyme kinetics", - "dna replication", - "gene expression", - "proteomics", - "metabolomics", - "oxidative stress", - ], - "Botany": [ - "botany", - "plant biology", - "plant physiology", - "taxonomy", - "flora", - "phytology", - "seed germination", - "photosynthesis", - "ethnobotany", - "mangrove", - "rainforest ecology", - "medicinal plants", - ], - "Cell Biology and Genetics": [ - "cell biology", - "genetics", - "cytology", - "heredity", - "dna", - "gene expression", - "chromosome", - "genetic mutation", - "stem cell", - "epigenetics", - "genomics", - "bioinformatics", - "crispr", - "pcr", - ], - "Chemistry": [ - "chemistry", - "organic chemistry", - "inorganic chemistry", - "analytical chemistry", - "chemical", - "spectroscopy", - "chromatography", - "electrochemistry", - "polymer chemistry", - "green chemistry", - "natural product chemistry", - "phytochemical", - "coordination chemistry", - ], - "Computer Science": [ - "computer science", - "machine learning", - "artificial intelligence", - "software engineering", - "data science", - "algorithm", - "programming", - "neural network", - "deep learning", - "natural language processing", - "computer vision", - "cloud computing", - "cybersecurity", - "blockchain", - "distributed systems", - "database", - "internet of things", - "iot", - "big data", - "data mining", - "robotics", - "human computer interaction", - ], - "Geosciences": [ - "geology", - "geophysics", - "earth science", - "mineralogy", - "petrology", - "stratigraphy", - "sedimentology", - "remote sensing", - "gis", - "hydrogeology", - "seismology", - "geochemistry", - "oil sand", - "crude oil", - ], - "Marine Sciences": [ - "marine biology", - "oceanography", - "aquatic", - "fisheries", - "coastal", - "lagos lagoon", - "atlantic ocean", - "mangrove ecology", - "coral reef", - "tidal", - "estuarine", - "plankton", - "benthic", - "aquaculture", - ], - "Mathematics": [ - "mathematics", - "algebra", - "calculus", - "topology", - "geometry", - "statistics", - "probability", - "number theory", - "differential equations", - "numerical analysis", - "mathematical modeling", - "stochastic process", - "optimization", - "combinatorics", - "graph theory", - ], - "Microbiology": [ - "microbiology", - "bacteriology", - "virology", - "mycology", - "microorganism", - "antimicrobial", - "antibiotic resistance", - "pathogen", - "bacterial infection", - "fungal", - "fermentation", - "probiotics", - "microbiome", - "food microbiology", - ], - "Physics": [ - "physics", - "quantum", - "thermodynamics", - "optics", - "mechanics", - "astrophysics", - "particle physics", - "condensed matter", - "semiconductor", - "laser", - "plasma physics", - "nuclear physics", - "solid state physics", - "electromagnetic", - "radiation", - ], - "Zoology": [ - "zoology", - "animal biology", - "entomology", - "parasitology", - "wildlife", - "ecology", - "animal behavior", - "herpetology", - "ornithology", - "mammalogy", - "invertebrate", - "vertebrate", - "biodiversity", - ], - }, - "Faculty of Engineering": { - "Chemical and Polymer Engineering": [ - "chemical engineering", - "polymer", - "petrochemical", - "process engineering", - "distillation", - "reaction kinetics", - "unit operations", - "petroleum refining", - "biofuel", - "nanomaterial synthesis", - ], - "Civil and Environmental Engineering": [ - "civil engineering", - "structural", - "concrete", - "transportation", - "environmental engineering", - "geotechnical", - "foundation", - "highway", - "bridge", - "water resources", - "hydraulics", - "wastewater treatment", - "solid waste", - "urban infrastructure", - ], - "Electrical and Electronics Engineering": [ - "electrical engineering", - "circuit", - "power systems", - "electronics", - "telecommunications", - "signal processing", - "control systems", - "embedded systems", - "wireless communication", - "5g", - "microelectronics", - "power electronics", - "smart grid", - "renewable energy systems", - ], - "Mechanical Engineering": [ - "mechanical", - "thermofluids", - "mechatronics", - "manufacturing", - "robotics", - "turbine", - "heat transfer", - "fluid mechanics", - "vibration", - "cad", - "finite element analysis", - "tribology", - ], - "Metallurgical and Materials Engineering": [ - "metallurgy", - "materials science", - "corrosion", - "alloy", - "ceramics", - "composite materials", - "biomaterials", - "fracture mechanics", - "heat treatment", - "welding", - "casting", - "nanocomposite", - ], - "Systems Engineering": [ - "systems engineering", - "operations research", - "optimization", - "industrial engineering", - "supply chain", - "project management", - "reliability engineering", - "lean manufacturing", - "six sigma", - ], - }, - "College of Medicine": { - "Anatomy": [ - "anatomy", - "morphology", - "histology", - "embryology", - "neuroanatomy", - "gross anatomy", - "clinical anatomy", - ], - "Physiology": [ - "physiology", - "cellular physiology", - "metabolism", - "homeostasis", - "organ function", - "cardiovascular physiology", - "neurophysiology", - "renal physiology", - "endocrine physiology", - ], - "Pharmacology": [ - "pharmacology", - "drug", - "toxicology", - "pharmacokinetics", - "therapeutics", - "pharmacodynamics", - "clinical pharmacology", - "herbal pharmacology", - "drug interaction", - ], - "Morbid Anatomy": [ - "pathology", - "autopsy", - "forensic", - "histopathology", - "biopsy", - ], - "Chemical Pathology": [ - "clinical chemistry", - "biochemical", - "metabolic disorder", - "laboratory diagnosis", - "biomarker", - ], - "Haematology and Blood Transfusion": [ - "haematology", - "blood", - "transfusion", - "anemia", - "coagulation", - "sickle cell", - "lymphoma", - "leukemia", - "platelet", - ], - "Medical Microbiology and Parasitology": [ - "medical microbiology", - "parasitology", - "infectious disease", - "antimicrobial", - "tropical disease", - "malaria", - "typhoid", - "tuberculosis", - "hiv", - "aids", - "antiretroviral", - "cholera", - "ebola", - ], - "Community Health and Primary Care": [ - "public health", - "epidemiology", - "community medicine", - "preventive medicine", - "health promotion", - "vaccination", - "disease burden", - "morbidity", - "mortality", - "determinants of health", - ], - "Medicine": [ - "internal medicine", - "cardiology", - "nephrology", - "endocrinology", - "gastroenterology", - "hepatology", - "neurology", - "pulmonology", - "diabetes mellitus", - "hypertension", - "chronic disease", - ], - "Obstetrics and Gynaecology": [ - "obstetrics", - "gynaecology", - "pregnancy", - "maternal", - "reproductive health", - "antenatal", - "postnatal", - "caesarean", - "eclampsia", - "preeclampsia", - "fertility", - "infertility", - "cervical cancer", - "maternal mortality", - ], - "Paediatrics": [ - "paediatrics", - "pediatrics", - "child health", - "neonatology", - "infant mortality", - "childhood malnutrition", - "vaccination schedule", - ], - "Surgery": [ - "surgery", - "surgical", - "operation", - "laparoscopy", - "trauma", - "emergency surgery", - "neurosurgery", - "thoracic surgery", - "colorectal surgery", - "vascular surgery", - ], - "Anaesthesia": [ - "anaesthesia", - "anesthesia", - "pain management", - "critical care", - "icu", - ], - "Ophthalmology": [ - "ophthalmology", - "eye", - "vision", - "retina", - "glaucoma", - "cataract", - ], - "Orthopaedics and Traumatology": [ - "orthopaedics", - "orthopedics", - "bone", - "fracture", - "joint", - "arthroplasty", - "scoliosis", - "osteoporosis", - "musculoskeletal", - ], - "Psychiatry": [ - "psychiatry", - "mental health", - "psychosis", - "depression", - "schizophrenia", - "anxiety disorder", - "bipolar", - "substance abuse", - "post traumatic stress", - "neurodevelopmental", - ], - "Radiology": [ - "radiology", - "imaging", - "x-ray", - "mri", - "ct scan", - "ultrasound", - "interventional radiology", - "nuclear medicine", - "pet scan", - ], - }, - "Faculty of Pharmacy": { - "Clinical Pharmacy and Pharmacy Administration": [ - "clinical pharmacy", - "pharmaceutical care", - "pharmacy practice", - "pharmacovigilance", - "drug utilization", - "adherence", - ], - "Pharmaceutical Chemistry": [ - "pharmaceutical chemistry", - "drug synthesis", - "medicinal chemistry", - "structure activity relationship", - "drug design", - "lead compound", - ], - "Pharmaceutics and Pharmaceutical Technology": [ - "pharmaceutics", - "drug formulation", - "dosage form", - "tablet", - "controlled release", - "nanoparticle drug delivery", - "bioavailability", - ], - "Pharmacognosy": [ - "pharmacognosy", - "natural products", - "herbal medicine", - "phytochemistry", - "ethnopharmacology", - "alkaloid", - "flavonoid", - "indigenous knowledge medicine", - "traditional medicine", - ], - }, - "Faculty of Dental Sciences": { - "Oral and Maxillofacial Surgery": [ - "oral surgery", - "maxillofacial", - "jaw", - "dental surgery", - "facial reconstruction", - "cleft palate", - ], - "Preventive Dentistry": [ - "preventive dentistry", - "oral hygiene", - "dental public health", - "dental caries prevention", - "oral cancer screening", - ], - "Restorative Dentistry": [ - "restorative dentistry", - "prosthodontics", - "endodontics", - "dental restoration", - "crown", - "dental implant", - ], - "Child Dental Health": [ - "paediatric dentistry", - "pediatric dentistry", - "child dental", - "early childhood caries", - "fluoride", - ], - }, - "Faculty of Basic Medical Sciences": { - "Anatomy": ["anatomy", "morphology", "histology", "gross anatomy"], - "Physiology": ["physiology", "cellular physiology", "organ function"], - "Biochemistry": ["biochemistry", "molecular biology", "enzyme", "metabolism"], - }, - "Faculty of Social Sciences": { - "Economics": [ - "economics", - "macroeconomics", - "microeconomics", - "econometrics", - "development economics", - "fiscal policy", - "monetary policy", - "economic growth", - "poverty", - "inequality", - "trade policy", - "nigerian economy", - "african development", - ], - "Geography": [ - "geography", - "gis", - "remote sensing", - "cartography", - "spatial analysis", - "land use", - "urban geography", - "population geography", - "climate geography", - ], - "Mass Communication": [ - "mass communication", - "journalism", - "media", - "broadcasting", - "public relations", - "advertising", - "social media", - "digital media", - "media literacy", - "press freedom", - "fake news", - ], - "Political Science": [ - "political science", - "governance", - "democracy", - "international relations", - "corruption", - "federalism", - "electoral", - "political party", - "conflict resolution", - "peacekeeping", - "diplomacy", - ], - "Psychology": [ - "psychology", - "cognitive", - "behavioral", - "clinical psychology", - "educational psychology", - "health psychology", - "trauma", - "counseling", - "psychotherapy", - "social psychology", - ], - "Sociology": [ - "sociology", - "social theory", - "social structure", - "demography", - "gender", - "migration", - "urbanization", - "family sociology", - "crime", - "deviance", - "social inequality", - ], - }, - "Faculty of Law": { - "Public Law": [ - "constitutional law", - "administrative law", - "human rights", - "public law", - "electoral law", - "environmental law", - "petroleum law", - "niger delta", - "freedom of information", - ], - "Private and Property Law": [ - "contract law", - "property law", - "land law", - "tort", - "succession law", - "equity", - "customary law", - ], - "Commercial and Industrial Law": [ - "commercial law", - "corporate law", - "business law", - "intellectual property", - "banking law", - "insurance law", - "maritime law", - "competition law", - ], - "International and Comparative Law": [ - "international law", - "comparative law", - "treaty", - "international trade law", - "international human rights", - "international criminal law", - "ecowas", - "african union law", - ], - }, - "Faculty of Education": { - "Arts and Social Sciences Education": [ - "education", - "pedagogy", - "curriculum", - "teaching methods", - "instructional design", - "educational psychology", - "classroom management", - ], - "Science and Technology Education": [ - "science education", - "technology education", - "stem education", - "mathematics education", - "physics education", - "coding education", - ], - "Educational Administration": [ - "educational administration", - "school management", - "educational leadership", - "policy implementation", - "higher education management", - "university governance", - ], - }, - "Faculty of Environmental Sciences": { - "Architecture": [ - "architecture", - "architectural design", - "building design", - "urban design", - "sustainable architecture", - "green building", - "vernacular architecture", - "african architecture", - ], - "Estate Management": [ - "estate management", - "property valuation", - "real estate", - "land management", - "property market", - "lagos real estate", - ], - "Quantity Surveying": [ - "quantity surveying", - "cost estimation", - "construction economics", - "bill of quantities", - "procurement", - ], - "Surveying and Geoinformatics": [ - "surveying", - "geoinformatics", - "geodesy", - "land surveying", - "photogrammetry", - "total station", - ], - "Urban and Regional Planning": [ - "urban planning", - "regional planning", - "town planning", - "city planning", - "master plan", - "zoning", - "housing policy", - "slum upgrading", - "smart city", - "lagos masterplan", - ], - }, - "Faculty of Management Sciences": { - "Actuarial Science and Insurance": [ - "actuarial science", - "insurance", - "risk management", - "actuarial", - "life insurance", - "pension", - "annuity", - ], - "Accounting": [ - "accounting", - "financial accounting", - "auditing", - "taxation", - "forensic accounting", - "ifrs", - "financial reporting", - ], - "Business Administration": [ - "business administration", - "management", - "organizational behavior", - "strategic management", - "entrepreneurship", - "sme", - "startup", - "leadership", - "corporate governance", - ], - "Employment Relations and Human Resource Management": [ - "human resource", - "hr management", - "industrial relations", - "personnel management", - "talent management", - "employee relations", - ], - "Finance": [ - "finance", - "corporate finance", - "investment", - "financial markets", - "capital market", - "stock exchange", - "nigerian stock exchange", - "portfolio management", - "microfinance", - "fintech", - ], - }, - "Special Collections": { - "Indigenous Knowledge": [ - "oral traditions", - "indigenous epistemologies", - "ethnobotanical", - "traditional ecological", - "folklore storytelling", - "ancestral wisdom", - "indigenous languages", - "ritual", - "community transmission", - "precolonial", - "african philosophy", - "customary law", - "traditional medicine", - ], - "African Literature": [ - "african literature", - "postcolonial literature", - "oral literature", - "yoruba drama", - "igbo literature", - "african poetry", - "narratology", - "literary criticism africa", - "cultural identity memory", - "heritage", - ], - "Cultural Heritage": [ - "intangible cultural heritage", - "cultural identity memory", - "heritage", - "postcolonial heritage", - "oral history", - "traditional customs", - "material culture", - "sacred symbolism", - "cultural continuity", - "historical analysis", - "archaeology africa", - ], - }, -} - -# ─── SDG Keyword Map (UN Sustainable Development Goals) ─────────────────────── -SDG_MAP = { - "SDG 1 — No Poverty": [ - "poverty", - "extreme poverty", - "social protection", - "financial inclusion", - "microfinance", - "income inequality", - "livelihood", - "subsistence", - ], - "SDG 2 — Zero Hunger": [ - "food security", - "hunger", - "malnutrition", - "agricultural productivity", - "food production", - "crop yield", - "famine", - "stunting", - "wasting", - ], - "SDG 3 — Good Health & Well-being": [ - "health", - "disease", - "mortality", - "morbidity", - "vaccination", - "hiv", - "malaria", - "tuberculosis", - "maternal health", - "child health", - "cancer", - "mental health", - "epidemic", - "pandemic", - "covid", - "universal health coverage", - ], - "SDG 4 — Quality Education": [ - "education", - "literacy", - "school enrollment", - "learning outcomes", - "curriculum", - "pedagogy", - "higher education", - "stem education", - "teacher training", - "access to education", - ], - "SDG 5 — Gender Equality": [ - "gender equality", - "women empowerment", - "gender based violence", - "gender gap", - "female education", - "maternal", - "reproductive rights", - "gender disparity", - "patriarchy", - ], - "SDG 6 — Clean Water & Sanitation": [ - "water quality", - "wastewater", - "sanitation", - "water treatment", - "drinking water", - "groundwater", - "water pollution", - "hygiene", - ], - "SDG 7 — Affordable & Clean Energy": [ - "renewable energy", - "solar energy", - "wind energy", - "photovoltaic", - "biomass", - "energy access", - "electricity", - "power generation", - "energy poverty", - "clean cooking", - ], - "SDG 8 — Decent Work & Economic Growth": [ - "economic growth", - "employment", - "unemployment", - "labour market", - "gdp", - "sme", - "entrepreneurship", - "productivity", - "decent work", - ], - "SDG 9 — Industry, Innovation & Infrastructure": [ - "infrastructure", - "innovation", - "industrialization", - "manufacturing", - "technology transfer", - "broadband", - "transportation network", - "smart city", - "industry 4.0", - ], - "SDG 10 — Reduced Inequalities": [ - "inequality", - "income gap", - "social exclusion", - "discrimination", - "marginalization", - "migrant", - "refugee", - "disability", - ], - "SDG 11 — Sustainable Cities & Communities": [ - "urban planning", - "housing", - "slum", - "urbanization", - "resilient city", - "public transport", - "disaster risk", - "cultural heritage", - "lagos", - ], - "SDG 12 — Responsible Consumption & Production": [ - "sustainable consumption", - "circular economy", - "waste management", - "plastic pollution", - "e-waste", - "life cycle assessment", - "recycling", - ], - "SDG 13 — Climate Action": [ - "climate change", - "global warming", - "carbon emission", - "greenhouse", - "climate adaptation", - "mitigation", - "deforestation", - "flood", - "sea level rise", - "drought", - "desertification", - ], - "SDG 14 — Life Below Water": [ - "marine", - "ocean", - "fisheries", - "coastal", - "coral reef", - "aquatic biodiversity", - "water pollution", - "plastic in ocean", - "lagos lagoon", - "atlantic", - ], - "SDG 15 — Life on Land": [ - "biodiversity", - "ecosystem", - "deforestation", - "land degradation", - "endangered species", - "wildlife", - "forest conservation", - "soil erosion", - "desertification", - ], - "SDG 16 — Peace, Justice & Strong Institutions": [ - "governance", - "corruption", - "rule of law", - "human rights", - "conflict", - "peacebuilding", - "democracy", - "transparency", - "accountability", - "institutional reform", - ], - "SDG 17 — Partnerships for the Goals": [ - "international cooperation", - "development aid", - "technology transfer", - "capacity building", - "data sharing", - "open access", - "south south", - "african union", - "ecowas", - "global partnership", - ], -} - -# Email pattern for UNILAG staff -UNILAG_EMAIL_PATTERN = r"[a-z]+@(unilag\.edu\.ng|cmul\.edu\.ng)" - - -class UNILAGClassifier: - """ - Production-grade TF-IDF–style classifier for UNILAG research papers. - - Features: - - IDF weighting: keywords unique to one department score higher than - keywords shared across many departments - - Multi-word phrase detection with proper word boundary matching - - SDG alignment mapping for UN Sustainable Development Goals - - classify_with_explanation() for full transparency - """ - - def __init__(self): - self.structure = UNILAG_STRUCTURE - self._idf_weights: Dict[str, float] = {} - self._all_keywords: Dict[str, List[Tuple[str, str]]] = ( - {} - ) # kw -> [(faculty, dept)] - self._build_idf_index() - - def _build_idf_index(self): - """Pre-compute IDF weights for all keywords across departments.""" - # Count how many (faculty, dept) pairs contain each keyword - kw_doc_counts: Dict[str, int] = {} - total_depts = 0 - - for faculty, departments in self.structure.items(): - for dept, keywords in departments.items(): - total_depts += 1 - for kw in keywords: - kw_lower = kw.lower() - kw_doc_counts[kw_lower] = kw_doc_counts.get(kw_lower, 0) + 1 - if kw_lower not in self._all_keywords: - self._all_keywords[kw_lower] = [] - self._all_keywords[kw_lower].append((faculty, dept)) - - # Compute IDF: log(N / df) — rarer keywords get higher weight - for kw, df in kw_doc_counts.items(): - self._idf_weights[kw] = math.log(total_depts / df) + 1.0 - - def _score_keyword(self, kw: str, matches: int) -> float: - """ - Compute TF-IDF–style score for a keyword. - - Match count (TF proxy): log(1 + matches) - - IDF weight: log(N/df) + 1 - - Length bonus: longer multi-word phrases are more specific - """ - if matches == 0: - return 0.0 - phrase_length_bonus = len(kw.split()) * 0.8 - idf = self._idf_weights.get(kw.lower(), 1.0) - tf = math.log(1 + matches) - return tf * idf * phrase_length_bonus - - def classify( - self, text_corpus: str, threshold: float = 0.3 - ) -> List[Tuple[str, str, float]]: - """ - Classify text into UNILAG faculty/department buckets. - - Args: - text_corpus: Combined text (title + abstract + keywords) - threshold: Minimum score to include in results - - Returns: - Sorted list of (faculty, department, score) tuples, best first. - """ - if not text_corpus: - return [] - - try: - text_lower = str(text_corpus).lower() - except Exception: - return [] - - results = [] - - for faculty, departments in self.structure.items(): - for dept, keywords in departments.items(): - total_score = 0.0 - matched_kws = [] - - for kw in keywords: - try: - pattern = rf"\b{re.escape(kw)}\b" - found = re.findall(pattern, text_lower, re.IGNORECASE) - count = len(found) - if count > 0: - score = self._score_keyword(kw, count) - total_score += score - matched_kws.append((kw, count, round(score, 3))) - except Exception: - continue - - if total_score >= threshold: - results.append((faculty, dept, round(total_score, 3))) - - results.sort(key=lambda x: x[2], reverse=True) - return results - - def classify_with_explanation( - self, text_corpus: str, threshold: float = 0.3, top_n: int = 5 - ) -> Dict[str, Any]: - """ - Returns classification results WITH full explanation of scoring. - - Returns: - { - 'best_faculty': str, - 'best_department': str, - 'confidence': float, - 'results': [(faculty, dept, score), ...], - 'matched_keywords': [(keyword, count, score), ...], - 'sdg_alignment': [{'sdg': str, 'score': float, 'matched': [str]}, ...], - 'explanation': str - } - """ - if not text_corpus: - return self._empty_explanation() - - try: - text_lower = str(text_corpus).lower() - except Exception: - return self._empty_explanation() - - # Step 1: Full classification with keyword tracking - all_results = [] - all_keyword_hits: Dict[str, Dict] = ( - {} - ) # (faculty, dept) -> {kw: (count, score)} - - for faculty, departments in self.structure.items(): - for dept, keywords in departments.items(): - total_score = 0.0 - kw_details = [] - - for kw in keywords: - try: - pattern = rf"\b{re.escape(kw)}\b" - found = re.findall(pattern, text_lower, re.IGNORECASE) - count = len(found) - if count > 0: - score = self._score_keyword(kw, count) - total_score += score - kw_details.append( - { - "keyword": kw, - "count": count, - "score": round(score, 3), - "idf": round( - self._idf_weights.get(kw.lower(), 1.0), 3 - ), - } - ) - except Exception: - continue - - if total_score >= threshold: - all_results.append( - { - "faculty": faculty, - "department": dept, - "score": round(total_score, 3), - "keywords": sorted(kw_details, key=lambda x: -x["score"]), - } - ) - - all_results.sort(key=lambda x: -x["score"]) - top_results = all_results[:top_n] - - # Step 2: SDG alignment - sdg_hits = self.detect_sdg_alignment(text_corpus) - - # Step 3: Build explanation - if top_results: - best = top_results[0] - explanation = ( - f"Best match: {best['department']} ({best['faculty']}) " - f"with score {best['score']:.2f}. " - f"Top keywords: {', '.join(k['keyword'] for k in best['keywords'][:3])}." - ) - if sdg_hits: - explanation += f" Aligned with {sdg_hits[0]['sdg']}." - else: - explanation = "No strong faculty/department match found." - - return { - "best_faculty": top_results[0]["faculty"] if top_results else None, - "best_department": top_results[0]["department"] if top_results else None, - "confidence": top_results[0]["score"] if top_results else 0.0, - "results": [ - (r["faculty"], r["department"], r["score"]) for r in top_results - ], - "top_keywords": top_results[0]["keywords"][:5] if top_results else [], - "sdg_alignment": sdg_hits[:3], - "explanation": explanation, - "all_matches": top_results, - } - - def _empty_explanation(self) -> Dict[str, Any]: - return { - "best_faculty": None, - "best_department": None, - "confidence": 0.0, - "results": [], - "top_keywords": [], - "sdg_alignment": [], - "explanation": "No text provided.", - "all_matches": [], - } - - def detect_sdg_alignment(self, text_corpus: str) -> List[Dict[str, Any]]: - """ - Detect which UN Sustainable Development Goals this paper aligns with. - - Returns: - List of {'sdg': str, 'score': float, 'matched_keywords': [str]} - sorted by score descending. - """ - if not text_corpus: - return [] - - try: - text_lower = str(text_corpus).lower() - except Exception: - return [] - - sdg_results = [] - for sdg_name, keywords in SDG_MAP.items(): - matched = [] - score = 0.0 - for kw in keywords: - try: - pattern = rf"\b{re.escape(kw)}\b" - found = re.findall(pattern, text_lower, re.IGNORECASE) - if found: - count = len(found) - # Simple scoring: longer keywords worth more - kw_score = count * len(kw.split()) * 0.5 - score += kw_score - matched.append(kw) - except Exception: - continue - - if score > 0: - sdg_results.append( - { - "sdg": sdg_name, - "score": round(score, 2), - "matched_keywords": matched[:5], - } - ) - - sdg_results.sort(key=lambda x: -x["score"]) - return sdg_results - - def get_best_classification( - self, text_corpus: str - ) -> Optional[Tuple[str, str, float]]: - """Returns the single best classification or None if no match.""" - try: - results = self.classify(text_corpus) - return results[0] if results else None - except Exception: - return None - - def get_keyword_density(self, text_corpus: str) -> List[Dict[str, Any]]: - """ - Extract top meaningful keywords from a text corpus using TF scoring. - Includes multi-word domain phrase detection with score boosts. - Used for the keyword cloud endpoint. - - Returns list of {'word': str, 'count': int, 'score': float} - """ - if not text_corpus: - return [] - - STOP_WORDS = { - "the", - "and", - "for", - "with", - "this", - "that", - "from", - "have", - "been", - "were", - "their", - "which", - "these", - "about", - "other", - "into", - "than", - "more", - "such", - "some", - "what", - "when", - "where", - "there", - "also", - "using", - "used", - "show", - "study", - "paper", - "research", - "result", - "analysis", - "based", - "present", - "data", - "method", - "effect", - "approach", - "review", - "found", - "between", - "different", - "however", - "while", - "both", - "each", - "thus", - "among", - "within", - "during", - "after", - "before", - "under", - "very", - "most", - "only", - "just", - "they", - "them", - } - - try: - text_lower = str(text_corpus).lower() - words = re.findall(r"\b[a-z]{4,}\b", text_lower) - counts: Dict[str, int] = {} - for w in words: - if w not in STOP_WORDS: - counts[w] = counts.get(w, 0) + 1 - - # Multi-word phrase detection from known domain keywords - phrase_counts: Dict[str, int] = {} - phrase_parts: set = set() - for kw in self._all_keywords: - if " " in kw and len(kw) >= 8: - occ = len(re.findall(re.escape(kw), text_lower)) - if occ > 0: - phrase_counts[kw] = occ - for part in kw.split(): - if len(part) >= 4: - phrase_parts.add(part) - - # Build result - phrases first with 4x domain boost - result = [] - for phrase, count in sorted(phrase_counts.items(), key=lambda x: -x[1])[ - :20 - ]: - result.append( - {"word": phrase, "count": count, "score": round(count * 4.0, 2)} - ) - - # Add single words, skipping those covered by phrases - for word, count in sorted(counts.items(), key=lambda x: -x[1]): - if word in phrase_parts: - continue - is_domain = word in self._all_keywords - score = count * (2.5 if is_domain else 1.0) - result.append({"word": word, "count": count, "score": round(score, 2)}) - - result.sort(key=lambda x: -x["score"]) - return result[:60] - except Exception: - return [] - - -# Singleton instance -classifier = UNILAGClassifier() +""" +URAAS UNILAG Academic Classifier — Production Grade +===================================================== +Features: + - TF-IDF–style weighted keyword scoring (rare/specific keywords score higher) + - Multi-word phrase detection with word boundary matching + - Inverse-document-frequency (IDF) weighting across departments + - SDG (UN Sustainable Development Goals 1–17) alignment detection + - classify_with_explanation() for transparency and debugging + - Full 2024 UNILAG Faculty & Department structure +""" + +import math +import re +from typing import Any, Dict, List, Optional, Tuple + +# ─── Complete UNILAG Faculty and Department Structure (2024) ─────────────────── +UNILAG_STRUCTURE = { + "Faculty of Arts": { + "Creative Arts": [ + "creative arts", + "theatre", + "drama", + "performance", + "visual arts", + "music", + "dance", + "cinematography", + "costume design", + "stage design", + "african theatre", + "yoruba drama", + "nollywood", + "film studies", + ], + "English": [ + "english literature", + "linguistics", + "language studies", + "literary criticism", + "phonetics", + "morphology", + "syntax", + "pragmatics", + "discourse analysis", + "stylistics", + "narrative theory", + "african literature in english", + "postcolonial literature", + ], + "History and Strategic Studies": [ + "history", + "strategic studies", + "military", + "warfare", + "historical analysis", + "colonialism", + "decolonization", + "lagos history", + "nigerian history", + "precolonial", + "empire", + "slave trade", + "nationalism", + ], + "Philosophy": [ + "philosophy", + "logic", + "ethics", + "metaphysics", + "epistemology", + "african philosophy", + "political philosophy", + "philosophy of mind", + "existentialism", + "ontology", + "moral philosophy", + ], + "Languages": [ + "french", + "german", + "russian", + "arabic", + "foreign language", + "translation", + "language pedagogy", + "yoruba", + "igbo", + "hausa", + "pidgin", + "nigerian languages", + "oral tradition", + "indigenous language", + "language policy", + "multilingualism", + "sociolinguistics", + ], + }, + "Faculty of Science": { + "Biochemistry": [ + "biochemistry", + "molecular biology", + "enzymology", + "metabolism", + "protein", + "lipid", + "carbohydrate", + "enzyme kinetics", + "dna replication", + "gene expression", + "proteomics", + "metabolomics", + "oxidative stress", + ], + "Botany": [ + "botany", + "plant biology", + "plant physiology", + "taxonomy", + "flora", + "phytology", + "seed germination", + "photosynthesis", + "ethnobotany", + "mangrove", + "rainforest ecology", + "medicinal plants", + ], + "Cell Biology and Genetics": [ + "cell biology", + "genetics", + "cytology", + "heredity", + "dna", + "gene expression", + "chromosome", + "genetic mutation", + "stem cell", + "epigenetics", + "genomics", + "bioinformatics", + "crispr", + "pcr", + ], + "Chemistry": [ + "chemistry", + "organic chemistry", + "inorganic chemistry", + "analytical chemistry", + "chemical", + "spectroscopy", + "chromatography", + "electrochemistry", + "polymer chemistry", + "green chemistry", + "natural product chemistry", + "phytochemical", + "coordination chemistry", + ], + "Computer Science": [ + "computer science", + "machine learning", + "artificial intelligence", + "software engineering", + "data science", + "algorithm", + "programming", + "neural network", + "deep learning", + "natural language processing", + "computer vision", + "cloud computing", + "cybersecurity", + "blockchain", + "distributed systems", + "database", + "internet of things", + "iot", + "big data", + "data mining", + "robotics", + "human computer interaction", + ], + "Geosciences": [ + "geology", + "geophysics", + "earth science", + "mineralogy", + "petrology", + "stratigraphy", + "sedimentology", + "remote sensing", + "gis", + "hydrogeology", + "seismology", + "geochemistry", + "oil sand", + "crude oil", + ], + "Marine Sciences": [ + "marine biology", + "oceanography", + "aquatic", + "fisheries", + "coastal", + "lagos lagoon", + "atlantic ocean", + "mangrove ecology", + "coral reef", + "tidal", + "estuarine", + "plankton", + "benthic", + "aquaculture", + ], + "Mathematics": [ + "mathematics", + "algebra", + "calculus", + "topology", + "geometry", + "statistics", + "probability", + "number theory", + "differential equations", + "numerical analysis", + "mathematical modeling", + "stochastic process", + "optimization", + "combinatorics", + "graph theory", + ], + "Microbiology": [ + "microbiology", + "bacteriology", + "virology", + "mycology", + "microorganism", + "antimicrobial", + "antibiotic resistance", + "pathogen", + "bacterial infection", + "fungal", + "fermentation", + "probiotics", + "microbiome", + "food microbiology", + ], + "Physics": [ + "physics", + "quantum", + "thermodynamics", + "optics", + "mechanics", + "astrophysics", + "particle physics", + "condensed matter", + "semiconductor", + "laser", + "plasma physics", + "nuclear physics", + "solid state physics", + "electromagnetic", + "radiation", + ], + "Zoology": [ + "zoology", + "animal biology", + "entomology", + "parasitology", + "wildlife", + "ecology", + "animal behavior", + "herpetology", + "ornithology", + "mammalogy", + "invertebrate", + "vertebrate", + "biodiversity", + ], + }, + "Faculty of Engineering": { + "Chemical and Polymer Engineering": [ + "chemical engineering", + "polymer", + "petrochemical", + "process engineering", + "distillation", + "reaction kinetics", + "unit operations", + "petroleum refining", + "biofuel", + "nanomaterial synthesis", + ], + "Civil and Environmental Engineering": [ + "civil engineering", + "structural", + "concrete", + "transportation", + "environmental engineering", + "geotechnical", + "foundation", + "highway", + "bridge", + "water resources", + "hydraulics", + "wastewater treatment", + "solid waste", + "urban infrastructure", + ], + "Electrical and Electronics Engineering": [ + "electrical engineering", + "circuit", + "power systems", + "electronics", + "telecommunications", + "signal processing", + "control systems", + "embedded systems", + "wireless communication", + "5g", + "microelectronics", + "power electronics", + "smart grid", + "renewable energy systems", + ], + "Mechanical Engineering": [ + "mechanical", + "thermofluids", + "mechatronics", + "manufacturing", + "robotics", + "turbine", + "heat transfer", + "fluid mechanics", + "vibration", + "cad", + "finite element analysis", + "tribology", + ], + "Metallurgical and Materials Engineering": [ + "metallurgy", + "materials science", + "corrosion", + "alloy", + "ceramics", + "composite materials", + "biomaterials", + "fracture mechanics", + "heat treatment", + "welding", + "casting", + "nanocomposite", + ], + "Systems Engineering": [ + "systems engineering", + "operations research", + "optimization", + "industrial engineering", + "supply chain", + "project management", + "reliability engineering", + "lean manufacturing", + "six sigma", + ], + }, + "College of Medicine": { + "Anatomy": [ + "anatomy", + "morphology", + "histology", + "embryology", + "neuroanatomy", + "gross anatomy", + "clinical anatomy", + ], + "Physiology": [ + "physiology", + "cellular physiology", + "metabolism", + "homeostasis", + "organ function", + "cardiovascular physiology", + "neurophysiology", + "renal physiology", + "endocrine physiology", + ], + "Pharmacology": [ + "pharmacology", + "drug", + "toxicology", + "pharmacokinetics", + "therapeutics", + "pharmacodynamics", + "clinical pharmacology", + "herbal pharmacology", + "drug interaction", + ], + "Morbid Anatomy": [ + "pathology", + "autopsy", + "forensic", + "histopathology", + "biopsy", + ], + "Chemical Pathology": [ + "clinical chemistry", + "biochemical", + "metabolic disorder", + "laboratory diagnosis", + "biomarker", + ], + "Haematology and Blood Transfusion": [ + "haematology", + "blood", + "transfusion", + "anemia", + "coagulation", + "sickle cell", + "lymphoma", + "leukemia", + "platelet", + ], + "Medical Microbiology and Parasitology": [ + "medical microbiology", + "parasitology", + "infectious disease", + "antimicrobial", + "tropical disease", + "malaria", + "typhoid", + "tuberculosis", + "hiv", + "aids", + "antiretroviral", + "cholera", + "ebola", + ], + "Community Health and Primary Care": [ + "public health", + "epidemiology", + "community medicine", + "preventive medicine", + "health promotion", + "vaccination", + "disease burden", + "morbidity", + "mortality", + "determinants of health", + ], + "Medicine": [ + "internal medicine", + "cardiology", + "nephrology", + "endocrinology", + "gastroenterology", + "hepatology", + "neurology", + "pulmonology", + "diabetes mellitus", + "hypertension", + "chronic disease", + ], + "Obstetrics and Gynaecology": [ + "obstetrics", + "gynaecology", + "pregnancy", + "maternal", + "reproductive health", + "antenatal", + "postnatal", + "caesarean", + "eclampsia", + "preeclampsia", + "fertility", + "infertility", + "cervical cancer", + "maternal mortality", + ], + "Paediatrics": [ + "paediatrics", + "pediatrics", + "child health", + "neonatology", + "infant mortality", + "childhood malnutrition", + "vaccination schedule", + ], + "Surgery": [ + "surgery", + "surgical", + "operation", + "laparoscopy", + "trauma", + "emergency surgery", + "neurosurgery", + "thoracic surgery", + "colorectal surgery", + "vascular surgery", + ], + "Anaesthesia": [ + "anaesthesia", + "anesthesia", + "pain management", + "critical care", + "icu", + ], + "Ophthalmology": [ + "ophthalmology", + "eye", + "vision", + "retina", + "glaucoma", + "cataract", + ], + "Orthopaedics and Traumatology": [ + "orthopaedics", + "orthopedics", + "bone", + "fracture", + "joint", + "arthroplasty", + "scoliosis", + "osteoporosis", + "musculoskeletal", + ], + "Psychiatry": [ + "psychiatry", + "mental health", + "psychosis", + "depression", + "schizophrenia", + "anxiety disorder", + "bipolar", + "substance abuse", + "post traumatic stress", + "neurodevelopmental", + ], + "Radiology": [ + "radiology", + "imaging", + "x-ray", + "mri", + "ct scan", + "ultrasound", + "interventional radiology", + "nuclear medicine", + "pet scan", + ], + }, + "Faculty of Pharmacy": { + "Clinical Pharmacy and Pharmacy Administration": [ + "clinical pharmacy", + "pharmaceutical care", + "pharmacy practice", + "pharmacovigilance", + "drug utilization", + "adherence", + ], + "Pharmaceutical Chemistry": [ + "pharmaceutical chemistry", + "drug synthesis", + "medicinal chemistry", + "structure activity relationship", + "drug design", + "lead compound", + ], + "Pharmaceutics and Pharmaceutical Technology": [ + "pharmaceutics", + "drug formulation", + "dosage form", + "tablet", + "controlled release", + "nanoparticle drug delivery", + "bioavailability", + ], + "Pharmacognosy": [ + "pharmacognosy", + "natural products", + "herbal medicine", + "phytochemistry", + "ethnopharmacology", + "alkaloid", + "flavonoid", + "indigenous knowledge medicine", + "traditional medicine", + ], + }, + "Faculty of Dental Sciences": { + "Oral and Maxillofacial Surgery": [ + "oral surgery", + "maxillofacial", + "jaw", + "dental surgery", + "facial reconstruction", + "cleft palate", + ], + "Preventive Dentistry": [ + "preventive dentistry", + "oral hygiene", + "dental public health", + "dental caries prevention", + "oral cancer screening", + ], + "Restorative Dentistry": [ + "restorative dentistry", + "prosthodontics", + "endodontics", + "dental restoration", + "crown", + "dental implant", + ], + "Child Dental Health": [ + "paediatric dentistry", + "pediatric dentistry", + "child dental", + "early childhood caries", + "fluoride", + ], + }, + "Faculty of Basic Medical Sciences": { + "Anatomy": ["anatomy", "morphology", "histology", "gross anatomy"], + "Physiology": ["physiology", "cellular physiology", "organ function"], + "Biochemistry": ["biochemistry", "molecular biology", "enzyme", "metabolism"], + }, + "Faculty of Social Sciences": { + "Economics": [ + "economics", + "macroeconomics", + "microeconomics", + "econometrics", + "development economics", + "fiscal policy", + "monetary policy", + "economic growth", + "poverty", + "inequality", + "trade policy", + "nigerian economy", + "african development", + ], + "Geography": [ + "geography", + "gis", + "remote sensing", + "cartography", + "spatial analysis", + "land use", + "urban geography", + "population geography", + "climate geography", + ], + "Mass Communication": [ + "mass communication", + "journalism", + "media", + "broadcasting", + "public relations", + "advertising", + "social media", + "digital media", + "media literacy", + "press freedom", + "fake news", + ], + "Political Science": [ + "political science", + "governance", + "democracy", + "international relations", + "corruption", + "federalism", + "electoral", + "political party", + "conflict resolution", + "peacekeeping", + "diplomacy", + ], + "Psychology": [ + "psychology", + "cognitive", + "behavioral", + "clinical psychology", + "educational psychology", + "health psychology", + "trauma", + "counseling", + "psychotherapy", + "social psychology", + ], + "Sociology": [ + "sociology", + "social theory", + "social structure", + "demography", + "gender", + "migration", + "urbanization", + "family sociology", + "crime", + "deviance", + "social inequality", + ], + }, + "Faculty of Law": { + "Public Law": [ + "constitutional law", + "administrative law", + "human rights", + "public law", + "electoral law", + "environmental law", + "petroleum law", + "niger delta", + "freedom of information", + ], + "Private and Property Law": [ + "contract law", + "property law", + "land law", + "tort", + "succession law", + "equity", + "customary law", + ], + "Commercial and Industrial Law": [ + "commercial law", + "corporate law", + "business law", + "intellectual property", + "banking law", + "insurance law", + "maritime law", + "competition law", + ], + "International and Comparative Law": [ + "international law", + "comparative law", + "treaty", + "international trade law", + "international human rights", + "international criminal law", + "ecowas", + "african union law", + ], + }, + "Faculty of Education": { + "Arts and Social Sciences Education": [ + "education", + "pedagogy", + "curriculum", + "teaching methods", + "instructional design", + "educational psychology", + "classroom management", + ], + "Science and Technology Education": [ + "science education", + "technology education", + "stem education", + "mathematics education", + "physics education", + "coding education", + ], + "Educational Administration": [ + "educational administration", + "school management", + "educational leadership", + "policy implementation", + "higher education management", + "university governance", + ], + }, + "Faculty of Environmental Sciences": { + "Architecture": [ + "architecture", + "architectural design", + "building design", + "urban design", + "sustainable architecture", + "green building", + "vernacular architecture", + "african architecture", + ], + "Estate Management": [ + "estate management", + "property valuation", + "real estate", + "land management", + "property market", + "lagos real estate", + ], + "Quantity Surveying": [ + "quantity surveying", + "cost estimation", + "construction economics", + "bill of quantities", + "procurement", + ], + "Surveying and Geoinformatics": [ + "surveying", + "geoinformatics", + "geodesy", + "land surveying", + "photogrammetry", + "total station", + ], + "Urban and Regional Planning": [ + "urban planning", + "regional planning", + "town planning", + "city planning", + "master plan", + "zoning", + "housing policy", + "slum upgrading", + "smart city", + "lagos masterplan", + ], + }, + "Faculty of Management Sciences": { + "Actuarial Science and Insurance": [ + "actuarial science", + "insurance", + "risk management", + "actuarial", + "life insurance", + "pension", + "annuity", + ], + "Accounting": [ + "accounting", + "financial accounting", + "auditing", + "taxation", + "forensic accounting", + "ifrs", + "financial reporting", + ], + "Business Administration": [ + "business administration", + "management", + "organizational behavior", + "strategic management", + "entrepreneurship", + "sme", + "startup", + "leadership", + "corporate governance", + ], + "Employment Relations and Human Resource Management": [ + "human resource", + "hr management", + "industrial relations", + "personnel management", + "talent management", + "employee relations", + ], + "Finance": [ + "finance", + "corporate finance", + "investment", + "financial markets", + "capital market", + "stock exchange", + "nigerian stock exchange", + "portfolio management", + "microfinance", + "fintech", + ], + }, + "Special Collections": { + "Indigenous Knowledge": [ + "oral traditions", + "indigenous epistemologies", + "ethnobotanical", + "traditional ecological", + "folklore storytelling", + "ancestral wisdom", + "indigenous languages", + "ritual", + "community transmission", + "precolonial", + "african philosophy", + "customary law", + "traditional medicine", + ], + "African Literature": [ + "african literature", + "postcolonial literature", + "oral literature", + "yoruba drama", + "igbo literature", + "african poetry", + "narratology", + "literary criticism africa", + "cultural identity memory", + "heritage", + ], + "Cultural Heritage": [ + "intangible cultural heritage", + "cultural identity memory", + "heritage", + "postcolonial heritage", + "oral history", + "traditional customs", + "material culture", + "sacred symbolism", + "cultural continuity", + "historical analysis", + "archaeology africa", + ], + }, +} + +# ─── SDG Keyword Map (UN Sustainable Development Goals) ─────────────────────── +SDG_MAP = { + "SDG 1 — No Poverty": [ + "poverty", + "extreme poverty", + "social protection", + "financial inclusion", + "microfinance", + "income inequality", + "livelihood", + "subsistence", + ], + "SDG 2 — Zero Hunger": [ + "food security", + "hunger", + "malnutrition", + "agricultural productivity", + "food production", + "crop yield", + "famine", + "stunting", + "wasting", + ], + "SDG 3 — Good Health & Well-being": [ + "health", + "disease", + "mortality", + "morbidity", + "vaccination", + "hiv", + "malaria", + "tuberculosis", + "maternal health", + "child health", + "cancer", + "mental health", + "epidemic", + "pandemic", + "covid", + "universal health coverage", + ], + "SDG 4 — Quality Education": [ + "education", + "literacy", + "school enrollment", + "learning outcomes", + "curriculum", + "pedagogy", + "higher education", + "stem education", + "teacher training", + "access to education", + ], + "SDG 5 — Gender Equality": [ + "gender equality", + "women empowerment", + "gender based violence", + "gender gap", + "female education", + "maternal", + "reproductive rights", + "gender disparity", + "patriarchy", + ], + "SDG 6 — Clean Water & Sanitation": [ + "water quality", + "wastewater", + "sanitation", + "water treatment", + "drinking water", + "groundwater", + "water pollution", + "hygiene", + ], + "SDG 7 — Affordable & Clean Energy": [ + "renewable energy", + "solar energy", + "wind energy", + "photovoltaic", + "biomass", + "energy access", + "electricity", + "power generation", + "energy poverty", + "clean cooking", + ], + "SDG 8 — Decent Work & Economic Growth": [ + "economic growth", + "employment", + "unemployment", + "labour market", + "gdp", + "sme", + "entrepreneurship", + "productivity", + "decent work", + ], + "SDG 9 — Industry, Innovation & Infrastructure": [ + "infrastructure", + "innovation", + "industrialization", + "manufacturing", + "technology transfer", + "broadband", + "transportation network", + "smart city", + "industry 4.0", + ], + "SDG 10 — Reduced Inequalities": [ + "inequality", + "income gap", + "social exclusion", + "discrimination", + "marginalization", + "migrant", + "refugee", + "disability", + ], + "SDG 11 — Sustainable Cities & Communities": [ + "urban planning", + "housing", + "slum", + "urbanization", + "resilient city", + "public transport", + "disaster risk", + "cultural heritage", + "lagos", + ], + "SDG 12 — Responsible Consumption & Production": [ + "sustainable consumption", + "circular economy", + "waste management", + "plastic pollution", + "e-waste", + "life cycle assessment", + "recycling", + ], + "SDG 13 — Climate Action": [ + "climate change", + "global warming", + "carbon emission", + "greenhouse", + "climate adaptation", + "mitigation", + "deforestation", + "flood", + "sea level rise", + "drought", + "desertification", + ], + "SDG 14 — Life Below Water": [ + "marine", + "ocean", + "fisheries", + "coastal", + "coral reef", + "aquatic biodiversity", + "water pollution", + "plastic in ocean", + "lagos lagoon", + "atlantic", + ], + "SDG 15 — Life on Land": [ + "biodiversity", + "ecosystem", + "deforestation", + "land degradation", + "endangered species", + "wildlife", + "forest conservation", + "soil erosion", + "desertification", + ], + "SDG 16 — Peace, Justice & Strong Institutions": [ + "governance", + "corruption", + "rule of law", + "human rights", + "conflict", + "peacebuilding", + "democracy", + "transparency", + "accountability", + "institutional reform", + ], + "SDG 17 — Partnerships for the Goals": [ + "international cooperation", + "development aid", + "technology transfer", + "capacity building", + "data sharing", + "open access", + "south south", + "african union", + "ecowas", + "global partnership", + ], +} + +# Email pattern for UNILAG staff +UNILAG_EMAIL_PATTERN = r"[a-z]+@(unilag\.edu\.ng|cmul\.edu\.ng)" + + +class UNILAGClassifier: + """ + Production-grade TF-IDF–style classifier for UNILAG research papers. + + Features: + - IDF weighting: keywords unique to one department score higher than + keywords shared across many departments + - Multi-word phrase detection with proper word boundary matching + - SDG alignment mapping for UN Sustainable Development Goals + - classify_with_explanation() for full transparency + """ + + def __init__(self): + self.structure = UNILAG_STRUCTURE + self._idf_weights: Dict[str, float] = {} + self._all_keywords: Dict[str, List[Tuple[str, str]]] = ( + {} + ) # kw -> [(faculty, dept)] + self._build_idf_index() + + def _build_idf_index(self): + """Pre-compute IDF weights for all keywords across departments.""" + # Count how many (faculty, dept) pairs contain each keyword + kw_doc_counts: Dict[str, int] = {} + total_depts = 0 + + for faculty, departments in self.structure.items(): + for dept, keywords in departments.items(): + total_depts += 1 + for kw in keywords: + kw_lower = kw.lower() + kw_doc_counts[kw_lower] = kw_doc_counts.get(kw_lower, 0) + 1 + if kw_lower not in self._all_keywords: + self._all_keywords[kw_lower] = [] + self._all_keywords[kw_lower].append((faculty, dept)) + + # Compute IDF: log(N / df) — rarer keywords get higher weight + for kw, df in kw_doc_counts.items(): + self._idf_weights[kw] = math.log(total_depts / df) + 1.0 + + def _score_keyword(self, kw: str, matches: int) -> float: + """ + Compute TF-IDF–style score for a keyword. + - Match count (TF proxy): log(1 + matches) + - IDF weight: log(N/df) + 1 + - Length bonus: longer multi-word phrases are more specific + """ + if matches == 0: + return 0.0 + phrase_length_bonus = len(kw.split()) * 0.8 + idf = self._idf_weights.get(kw.lower(), 1.0) + tf = math.log(1 + matches) + return tf * idf * phrase_length_bonus + + def classify( + self, text_corpus: str, threshold: float = 0.3 + ) -> List[Tuple[str, str, float]]: + """ + Classify text into UNILAG faculty/department buckets. + + Args: + text_corpus: Combined text (title + abstract + keywords) + threshold: Minimum score to include in results + + Returns: + Sorted list of (faculty, department, score) tuples, best first. + """ + if not text_corpus: + return [] + + try: + text_lower = str(text_corpus).lower() + except Exception: + return [] + + results = [] + + for faculty, departments in self.structure.items(): + for dept, keywords in departments.items(): + total_score = 0.0 + matched_kws = [] + + for kw in keywords: + try: + pattern = rf"\b{re.escape(kw)}\b" + found = re.findall(pattern, text_lower, re.IGNORECASE) + count = len(found) + if count > 0: + score = self._score_keyword(kw, count) + total_score += score + matched_kws.append((kw, count, round(score, 3))) + except Exception: + continue + + if total_score >= threshold: + results.append((faculty, dept, round(total_score, 3))) + + results.sort(key=lambda x: x[2], reverse=True) + return results + + def classify_with_explanation( + self, text_corpus: str, threshold: float = 0.3, top_n: int = 5 + ) -> Dict[str, Any]: + """ + Returns classification results WITH full explanation of scoring. + + Returns: + { + 'best_faculty': str, + 'best_department': str, + 'confidence': float, + 'results': [(faculty, dept, score), ...], + 'matched_keywords': [(keyword, count, score), ...], + 'sdg_alignment': [{'sdg': str, 'score': float, 'matched': [str]}, ...], + 'explanation': str + } + """ + if not text_corpus: + return self._empty_explanation() + + try: + text_lower = str(text_corpus).lower() + except Exception: + return self._empty_explanation() + + # Step 1: Full classification with keyword tracking + all_results = [] + all_keyword_hits: Dict[str, Dict] = ( + {} + ) # (faculty, dept) -> {kw: (count, score)} + + for faculty, departments in self.structure.items(): + for dept, keywords in departments.items(): + total_score = 0.0 + kw_details = [] + + for kw in keywords: + try: + pattern = rf"\b{re.escape(kw)}\b" + found = re.findall(pattern, text_lower, re.IGNORECASE) + count = len(found) + if count > 0: + score = self._score_keyword(kw, count) + total_score += score + kw_details.append( + { + "keyword": kw, + "count": count, + "score": round(score, 3), + "idf": round( + self._idf_weights.get(kw.lower(), 1.0), 3 + ), + } + ) + except Exception: + continue + + if total_score >= threshold: + all_results.append( + { + "faculty": faculty, + "department": dept, + "score": round(total_score, 3), + "keywords": sorted(kw_details, key=lambda x: -x["score"]), + } + ) + + all_results.sort(key=lambda x: -x["score"]) + top_results = all_results[:top_n] + + # Step 2: SDG alignment + sdg_hits = self.detect_sdg_alignment(text_corpus) + + # Step 3: Build explanation + if top_results: + best = top_results[0] + explanation = ( + f"Best match: {best['department']} ({best['faculty']}) " + f"with score {best['score']:.2f}. " + f"Top keywords: {', '.join(k['keyword'] for k in best['keywords'][:3])}." + ) + if sdg_hits: + explanation += f" Aligned with {sdg_hits[0]['sdg']}." + else: + explanation = "No strong faculty/department match found." + + return { + "best_faculty": top_results[0]["faculty"] if top_results else None, + "best_department": top_results[0]["department"] if top_results else None, + "confidence": top_results[0]["score"] if top_results else 0.0, + "results": [ + (r["faculty"], r["department"], r["score"]) for r in top_results + ], + "top_keywords": top_results[0]["keywords"][:5] if top_results else [], + "sdg_alignment": sdg_hits[:3], + "explanation": explanation, + "all_matches": top_results, + } + + def _empty_explanation(self) -> Dict[str, Any]: + return { + "best_faculty": None, + "best_department": None, + "confidence": 0.0, + "results": [], + "top_keywords": [], + "sdg_alignment": [], + "explanation": "No text provided.", + "all_matches": [], + } + + def detect_sdg_alignment(self, text_corpus: str) -> List[Dict[str, Any]]: + """ + Detect which UN Sustainable Development Goals this paper aligns with. + + Returns: + List of {'sdg': str, 'score': float, 'matched_keywords': [str]} + sorted by score descending. + """ + if not text_corpus: + return [] + + try: + text_lower = str(text_corpus).lower() + except Exception: + return [] + + sdg_results = [] + for sdg_name, keywords in SDG_MAP.items(): + matched = [] + score = 0.0 + for kw in keywords: + try: + pattern = rf"\b{re.escape(kw)}\b" + found = re.findall(pattern, text_lower, re.IGNORECASE) + if found: + count = len(found) + # Simple scoring: longer keywords worth more + kw_score = count * len(kw.split()) * 0.5 + score += kw_score + matched.append(kw) + except Exception: + continue + + if score > 0: + sdg_results.append( + { + "sdg": sdg_name, + "score": round(score, 2), + "matched_keywords": matched[:5], + } + ) + + sdg_results.sort(key=lambda x: -x["score"]) + return sdg_results + + def get_best_classification( + self, text_corpus: str + ) -> Optional[Tuple[str, str, float]]: + """Returns the single best classification or None if no match.""" + try: + results = self.classify(text_corpus) + return results[0] if results else None + except Exception: + return None + + def get_keyword_density(self, text_corpus: str) -> List[Dict[str, Any]]: + """ + Extract top meaningful keywords from a text corpus using TF scoring. + Includes multi-word domain phrase detection with score boosts. + Used for the keyword cloud endpoint. + + Returns list of {'word': str, 'count': int, 'score': float} + """ + if not text_corpus: + return [] + + STOP_WORDS = { + "the", + "and", + "for", + "with", + "this", + "that", + "from", + "have", + "been", + "were", + "their", + "which", + "these", + "about", + "other", + "into", + "than", + "more", + "such", + "some", + "what", + "when", + "where", + "there", + "also", + "using", + "used", + "show", + "study", + "paper", + "research", + "result", + "analysis", + "based", + "present", + "data", + "method", + "effect", + "approach", + "review", + "found", + "between", + "different", + "however", + "while", + "both", + "each", + "thus", + "among", + "within", + "during", + "after", + "before", + "under", + "very", + "most", + "only", + "just", + "they", + "them", + } + + try: + text_lower = str(text_corpus).lower() + words = re.findall(r"\b[a-z]{4,}\b", text_lower) + counts: Dict[str, int] = {} + for w in words: + if w not in STOP_WORDS: + counts[w] = counts.get(w, 0) + 1 + + # Multi-word phrase detection from known domain keywords + phrase_counts: Dict[str, int] = {} + phrase_parts: set = set() + for kw in self._all_keywords: + if " " in kw and len(kw) >= 8: + occ = len(re.findall(re.escape(kw), text_lower)) + if occ > 0: + phrase_counts[kw] = occ + for part in kw.split(): + if len(part) >= 4: + phrase_parts.add(part) + + # Build result - phrases first with 4x domain boost + result = [] + for phrase, count in sorted(phrase_counts.items(), key=lambda x: -x[1])[ + :20 + ]: + result.append( + {"word": phrase, "count": count, "score": round(count * 4.0, 2)} + ) + + # Add single words, skipping those covered by phrases + for word, count in sorted(counts.items(), key=lambda x: -x[1]): + if word in phrase_parts: + continue + is_domain = word in self._all_keywords + score = count * (2.5 if is_domain else 1.0) + result.append({"word": word, "count": count, "score": round(score, 2)}) + + result.sort(key=lambda x: -x["score"]) + return result[:60] + except Exception: + return [] + + +# Singleton instance +classifier = UNILAGClassifier()