app.py

"""
ResearchBee — FastAPI backend v1.4.0

Changes from v1.3 (verified-sources release):
- /api/check-license is now DOI-first and GROUNDED. Policy facts come from the
  OA.Works Permissions API (the dataset behind cOAlition S's Journal Checker
  Tool), never from the LLM. The LLM writes advisory prose only.
- Khazna deposit status via a nightly OAI-PMH harvest (three-valued: checked /
  in_khazna / needs_deposit — "could not check" is never reported as "absent").
- Every policy answer carries a provenance label. Any LLM-derived
  policy_status is forcibly downgraded to "Not confirmed".
- Repository finder constrained to a curated registry; the LLM selects and
  explains, the server supplies the facts.

Changes from v1.2:
- Rate limiting middleware (sliding window, 20 req/min global, 10/min chat per IP)
- /api/chat now streams via SSE (StreamingResponse)
- /api/openalex-works — related literature per journal (OpenAlex)
- /api/altmetric — journal attention score badge (Altmetric free API)
"""
import os
import json
import re
import asyncio
import logging
import time
from collections import defaultdict
from contextlib import asynccontextmanager
from typing import Optional, List, AsyncGenerator

from fastapi import FastAPI, HTTPException, Request
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse, StreamingResponse
from pydantic import BaseModel, Field

from openai import AsyncOpenAI, RateLimitError, APIStatusError
from tenacity import (
    retry, stop_after_attempt, wait_exponential,
    retry_if_exception_type, before_sleep_log,
)
import httpx

from scimago_index import lookup_any, load_scimago, search_by_subject, load_subject_index
import openalex
from prompts import (
    JOURNAL_SYSTEM_PROMPT, LICENSE_SYSTEM_PROMPT, REPO_SYSTEM_PROMPT,
    SUBJECT_NORMALISE_PROMPT, COVER_LETTER_PROMPT, CHAT_SYSTEM_PROMPT_BASE,
)
from ku_knowledge import get_knowledge
from utils import norm_issn

# ── Verified-source modules (no LLM inside any of these) ──────────────────
import oaworks
import doi_utils
import repositories
from khazna_loader import khazna_index
from repositories import REGISTRY, registry_for_prompt, hydrate_selection
from prompts import ADVISORY_NOTE_PROMPT

logger = logging.getLogger(__name__)

# ── Constants ──────────────────────────────────────────────────────────────
# Model routing — all separate quota buckets at Tier 4
MODEL       = "gpt-4.1-mini"  # journal analysis, chat, repo finder — 10M TPM, Tier 4
MODEL_LIGHT = "gpt-4.1-nano"  # subject browse, cover letter, license — 10M TPM, Tier 4
# gpt-4.1-mini: $0.04/1M input (4x cheaper than gpt-4o-mini, same quality for JSON tasks)
# gpt-4.1-nano: $0.01/1M input (ultra-cheap for simple normalisation calls)
# Both have completely separate quota from gpt-4o-mini

# ── KU APC agreement publishers (Option B: publisher + top 15% check) ────────
# KU has agreements with 6 publishers. Badge shown if publisher matches AND
# journal is in top 15% Scopus (checked via SCImago quartile Q1/Q2 at enrichment).
# Always shown with "verify eligibility" caveat — not a guarantee.
# Aliases that are ordinary English and would over-match as substrings.
# These only count when they lead the publisher string.
AMBIGUOUS_ALIASES = {
    "academic press", "saunders", "masson", "adis", "informa",
    "nature research", "nature publishing",
}

KU_APC_PUBLISHER_GROUPS = [
    # ACS
    {"american chemical society"},
    # Elsevier — all regional entities
    {"elsevier", "elsevier b.v.", "elsevier bv", "elsevier ltd", "elsevier inc.",
     "elsevier inc", "elsevier ireland ltd", "elsevier gmbh", "elsevier masson",
     "elsevier singapore", "elsevier australia", "elsevier beijing",
     "elsevier editora", "elsevier espana",
     # Imprints whose SCImago string never contains "Elsevier".
     # NB: normalisation strips dots, so "W.B. Saunders" becomes "wb saunders"
     # — that variant must be listed explicitly or ~108 titles are missed.
     "academic press", "cell press", "wb saunders", "w b saunders",
     "churchill livingstone", "mosby", "pergamon", "saunders",
     "masson", "urban and fischer", "hanley and belfus"},
    # IEEE — all society variants
    {"ieee", "ieee advancing technology", "ieee communications society",
     "ieee computational intelligence", "ieee computer society",
     "ieee electron devices", "ieee microwave"},
    # Springer Nature — all Springer variants
    {"springer", "springer nature", "springernature", "nature portfolio",
     "nature publishing", "springer international publishing",
     "springer international publishing ag", "springer berlin", "springer boston",
     "springer china", "springer netherlands", "springer new york",
     "springer london", "springer japan", "springer verlag",
     "springer science and business media", "springer publishing",
     "springer science and business media llc", "springer nature america",
     "biomed central", "palgrave", "palgrave macmillan", "adis",
     "birkhauser", "nature research"},
    # Taylor & Francis — all T&F variants (note: & in SCImago CSV)
    {"taylor and francis", "taylor & francis", "taylor & francis",
     "routledge", "routledge, taylor", "taylor and francis ltd",
     "taylor and francis inc", "taylor and francis a.s",
     "taylor and francis asia pacific", "informa", "informa uk",
     "informa uk limited", "psychology press", "crc press",
     "dove medical press", "taylor and francis group"},
    # Wiley — all Wiley variants
    {"wiley", "wiley-blackwell", "wiley-blackwell publishing",
     "john wiley and sons", "john wiley & sons", "john wiley & sons",
     "wiley-liss", "wiley-vch", "wiley - vch",
     "blackwell", "blackwell publishing", "blackwell science",
     "john wiley and sons inc", "john wiley and sons ltd"},
]

def is_ku_apc_covered(publisher: str, quartile: str = "") -> bool:
    """
    Q1-only check: publisher must match one of KU's 6 agreement publishers
    AND journal must be Q1 by Scopus/SCImago.
    Q1 is used as proxy for top 15% — always shown with verification caveat.
    Q2/Q3/Q4 or unknown quartile → badge not shown.
    """
    pub = (publisher or "").lower().strip()
    if not pub:
        return False
    # Must be Q1 — strictly
    if quartile != "Q1":
        return False
    # Normalize: strip HTML entities, punctuation variants
    def norm(s):
        return (s.lower()
                 .replace("&", "&").replace("&", "and")
                 .replace(".", "").replace(",", "")
                 .replace("-", " ").strip())

    pub_norm  = norm(pub)
    pub_tokens = pub_norm.split()

    for group in KU_APC_PUBLISHER_GROUPS:
        for alias in group:
            alias_norm   = norm(alias)
            alias_tokens = alias_norm.split()

            # Exact match
            if pub_norm == alias_norm:
                return True

            # Generic aliases must appear at the START of the publisher string.
            # Without this, "Maximum Academic Press" (not Elsevier) matches
            # "academic press", wrongly awarding 20 titles a KU APC badge.
            if alias_norm in AMBIGUOUS_ALIASES:
                if pub_norm.startswith(alias_norm + " ") or pub_norm == alias_norm:
                    return True
                continue

            # Distinctive aliases: contiguous token-sequence match, so
            # "john wiley and sons inc" matches "wiley" but "wileyfish" does not.
            n = len(alias_tokens)
            for i in range(len(pub_tokens) - n + 1):
                if pub_tokens[i:i + n] == alias_tokens:
                    return True
    return False

# ── Rate limiter — sliding window in-memory ────────────────────────────────
_rate_store: dict = defaultdict(list)   # ip → [timestamps]

RATE_LIMITS = {
    "global": (20, 60),   # 20 requests per 60 seconds per IP
    "chat":   (10, 60),   # 10 chat requests per 60 seconds per IP
}

def check_rate_limit(ip: str, bucket: str = "global") -> bool:
    """Returns True if allowed, False if rate limited."""
    max_req, window = RATE_LIMITS[bucket]
    key = f"{ip}:{bucket}"
    now = time.time()
    _rate_store[key] = [t for t in _rate_store[key] if now - t < window]
    if len(_rate_store[key]) >= max_req:
        return False
    _rate_store[key].append(now)
    return True

def get_client_ip(request: Request) -> str:
    """Extract real IP from request, respecting proxy headers."""
    forwarded = request.headers.get("X-Forwarded-For")
    if forwarded:
        return forwarded.split(",")[0].strip()
    return request.client.host if request.client else "unknown"

OPF_BASE = "https://v2.sherpa.ac.uk/cgi/retrieve"  # Open Policy Finder (Jisc)


# ── Khazna — KU Institutional Repository (always first in results) ─────────
KHAZNA = {
    "name": "Khazna — Khalifa University Research Portal",
    "type": "Institutional",
    "url": "https://khazna.ku.ac.ae",
    "scope": "Khalifa University institutional repository for articles, theses, datasets and research output.",
    "fit_reason": "Primary institutional repository for all KU researchers. Deposit here for KU compliance and visibility.",
    "data_types_accepted": "All types — articles, datasets, theses, preprints, reports",
    "max_file_size": "Contact library for large datasets",
    "licences_supported": "CC BY, CC BY-NC, CC BY-NC-ND, Restricted",
    "persistent_identifier": "Handle",
    "versioning": "Yes",
    "embargo_support": "Yes",
    "access_model": "Mixed",
    "cost": "Free",
    "certification": "Institutional",
    "fair_alignment": "High",
    "funder_compliance_note": "Satisfies KU institutional OA and RDM policy requirements.",
    "sensitive_data_suitable": "Conditional",
    "confidence": "High",
    "verification_status": "Confirmed — KU institutional repository",
    "risk_flag": None,
    "is_khazna": True,
    "verify_links": {
        "re3data":     "https://www.re3data.org/search?query=Khalifa+University",
        "fairsharing": "https://fairsharing.org/search?q=Khalifa+University",
        "khazna_contact": "mailto:khazna@ku.ac.ae",
    },
}

KHAZNA_DEPOSIT_CARD = {
    "title": "🏛️ Deposit to Khazna — KU Institutional Repository",
    "url": "https://khazna.ku.ac.ae",
    "contact": "khazna@ku.ac.ae",
    "library_url": "https://library.ku.ac.ae/lib",
    "message": "As a KU researcher, deposit your accepted manuscript (or metadata record if under embargo) to Khazna for institutional visibility and compliance.",
    "metadata_tip": "Even if depositing data in a domain-specific repository, always register metadata in Khazna so your work appears in KU's research portfolio.",
}

ROUTE_KEYWORDS = {
    "journals": [
        "find journal", "find a journal", "which journal", "where to publish",
        "submit my paper", "submit manuscript", "journal recommendation",
        "journal for my", "best journal", "suitable journal", "publish my paper",
        "journals for", "browse journal", "journal match", "manuscript analysis",
        "analyse my paper", "analyze my paper", "journal submission",
    ],
    "license": [
        "self-archive", "self archive", "can i deposit", "postprint", "preprint",
        "accepted manuscript", "embargo", "archiving rights", "open policy finder",
        "sherpa", "green oa policy", "self-archiving", "deposit my paper",
        "deposit my article", "version of record", "vor", "aam", "repository rights",
        "can i post", "can i upload my paper",
    ],
    "data": [
        "dataset", "data repository", "research data", "deposit data", "zenodo",
        "dryad", "figshare", "where to deposit data", "data management",
        "rdm", "fair data", "data deposit", "store my data", "share my data",
    ],
}


def quick_route(message: str) -> Optional[str]:
    """Keyword pre-filter — returns tab name or None for LLM to decide."""
    msg = message.lower()
    for tab, keywords in ROUTE_KEYWORDS.items():
        if any(k in msg for k in keywords):
            return tab
    return None


# ── Open Policy Finder (Jisc) integration ────────────────────────────────
async def fetch_opf_policy(journal_name: str, issn: Optional[str] = None) -> str:
    """
    Query Open Policy Finder (formerly SHERPA RoMEO) for journal self-archiving policy.
    Returns a formatted string summary or empty string if not found.
    API: https://openpolicyfinder.jisc.ac.uk/api/
    No API key required for basic queries.
    """
    try:
        client = openalex._get_client()  # reuse shared httpx client

        # Try ISSN first (more precise), fall back to title search
        if issn:
            clean_issn = norm_issn(issn) or issn.strip()
            params = {
                "item-type": "publication",
                "format":    "Json",
                "filter":    json.dumps([["issn", "equals", clean_issn]]),
            }
        else:
            params = {
                "item-type": "publication",
                "format":    "Json",
                "filter":    json.dumps([["title", "equals", journal_name]]),
            }

        r = await client.get(OPF_BASE, params=params, timeout=8.0)
        if not r.is_success:
            return ""

        data = r.json()
        items = data.get("items", [])
        if not items:
            return ""

        pub = items[0]
        title = pub.get("title", [{}])[0].get("title", journal_name)
        policies = pub.get("publisher_policy", [])
        if not policies:
            return f"Open Policy Finder found '{title}' but no self-archiving policy listed."

        lines = [f"Open Policy Finder data for: {title}"]
        for policy in policies[:2]:  # max 2 policies
            permitted = policy.get("permitted_oa", [])
            for p in permitted[:3]:  # max 3 permissions per policy
                article_version = ", ".join(p.get("article_version", ["Unknown version"]))
                location = p.get("location", {})
                repos = ", ".join(location.get("named_repository", []) +
                                  location.get("location", []))
                conditions = "; ".join(p.get("conditions", []))
                embargo_months = p.get("embargo", {}).get("amount", 0)
                embargo_units  = p.get("embargo", {}).get("units", "months")
                embargo_str    = f"{embargo_months} {embargo_units}" if embargo_months else "No embargo"
                licence        = ", ".join([lic.get("licence", "") for lic in p.get("license", [])])

                lines.append(
                    f"- Version: {article_version} | "
                    f"Location: {repos or 'any repository'} | "
                    f"Embargo: {embargo_str} | "
                    f"Licence: {licence or 'not specified'} | "
                    f"Conditions: {conditions or 'none'}"
                )

        opf_url = f"https://openpolicyfinder.jisc.ac.uk/id/publication/{pub.get('id', '')}"
        lines.append(f"Full policy: {opf_url}")
        return "\n".join(lines)

    except Exception as e:
        logger.warning(f"[OPF] lookup failed for {journal_name}: {e}")
        return ""


def detect_journal_policy_question(message: str) -> Optional[str]:
    """
    Detect if the user is asking about a specific journal's self-archiving policy.
    Returns the journal name if detected, else None.
    Uses simple heuristics — good enough for 90% of cases.
    """
    msg = message.lower()
    policy_triggers = [
        "self-archiv", "self archiv", "can i deposit", "can i post",
        "embargo", "postprint", "accepted manuscript", "preprint policy",
        "archiving rights", "green oa", "repository policy",
        "version of record", "aam", "open access rights",
    ]
    if not any(t in msg for t in policy_triggers):
        return None

    # Try to extract journal name — look for quoted names or "in X journal" patterns
    quoted = re.findall(r'["\']([^"\']{3,80})["\']', message)
    if quoted:
        return quoted[0]

    patterns = [
        r'\bin\s+([A-Z][A-Za-z &\-]{3,60}(?:Journal|Review|Letters|Reports|Science|Nature|Communications|Research)?)',
        r'\bfor\s+([A-Z][A-Za-z &\-]{3,60}(?:Journal|Review|Letters|Reports|Science|Nature|Communications|Research)?)',
        r'\b([A-Z][A-Za-z &\-]{3,60}(?:Journal|Review|Letters|Reports|Science|Nature|Communications|Research))\b',
    ]
    for pat in patterns:
        m = re.search(pat, message)
        if m:
            candidate = m.group(1).strip()
            if len(candidate) > 3:
                return candidate
    return None


# ── Singleton OpenAI client ────────────────────────────────────────────────
_openai_client: Optional[AsyncOpenAI] = None


def get_client() -> AsyncOpenAI:
    global _openai_client
    key = os.environ.get("OPENAI_API_KEY")
    if not key:
        raise HTTPException(500, "OPENAI_API_KEY not configured")
    if _openai_client is None:
        _openai_client = AsyncOpenAI(api_key=key)
    return _openai_client


# ── Lifespan ───────────────────────────────────────────────────────────────
@asynccontextmanager
async def lifespan(app: FastAPI):
    load_scimago()
    load_subject_index()
    openalex.init_client()

    # Khazna index is produced by a nightly GitHub Actions harvest and stored
    # in an HF Dataset. Loading is non-blocking: the Space serves immediately
    # and Khazna lookups report "not checked" for the few seconds until it lands.
    asyncio.create_task(asyncio.to_thread(khazna_index.reload))

    scheduler = None
    try:
        from apscheduler.schedulers.asyncio import AsyncIOScheduler
        from apscheduler.triggers.cron import CronTrigger
        scheduler = AsyncIOScheduler(timezone="Asia/Dubai")
        scheduler.add_job(
            lambda: asyncio.create_task(asyncio.to_thread(khazna_index.reload)),
            CronTrigger(hour=23, minute=30),      # 30 min after the 23:00 harvest
            id="khazna_refresh", replace_existing=True,
        )
        scheduler.start()
        logger.info("Khazna refresh scheduled for 23:30 Asia/Dubai")
    except Exception as e:
        logger.warning("Khazna refresh scheduler unavailable: %s", e)

    yield

    if scheduler:
        scheduler.shutdown(wait=False)
    await openalex.close_client()


# ── App ────────────────────────────────────────────────────────────────────
app = FastAPI(title="ResearchBee API", version="1.3.0", lifespan=lifespan)

app.add_middleware(
    CORSMiddleware,
    allow_origins=[
        "https://ku-library.github.io",
        "http://localhost",
        "http://127.0.0.1",
        "http://localhost:3000",
        "http://127.0.0.1:3000",
    ],
    allow_methods=["GET", "POST", "OPTIONS"],
    allow_headers=["Content-Type"],
)


# ── Helpers ────────────────────────────────────────────────────────────────
def lang_instruction(language: str) -> str:
    return "Please respond entirely in Arabic." if language == "arabic" else ""


def build_verify_links(name: str, issn: Optional[str], sourceid: str = "") -> dict:
    q = re.sub(r"[^\w\s]", "", name).strip()
    q_encoded = q.replace(" ", "%20")
    ni = norm_issn(issn)
    return {
        "scimago":        (f"https://www.scimagojr.com/journalsearch.php?q={ni}&tip=iss"
                           if ni else f"https://www.scimagojr.com/journalsearch.php?q={q_encoded}&tip=jou"),
        "sherpa_romeo":   (f"https://openpolicyfinder.jisc.ac.uk/search?search={q_encoded}"
                           "&per_page=10&publication_page=1&publisher_page=1&funder_page=1"),
        "doaj":           (f"https://doaj.org/toc/{ni.replace('-', '')}"
                           if ni else f"https://doaj.org/search/journals?source=%7B%22query%22%3A%7B%22query_string%22%3A%7B%22query%22%3A%22{q_encoded}%22%7D%7D%7D"),
        "openalex":       (f"https://openalex.org/sources/issn:{ni}"
                           if ni else f"https://openalex.org/sources?filter=display_name.search:{q_encoded}"),
        "scopus_sources": "https://www.scopus.com/sources",
        "issn_display":   ni or "",
        "name_display":   name.strip(),
    }


def enrich_ranking(journal: dict, scimago: Optional[dict], oa_pct: Optional[dict]) -> dict:
    if scimago:
        q = scimago.get("quartile", "")
        return {
            "requested": True, "source": "Verified from journal data", "year": "2025",
            "category": scimago.get("categories", ""), "metric_value": None,
            "percentile": f"Rank #{scimago.get('rank', '—')} globally" if scimago.get("rank") else None,
            "quartile": q if q and q != "-" else None,
            "interpretation": f"Quartile verified from journal metadata (2025 dataset, matched by ISSN {scimago.get('matched_issn', '')}).",
            "verification_status": "Confirmed", "h_index": scimago.get("h_index"),
        }
    elif oa_pct:
        return {
            "requested": True, "source": "OpenAlex-derived percentile", "year": "2025",
            "category": oa_pct.get("concept"), "metric_value": None,
            "percentile": f"{oa_pct['percentile']}th percentile within {oa_pct['peer_count']} {oa_pct['concept']} journals",
            "quartile": oa_pct.get("quartile"),
            "interpretation": "OpenAlex-derived percentile — NOT a Scopus/SCImago quartile.",
            "verification_status": "OpenAlex-derived (not a Scopus/SCImago metric)",
        }
    else:
        return {
            "requested": True, "source": "Not available", "year": None,
            "category": None, "metric_value": None, "percentile": None, "quartile": None,
            "interpretation": "No journal metrics match found. Use verify links to check Scopus Sources directly.",
            "verification_status": "Not confirmed — verify manually via Scopus Sources link",
        }


def _scimago_title_fallback(title: str) -> Optional[dict]:
    """
    Fallback: fuzzy title match in SCImago when ISSN lookup fails.
    Used when LLM returns a wrong/hallucinated ISSN.
    """
    if not title:
        return None
    rows = load_subject_index()
    title_lower = title.lower().strip()
    # Exact title match first
    for r in rows:
        if r["title"].lower() == title_lower:
            ni = norm_issn(r.get("issn", ""))
            if ni:
                return lookup_any([ni])
    # Partial match — title contains all significant words
    words = [w for w in title_lower.split() if len(w) > 3]
    if len(words) >= 2:
        for r in rows:
            rt = r["title"].lower()
            if all(w in rt for w in words):
                ni = norm_issn(r.get("issn", ""))
                if ni:
                    result = lookup_any([ni])
                    if result:
                        return result
    return None


async def enrich_journals(journals: list) -> list:
    enriched_oa = await openalex.enrich_all(journals)
    result = []
    for j in enriched_oa:
        issn = j.get("issn")
        oa = j.get("openalex") or {}
        all_issns = [issn, oa.get("issn_l")] + (oa.get("issns") or [])
        scimago = lookup_any(all_issns)

        # ── Title fallback: if ISSN lookup fails, try matching by journal name ──
        if not scimago:
            scimago = _scimago_title_fallback(j.get("name", ""))
            if scimago:
                logger.info(f"[Enrich] Title fallback matched: {j.get('name')}")

        oa_pct = j.get("oa_percentile")
        final_issn = (scimago or {}).get("matched_issn") or oa.get("issn_l") or issn
        j["issn"] = final_issn
        j["ranking"] = enrich_ranking(j, scimago, oa_pct)
        sourceid = (scimago or {}).get("sourceid", "")
        j["verify_links"] = build_verify_links(j.get("name", ""), final_issn, sourceid)
        j.pop("oa_percentile", None)
        result.append(j)
    return result


def enrich_extended(extended: list) -> list:
    result = []
    for j in extended:
        issn = j.get("issn")
        scimago = lookup_any([issn]) if issn else None
        q        = (scimago or {}).get("quartile", "")
        h_index  = (scimago or {}).get("h_index", "")
        publisher= (scimago or {}).get("publisher", "") or j.get("publisher", "")
        oa       = (scimago or {}).get("open_access", "No")
        country  = (scimago or {}).get("country", "")
        sourceid = (scimago or {}).get("sourceid", "")
        j["quartile"]    = q if q and q != "-" else None
        j["h_index"]     = int(h_index) if str(h_index).isdigit() else None
        j["publisher"]   = publisher
        j["open_access"] = oa
        j["country"]     = country
        j["verify_links"]  = build_verify_links(j.get("name", ""), issn, sourceid)
        j["ku_apc_covered"] = is_ku_apc_covered(publisher, q)
        result.append(j)
    return result


# ── OpenAI call with retry ─────────────────────────────────────────────────
@retry(
    retry=retry_if_exception_type((RateLimitError, APIStatusError)),
    stop=stop_after_attempt(3),
    wait=wait_exponential(multiplier=1, min=2, max=10),
    before_sleep=before_sleep_log(logger, logging.WARNING),
    reraise=True,
)
async def call_openai(client: AsyncOpenAI, system: str, user: str, max_tokens: int = 4096, model: str = None) -> dict:
    """Call OpenAI with automatic retry on rate limit (up to 2 retries, 2s backoff)."""
    _model = model or MODEL
    for attempt in range(3):
        try:
            resp = await client.chat.completions.create(
                model=_model,
                response_format={"type": "json_object"},
                messages=[
                    {"role": "system", "content": system},
                    {"role": "user",   "content": user},
                ],
                max_tokens=max_tokens,
                temperature=0.3,
            )
            content = resp.choices[0].message.content or "{}"
            try:
                result = json.loads(content)
                if "error" in result and isinstance(result["error"], dict):
                    raise ValueError(f"OpenAI returned error: {result['error']}")
                return result
            except json.JSONDecodeError:
                m = re.search(r"\{[\s\S]*\}", content)
                return json.loads(m.group(0)) if m else {"error": "Failed to parse response"}
        except RateLimitError as e:
            err_msg = str(e).lower()
            # Quota exceeded = billing cap, not a rate spike — don't retry
            if "quota" in err_msg or "billing" in err_msg or "insufficient_quota" in err_msg:
                logger.error(f"[OpenAI] Quota/billing limit hit: {e}")
                raise HTTPException(429, "OpenAI quota exceeded. Please contact the library administrator.")
            # Rate limit = TPM/RPM spike — retry with backoff
            logger.warning(f"[OpenAI] Rate limit attempt {attempt+1}/3: {e}")
            if attempt < 2:
                await asyncio.sleep(2 ** attempt)
                continue
            raise HTTPException(429, "OpenAI rate limit exceeded. Please try again in a moment.")
        except Exception:
            raise


# ── Request models ─────────────────────────────────────────────────────────
_T300 = Field("", max_length=300)
_T500 = Field("", max_length=500)


class ManuscriptInput(BaseModel):
    title:               str = Field(..., max_length=300)
    abstract:            str = Field(..., max_length=3000)
    keywords:            str = _T500
    discipline:          str = _T300
    article_type:        str = Field("Original research article", max_length=100)
    methods:             str = _T500
    audience:            str = _T300
    funder:              str = _T300
    institution:         str = _T300
    country:             str = _T300
    apc_budget:          str = _T300
    oa_preference:       str = Field("Either", max_length=50)
    ranking_preference:  str = Field("No filter", max_length=50)
    ranking_source:      str = Field("Either", max_length=50)
    speed_preference:    str = Field("Standard", max_length=50)
    preferred_journals:  str = _T500
    avoid_journals:      str = _T500
    repository_target:   str = Field("Institutional repository", max_length=100)


class JournalRequest(BaseModel):
    manuscript: ManuscriptInput
    model:      str = Field("gpt-4o-mini", max_length=30)
    language:   str = Field("english", max_length=10)


class LicenseInput(BaseModel):
    # journal_name is no longer required: a DOI alone is a complete request,
    # and the endpoint raises 422 if neither is supplied.
    journal_name:         str = Field("", max_length=300)
    doi:                  str = Field("", max_length=400)   # DOI or article URL
    issn:                 str = Field("", max_length=20)
    publisher:            str = _T300
    manuscript_version:   str = Field("All versions", max_length=50)
    funder:               str = _T300
    intended_repository:  str = Field("Institutional repository", max_length=100)
    intended_licence:     str = Field("CC BY", max_length=50)
    notes:                str = _T500


class LicenseRequest(BaseModel):
    license_input: LicenseInput
    model:         str = Field("gpt-4o-mini", max_length=30)
    language:      str = Field("english", max_length=10)


class DatasetInput(BaseModel):
    title:                  str = Field(..., max_length=300)
    description:            str = Field(..., max_length=2000)
    discipline:             str = Field(..., max_length=300)
    data_types:             str = Field(..., max_length=300)
    file_formats:           str = _T300
    approx_size:            str = _T300
    sensitivity:            str = Field("Open", max_length=50)
    contains_personal_data: str = Field("No", max_length=50)
    licence_intent:         str = Field("CC BY", max_length=50)
    funder:                 str = _T300
    institution:            str = _T300
    country:                str = _T300
    needs_doi:              str = Field("Yes", max_length=10)
    needs_versioning:       str = Field("Yes", max_length=10)
    embargo_required:       str = Field("No", max_length=10)
    preferred_repository:   str = _T300
    related_publication:    str = _T300
    notes:                  str = _T500


class RepoRequest(BaseModel):
    dataset:  DatasetInput
    model:    str = Field("gpt-4o-mini", max_length=30)
    language: str = Field("english", max_length=10)


class SubjectRequest(BaseModel):
    subject:  str = Field(..., min_length=1, max_length=200)
    model:    str = Field("gpt-4o-mini", max_length=30)
    language: str = Field("english", max_length=10)


class CoverLetterRequest(BaseModel):
    manuscript_title: str = Field(..., max_length=300)
    abstract:         str = Field(..., max_length=3000)
    journal_name:     str = Field(..., max_length=300)
    publisher:        str = Field("", max_length=200)
    article_type:     str = Field("Original research article", max_length=100)
    author_name:      str = Field("", max_length=200)
    discipline:       str = Field("", max_length=200)
    language:         str = Field("english", max_length=10)
    model:            str = Field("gpt-4o-mini", max_length=30)


class ExportRequest(BaseModel):
    journals: list
    format:   str = Field("bibtex", max_length=10)


class ChatMessage(BaseModel):
    role:    str = Field(..., pattern="^(user|assistant)$")
    content: str = Field(..., max_length=2000)


class ChatRequest(BaseModel):
    messages: List[ChatMessage] = Field(..., max_length=20)
    language: str = Field("english", max_length=10)


# ── Routes ─────────────────────────────────────────────────────────────────
@app.get("/api/health")
async def health():
    """
    Service status plus data-source provenance.

    The `sources` block is the honest answer to "where do this system's facts
    come from, and are those sources actually reachable right now?"
    """
    return {
        "status": "ok",
        "service": "ResearchBee",
        "version": "1.4.0",
        "sources": {
            "oaworks": {
                "endpoint": oaworks.OA_WORKS_BASE + oaworks.PERMISSIONS_PATH,
                "auth": "none required",
                "ku_ror_set": bool(oaworks.KU_ROR),
            },
            "khazna": khazna_index.status,
            "repository_registry": {"entries": len(REGISTRY)},
            "scimago": {"loaded": True},
        },
    }


@app.post("/api/admin/reload-khazna")
async def reload_khazna(request: Request):
    """
    Force an immediate reload of the Khazna index.

    Called by the harvest workflow after a successful publish, so the Space
    picks up new data within seconds instead of waiting for the 23:30 refresh.
    Idempotent and harmless: it only re-reads a public dataset.
    """
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests.")
    ok = await asyncio.to_thread(khazna_index.reload)
    return {"reloaded": ok, "khazna": khazna_index.status}


@app.post("/api/chat-debug2")
async def chat_debug2(request: Request):
    """Test raw request parsing for chat."""
    try:
        body = await request.json()
        return {"body_received": body, "step": "json_parse_ok"}
    except Exception as e:
        return {"failed_at": "json_parse", "error": str(e)}

@app.post("/api/chat-debug3")
async def chat_debug3(req: ChatRequest):
    """Test ChatRequest Pydantic validation."""
    return {
        "validation_ok": True,
        "message_count": len(req.messages),
        "first_role": req.messages[0].role if req.messages else None,
        "language": req.language,
    }

@app.post("/api/chat-debug4")
async def chat_debug4(req: ChatRequest, request: Request):
    """Step through the exact chat endpoint logic and report where it fails."""
    import traceback
    results = {}
    try:
        # Step 1: rate limit
        ip = get_client_ip(request)
        results["s1_ip"] = ip

        # Step 2: last message
        last_msg = req.messages[-1].content.strip()
        results["s2_last_msg"] = last_msg

        # Step 3: quick_route
        fast_route = quick_route(last_msg)
        results["s3_fast_route"] = fast_route

        # Step 4: detect journal
        journal_name = detect_journal_policy_question(last_msg)
        results["s4_journal"] = journal_name

        # Step 5: build system prompt
        system = CHAT_SYSTEM_PROMPT_BASE.replace(
            "{KU_KNOWLEDGE}", get_knowledge()
        ).replace(
            "{OPF_DATA}", "No OPF data."
        )
        results["s5_system_len"] = len(system)

        # Step 6: build messages
        messages = [{"role": "system", "content": system}]
        messages += [{"role": m.role, "content": m.content} for m in req.messages]
        results["s6_msg_count"] = len(messages)

        # Step 7: get client
        client = get_client()
        results["s7_client"] = "ok"

        # Step 8: streaming create call
        response = await client.chat.completions.create(
            model=MODEL,
            messages=messages,
            max_tokens=20,
            temperature=0.3,
            stream=True,
        )
        tokens = []
        async for chunk in response:
            token = chunk.choices[0].delta.content if chunk.choices else None
            if token:
                tokens.append(token)
        results["s8_stream_tokens"] = "".join(tokens)

        return {"all_steps_ok": True, "results": results}

    except Exception as e:
        results["error"] = str(e)
        results["traceback"] = traceback.format_exc()
        return {"failed": True, "results": results}

@app.get("/api/chat-debug")
async def chat_debug():
    """Diagnostic endpoint — tests each step of chat without streaming."""
    results = {}
    try:
        from ku_knowledge import get_knowledge
        kb = get_knowledge()
        results["kb_length"] = len(kb)
        results["step1"] = "KB loaded OK"
    except Exception as e:
        return {"failed_at": "kb", "error": str(e)}

    try:
        from prompts import CHAT_SYSTEM_PROMPT_BASE
        system = CHAT_SYSTEM_PROMPT_BASE.replace("{KU_KNOWLEDGE}", kb).replace("{OPF_DATA}", "none")
        results["system_length"] = len(system)
        results["step2"] = "Prompt OK"
    except Exception as e:
        return {"failed_at": "prompt", "error": str(e)}

    try:
        client = get_client()
        results["step3"] = "Client OK"
    except Exception as e:
        return {"failed_at": "client", "error": str(e)}

    try:
        resp = await client.chat.completions.create(
            model=MODEL,
            messages=[
                {"role": "system", "content": "Reply with just: OK"},
                {"role": "user",   "content": "test"},
            ],
            max_tokens=5,
            temperature=0,
            stream=False,
        )
        results["step4"] = "LLM call OK: " + resp.choices[0].message.content
    except Exception as e:
        return {"failed_at": "llm_call", "error": str(e), "results_so_far": results}

    try:
        resp2 = await client.chat.completions.create(
            model=MODEL,
            messages=[
                {"role": "system", "content": "Reply with just: OK"},
                {"role": "user",   "content": "test"},
            ],
            max_tokens=5,
            temperature=0,
            stream=True,
        )
        tokens = []
        async for chunk in resp2:
            t = chunk.choices[0].delta.content if chunk.choices else None
            if t: tokens.append(t)
        results["step5"] = "Streaming OK: " + "".join(tokens)
    except Exception as e:
        return {"failed_at": "streaming", "error": str(e), "results_so_far": results}

    return {"all_ok": True, "results": results}


# ── OpenAlex related works per journal ────────────────────────────────────
@app.get("/api/openalex-works")
async def openalex_works(request: Request, source_id: str, per_page: int = 5):
    """
    Fetch recent highly-cited works from an OpenAlex source (journal).
    source_id: OpenAlex source ID e.g. S137773608
    """
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests. Please wait a moment.")

    if not source_id or len(source_id) > 30:
        raise HTTPException(400, "Invalid source_id")

    try:
        client = openalex._get_client()
        clean_id = source_id.strip().lstrip("S")
        r = await client.get(
            "https://api.openalex.org/works",
            params={
                "filter":   f"primary_location.source.id:S{clean_id},is_retracted:false",
                "sort":     "cited_by_count:desc",
                "per_page": min(per_page, 8),
                "select":   "id,title,publication_year,doi,cited_by_count,authorships",
            },
            timeout=6.0,
        )
        if not r.is_success:
            return {"works": []}

        works = []
        for w in r.json().get("results", []):
            authors = [
                a.get("author", {}).get("display_name", "")
                for a in (w.get("authorships") or [])[:3]
            ]
            works.append({
                "title":        w.get("title", ""),
                "year":         w.get("publication_year"),
                "doi":          w.get("doi", ""),
                "cited_by":     w.get("cited_by_count", 0),
                "authors":      authors,
                "openalex_url": w.get("id", ""),
            })
        return {"works": works}
    except Exception as e:
        logger.warning(f"[OpenAlex works] {e}")
        return {"works": []}


# ── Trending papers by concept/topic ─────────────────────────────────────
@app.get("/api/trending-papers")
async def trending_papers(request: Request, concept: str = "", concept_id: str = "", per_page: int = 5):
    """
    Fetch most-cited papers in last 3 years for a given concept/topic via OpenAlex.
    concept_id: OpenAlex concept ID (e.g. C41008148) — preferred
    concept:    display name fallback (e.g. "Machine Learning")
    """
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests.")

    if not concept and not concept_id:
        return {"works": []}

    try:
        from datetime import datetime
        year_cutoff = datetime.now().year - 3
        client = openalex._get_client()

        # Build filter — prefer concept_id for accuracy
        if concept_id:
            clean_id = concept_id.strip().lstrip("C")
            filter_str = f"concepts.id:C{clean_id},publication_year:>{year_cutoff},is_retracted:false"
        else:
            # Use keyword search via default_search
            filter_str = f"publication_year:>{year_cutoff},is_retracted:false"

        params = {
            "filter":   filter_str,
            "sort":     "cited_by_count:desc",
            "per_page": min(per_page, 8),
            "select":   "id,title,publication_year,doi,cited_by_count,authorships,primary_location",
        }

        # If only concept name, use search param instead of filter
        if not concept_id and concept:
            params["search"] = concept
            params["filter"] = f"publication_year:>{year_cutoff},is_retracted:false"

        r = await client.get(
            "https://api.openalex.org/works",
            params=params,
            timeout=8.0,
        )
        if not r.is_success:
            return {"works": []}

        works = []
        for w in r.json().get("results", []):
            authors = [
                a.get("author", {}).get("display_name", "")
                for a in (w.get("authorships") or [])[:3]
            ]
            journal = (w.get("primary_location") or {}).get("source", {}) or {}
            works.append({
                "title":        w.get("title", ""),
                "year":         w.get("publication_year"),
                "doi":          w.get("doi", ""),
                "cited_by":     w.get("cited_by_count", 0),
                "authors":      [a for a in authors if a],
                "journal":      journal.get("display_name", ""),
                "openalex_url": w.get("id", ""),
            })
        return {"works": works}
    except Exception as e:
        logger.warning(f"[TrendingPapers] {e}")
        return {"works": []}


# ── Altmetric journal attention ───────────────────────────────────────────
@app.get("/api/altmetric")
async def altmetric_score(request: Request, issn: str):
    """
    Fetch Altmetric attention data for a journal via its ISSN.
    Uses Altmetric free API — no key required for basic queries.
    """
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests. Please wait a moment.")

    clean_issn = norm_issn(issn)
    if not clean_issn:
        return {"score": None, "url": None}

    try:
        client = openalex._get_client()
        r = await client.get(
            f"https://api.altmetric.com/v1/issn/{clean_issn.replace('-','')}",
            timeout=5.0,
        )
        if not r.is_success:
            return {"score": None, "url": None, "issn": clean_issn}

        data = r.json()
        return {
            "score":     round(data.get("score", 0)),
            "url":       data.get("details_url"),
            "image":     data.get("images", {}).get("small"),
            "title":     data.get("title"),
            "issn":      clean_issn,
        }
    except Exception as e:
        logger.warning(f"[Altmetric] {e}")
        return {"score": None, "url": None, "issn": clean_issn}


@app.post("/api/analyze-journal")
async def analyze_journal(req: JournalRequest, request: Request):
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests. Please wait a moment and try again.")
    from scimago_index import _subject_cache, _scimago_cache
    if not _subject_cache or not _scimago_cache:
        load_scimago()
        load_subject_index()
    client = get_client()
    lang = lang_instruction(req.language)
    user_msg = (
        f"{lang}Analyse this manuscript and return the structured JSON response.\n\n"
        f"MANUSCRIPT INPUT:\n{json.dumps(req.manuscript.dict(), indent=2)}\n\n"
        "IMPORTANT: Return EXACTLY 10 detailed journals in journals array and EXACTLY 20 quick matches in extended_list (30 total).\n\n"
        "Return JSON only."
    )
    parsed = await call_openai(client, JOURNAL_SYSTEM_PROMPT, user_msg)
    if "error" in parsed:
        raise HTTPException(500, parsed["error"])

    if isinstance(parsed.get("journals"), list):
        parsed["journals"] = await enrich_journals(parsed["journals"])
    if isinstance(parsed.get("extended_list"), list):
        parsed["extended_list"] = enrich_extended(parsed["extended_list"])

    # Tag journals where KU has an APC agreement (Option B: publisher + quartile)
    for j in parsed.get("journals", []):
        q = (j.get("ranking") or {}).get("quartile", "")
        j["ku_apc_covered"] = is_ku_apc_covered(j.get("publisher", ""), q)
    for j in parsed.get("extended_list", []):
        j["ku_apc_covered"] = is_ku_apc_covered(j.get("publisher", ""), j.get("quartile", ""))

    parsed["khazna"] = KHAZNA_DEPOSIT_CARD
    return {"result": parsed}


# ── Khazna deposit status ─────────────────────────────────────────────────
def khazna_status_for(doi: str) -> dict:
    """
    THREE-VALUED on purpose.

    "could not check" and "not in Khazna" are different answers. Conflating them
    would tell a researcher to deposit something already deposited, or imply an
    absence we never actually verified.
    """
    if not doi:
        return {"checked": False, "reason": "no DOI supplied"}
    if not khazna_index.available:
        return {"checked": False, "reason": "Khazna index unavailable"}
    rec = khazna_index.get(doi)
    if rec is None:
        return {"checked": True, "in_khazna": False,
                "message": "This article does not appear in the KU repository, Khazna."}
    state = rec.get("deposit_state", "undetermined")
    return {
        "checked": True,
        "in_khazna": True,
        "deposit_state": state,
        "embargo_end": rec.get("embargo_end"),
        "portal_url": rec.get("portal_url", ""),
        "needs_deposit": state in ("metadata_only", "closed"),
        "message": {
            "open":          "Already in the KU repository, Khazna, with an open full text — nothing further needed.",
            "embargoed":     "In the KU repository, Khazna, with an embargoed file.",
            "restricted":    "In the KU repository, Khazna, but the attached file is restricted.",
            "closed":        "In the KU repository, Khazna, as a closed record — an open version may be permitted.",
            # Khazna is currently a metadata-only CRIS: records exist for KU
            # output but no files are attached. This is therefore the normal
            # case, and the message is framed as an opportunity rather than a
            # problem with the author's record.
            "metadata_only": ("Your publication is already recorded in the KU repository, "
                              "Khazna, but no full text is attached. Adding the permitted "
                              "version makes it openly readable and discoverable."),
            "undetermined":  "In the KU repository, Khazna; access status could not be determined.",
        }.get(state, "In the KU repository, Khazna."),
    }


async def _advisory_note(green: dict, li: "LicenseInput", lang: str) -> str:
    """The LLM writes prose from verified facts only. It never supplies a value."""
    facts = {
        "can_archive":        green.get("_can_archive"),
        "versions_permitted": green.get("_versions"),
        "best_version":       green.get("_best_version"),
        "locations":          green.get("_locations"),
        "embargo_months":     green.get("_embargo_months"),
        "embargo_stated":     green.get("_embargo_months") is not None,
        "embargo_end":        green.get("embargo_end"),
        "licence":            green.get("_licence"),
        "journal_oa_type":    green.get("journal_oa_type"),
        "institutional_repository_permitted": green.get("_institutional_ok"),
        "author_intent": {
            "repository": li.intended_repository,
            "licence":    li.intended_licence,
            "version":    li.manuscript_version,
            "funder":     li.funder,
        },
    }
    try:
        out = await call_openai(
            get_client(), ADVISORY_NOTE_PROMPT,
            lang + json.dumps(facts, ensure_ascii=False),
            max_tokens=280, model=MODEL_LIGHT,
        )
        return out.get("note", "") if isinstance(out, dict) else ""
    except Exception as e:
        logger.warning("advisory note failed: %s", e)
        return ""


@app.post("/api/check-license")
async def check_license(req: LicenseRequest, request: Request):
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests. Please wait a moment and try again.")

    li = req.license_input
    lang = lang_instruction(req.language)
    doi = doi_utils.extract_doi(li.doi) or doi_utils.extract_doi(li.notes)

    # ══ PATH A — DOI supplied: verified, article-level, no LLM policy facts ══
    if doi:
        permissions, khazna = await asyncio.gather(
            oaworks.fetch_permissions(doi, ror=oaworks.KU_ROR),
            asyncio.to_thread(khazna_status_for, doi),
        )
        green = oaworks.parse_permissions(permissions)

        if green is None:
            green = oaworks.not_confirmed(
                li.journal_name, li.issn,
                reason="no permission record for this DOI in OA.Works")
        else:
            green["repository_action_note"] = await _advisory_note(green, li, lang)

        journal = {
            "name":      li.journal_name or green.get("copyright_name", ""),
            "issn":      li.issn or ((green.get("issuer_issns") or [""])[0]),
            "publisher": li.publisher or green.get("policy_issuer", ""),
            "green_oa":  green,
        }
        enriched = await enrich_journals([journal])

        # Subject repositories, but ONLY when OA.Works says one is permitted.
        # Suggesting arXiv for a journal that forbids subject-repository
        # deposit would be exactly the kind of confident-but-wrong advice this
        # build exists to prevent.
        subject_repos = []
        if any("subject repository" in loc.lower()
               for loc in green.get("_locations", [])):
            sci = lookup_any([enriched[0].get("issn"), li.issn]) or {}
            subject_repos = repositories.suggest_subject_repositories(
                sci.get("areas", ""), sci.get("categories", ""))

        result = {
            "journals": enriched,
            "subject_repositories": subject_repos,
            "repository_recommendation": oaworks.recommendation_from(green),
            "next_actions": oaworks.next_actions_from(green, doi, khazna),
            "global_notes": "",
            "doi": doi,
            "doi_url": doi_utils.doi_url(doi),
            "shareyourpaper_url": oaworks.share_link(doi),
            "khazna_status": khazna,
            "source_mode": "doi",
            "khazna": KHAZNA_DEPOSIT_CARD,
        }
        return {"result": result}

    # ══ PATH B — no DOI: journal-level, explicitly UNVERIFIED ═══════════════
    if not li.journal_name:
        raise HTTPException(422, "Provide either an article DOI/URL or a journal name.")

    client = get_client()
    issn = doi_utils.normalise_issn(li.issn) or li.issn
    user_msg = (
        f"{lang}Explain what this author should check regarding self-archiving. "
        f"Return JSON only. You have NO policy database — do not state facts.\n\n"
        f"Journal name: {li.journal_name}\nISSN: {issn or 'Not provided'}\n"
        f"Publisher: {li.publisher or 'Not provided'}\nVersion: {li.manuscript_version}\n"
        f"Funder: {li.funder or 'None'}\nIntended repository: {li.intended_repository}\n"
        f"Intended licence: {li.intended_licence}\nNotes: {li.notes or 'None'}"
    )
    parsed = await call_openai(client, LICENSE_SYSTEM_PROMPT, user_msg,
                               max_tokens=1200, model=MODEL_LIGHT)
    if "error" in parsed:
        raise HTTPException(500, parsed["error"])

    # HARD GUARD: the model has no database, so it may never claim confirmation.
    verify = "https://openpolicyfinder.jisc.ac.uk/" + (f"search?term={issn}" if issn else "")
    for j_ in (parsed.get("journals") or []):
        g = j_.setdefault("green_oa", {})
        g["policy_status"] = "Not confirmed"
        g["risk_flag"] = ("Journal-level estimate only — not verified against a policy "
                          "database. Enter your article's DOI for a confirmed answer.")
        g["evidence_note"] = ("No verified permission record was retrieved. "
                              f"Verify at {verify}")
        g["_verify_url"] = verify
        if not j_.get("issn"):
            j_["issn"] = issn

    if isinstance(parsed.get("journals"), list):
        parsed["journals"] = await enrich_journals(parsed["journals"])

    rec = parsed.setdefault("repository_recommendation", {})
    rec["manual_checks_required"] = (rec.get("manual_checks_required") or []) + [
        "Verify the policy at Open Policy Finder before depositing."]
    parsed["khazna_status"] = {"checked": False, "reason": "no DOI supplied"}
    parsed["source_mode"] = "journal"
    parsed["khazna"] = KHAZNA_DEPOSIT_CARD
    return {"result": parsed}


@app.post("/api/find-repository")
async def find_repository(req: RepoRequest, request: Request):
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests. Please wait a moment and try again.")
    client = get_client()
    lang = lang_instruction(req.language)
    user_msg = (
        f"{lang}Select suitable repositories for this dataset FROM THE REGISTRY BELOW. "
        f"Return JSON only. Use the exact \"id\" values. Do not invent repositories "
        f"and do not state factual repository attributes.\n\n"
        f"AVAILABLE REGISTRY:\n{registry_for_prompt()}\n\n"
        f"DATASET INPUT:\n{json.dumps(req.dataset.dict(), indent=2)}"
    )
    parsed = await call_openai(client, REPO_SYSTEM_PROMPT, user_msg, max_tokens=2000)
    if "error" in parsed:
        raise HTTPException(500, parsed["error"])

    # The model selected ids and wrote fit_reasons; the SERVER supplies every
    # factual field. Unknown ids are silently dropped.
    repos = hydrate_selection(parsed.get("repositories", []))

    # Khazna always leads, and is never a model choice.
    repos = [r for r in repos if not r.get("is_khazna")]
    repos.insert(0, repositories.khazna_entry())

    for r in repos:
        if not r.get("is_khazna"):
            clean_name = re.sub(r"[^\w\s]", "", r.get("name", ""))
            r["verify_links"] = {
                "re3data":     f"https://www.re3data.org/search?query={clean_name}",
                "fairsharing": f"https://fairsharing.org/search?q={clean_name}",
            }

    parsed["repositories"] = repos
    parsed["registry_note"] = (
        "Repositories are selected from a registry curated by KU Library. "
        "Factual details are served from verified records, not generated."
    )
    return {"result": parsed}


@app.post("/api/browse-subject-debug")
async def browse_subject_debug(req: SubjectRequest):
    """Diagnostic: step through browse-subject and report where it fails."""
    import traceback
    results = {}
    try:
        client = get_client()
        results["s1"] = "client OK"

        parsed = await call_openai(
            client, SUBJECT_NORMALISE_PROMPT,
            f"Subject input: {req.subject.strip()}"
        )
        results["s2"] = f"LLM OK: conf={parsed.get('confidence')} norm={parsed.get('normalised','')[:40]}"

        category = parsed.get("scimago_category", "")
        area     = parsed.get("scimago_area", "")
        journals = search_by_subject(category, area, top_n=5)
        results["s3"] = f"search OK: {len(journals)} journals"

        if journals:
            j = journals[0]
            issn = j.get("issn")
            vl = build_verify_links(j["title"], issn, j.get("sourceid",""))
            results["s4"] = f"build_verify_links OK"
            covered = is_ku_apc_covered(j.get("publisher",""), j.get("quartile",""))
            results["s5"] = f"is_ku_apc_covered OK: {covered}"

        return {"ok": True, "results": results}
    except Exception as e:
        results["error"] = str(e)
        results["traceback"] = traceback.format_exc()[-500:]
        return {"ok": False, "results": results}


@app.post("/api/browse-subject")
async def browse_subject(req: SubjectRequest):
    # Guard: ensure SCImago index is loaded (cold-start protection)
    from scimago_index import _subject_cache, _scimago_cache
    if not _subject_cache or not _scimago_cache:
        load_scimago()
        load_subject_index()

    client = get_client()
    try:
        parsed = await call_openai(
            client,
            SUBJECT_NORMALISE_PROMPT,
            f"{lang_instruction(req.language)}Subject input: {req.subject.strip()}",
            max_tokens=200,
            model=MODEL_LIGHT,
        )
    except RateLimitError:
        raise HTTPException(429, "OpenAI rate limit reached. Please try again in a moment.")
    except APIStatusError as e:
        logger.error(f"[BrowseSubject] OpenAI API error: {e}")
        raise HTTPException(502, f"AI service error: {e.status_code}")
    except Exception as e:
        logger.error(f"[BrowseSubject] Unexpected error: {e}")
        raise HTTPException(500, f"Error processing subject: {str(e)}")

    confidence = parsed.get("confidence", "not_found")
    normalised  = parsed.get("normalised", req.subject)
    category    = parsed.get("scimago_category", "")
    area        = parsed.get("scimago_area", "")
    user_message= parsed.get("user_message")

    if confidence == "not_found":
        return {"result": {
            "confidence": "not_found", "normalised": normalised,
            "user_message": user_message or f"Could not identify a valid academic subject for '{req.subject}'.",
            "journals": [],
        }}

    journals = search_by_subject(category, area, top_n=30)
    fallback_used = False
    if not journals and area:
        journals = search_by_subject("", area, top_n=30)
        fallback_used = True

    if not journals:
        return {"result": {
            "confidence": "not_found", "normalised": normalised,
            "user_message": f"No journals found for '{normalised}'. Try a broader term.",
            "journals": [],
        }}

    enriched = []
    for j in journals:
        issn     = j.get("issn")
        sourceid = j.get("sourceid", "")
        vl = build_verify_links(j["title"], issn, sourceid)
        q  = j.get("quartile", "")
        enriched.append({
            "name":        j["title"],
            "issn":        issn or "",
            "quartile":    q if q and q != "-" else None,
            "h_index":     j.get("h_index"),
            "publisher":   j.get("publisher", ""),
            "open_access": j.get("open_access", "No"),
            "country":     j.get("country", ""),
            "categories":  j.get("categories", ""),
            "verify_links": vl,
            "ku_apc_covered": is_ku_apc_covered(j.get("publisher", ""), j.get("quartile", "")),
        })

    msg = user_message
    if fallback_used and not msg:
        msg = f"Exact category not found. Showing top journals in the broader '{area}' area."

    return {"result": {
        "confidence":       confidence,
        "normalised":       normalised,
        "scimago_category": category,
        "scimago_area":     area,
        "user_message":     msg,
        "journals":         enriched,
        "total":            len(enriched),
    }}


@app.post("/api/export-citations")
async def export_citations(req: ExportRequest):
    fmt = req.format.lower()
    journals = req.journals or []

    if fmt == "bibtex":
        lines = []
        for i, j in enumerate(journals):
            key = re.sub(r"[^a-zA-Z0-9]", "", j.get("name", f"journal{i}"))[:20]
            lines.append(
                f"@article{{{key},\n"
                f"  title     = {{{j.get('name', '')}}},\n"
                f"  journal   = {{{j.get('name', '')}}},\n"
                f"  publisher = {{{j.get('publisher', '')}}},\n"
                f"  issn      = {{{j.get('issn', '')}}},\n"
                f"  note      = {{Quartile: {j.get('quartile') or (j.get('ranking') or {}).get('quartile', '—')}. Recommended by ResearchBee.}}\n}}"
            )
        return {"content": "\n\n".join(lines), "filename": "journals.bib", "mimetype": "text/plain"}

    elif fmt == "ris":
        lines = []
        for j in journals:
            q = j.get("quartile") or (j.get("ranking") or {}).get("quartile", "")
            lines.append(
                f"TY  - JOUR\nTI  - {j.get('name', '')}\n"
                f"PB  - {j.get('publisher', '')}\nSN  - {j.get('issn', '')}\n"
                f"N1  - Quartile: {q}. Recommended by ResearchBee.\nER  - "
            )
        return {"content": "\n\n".join(lines), "filename": "journals.ris", "mimetype": "text/plain"}

    raise HTTPException(400, "Format must be bibtex or ris")


@app.post("/api/cover-letter")
async def generate_cover_letter(req: CoverLetterRequest):
    client = get_client()
    lang = "Respond in Arabic." if req.language == "arabic" else "Respond in English."
    user_msg = (
        f"{lang}\n\nGenerate a cover letter for the following submission:\n\n"
        f"Manuscript title: {req.manuscript_title}\nAbstract: {req.abstract}\n"
        f"Article type: {req.article_type}\nTarget journal: {req.journal_name}\n"
        f"Publisher: {req.publisher or 'Not specified'}\nDiscipline: {req.discipline or 'Not specified'}\n"
        f"Author name: {req.author_name or 'The Author(s)'}\n\nReturn JSON only."
    )
    parsed = await call_openai(client, COVER_LETTER_PROMPT, user_msg, max_tokens=800, model=MODEL_LIGHT)
    if "error" in parsed:
        raise HTTPException(500, parsed["error"])
    return {"result": parsed}




# ── Chat endpoint — streaming SSE ─────────────────────────────────────────
@app.post("/api/chat")
async def chat(req: ChatRequest, request: Request):
    """
    Three-layer resolution with streaming:
    1. Keyword pre-filter — instant routing, no LLM call
    2. Open Policy Finder — live policy lookup for specific journals
    3. LLM stream — tokens stream via SSE; route extracted from full reply at end
    """
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "chat"):
        raise HTTPException(429, "Chat limit reached. Please wait a minute before sending more messages.")

    if not req.messages:
        async def empty_stream():
            greeting = "Hi! How can I help you with publishing or Open Access today?"
            yield f'data: {json.dumps({"token": greeting, "done": False})}\n\n'
            yield f'data: {json.dumps({"token": "", "done": True, "reply": greeting, "route": None, "prefill": {}})}\n\n'
        return StreamingResponse(empty_stream(), media_type="text/event-stream",
                                 headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})

    last_msg = req.messages[-1].content.strip()

    # ── Layer 1: keyword pre-filter — instant, no streaming needed ────────
    fast_route = quick_route(last_msg)
    if fast_route:
        TAB_DESCRIPTIONS = {
            "journals": "Journal Submission tool",
            "license":  "License Checking tool",
            "data":     "Data Repository tool",
        }
        reply = (
            f"The {TAB_DESCRIPTIONS[fast_route]} above is perfect for this! "
            f"It'll give you accurate, structured results in seconds."
        )
        if req.language == "arabic":
            reply = f"أداة {TAB_DESCRIPTIONS[fast_route]} المتاحة أعلاه مناسبة تماماً لهذا! ستمنحك نتائج دقيقة ومنظمة في ثوانٍ."
        # Return as SSE with route
        async def fast_stream():
            payload = json.dumps({"token": reply, "done": False})
            yield f"data: {payload}\n\n"
            done = json.dumps({"token": "", "done": True, "route": fast_route, "prefill": {}})
            yield f"data: {done}\n\n"
        return StreamingResponse(fast_stream(), media_type="text/event-stream",
                                 headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})

    # ── Layer 2: Open Policy Finder ───────────────────────────────────────
    opf_data = ""
    journal_name = detect_journal_policy_question(last_msg)
    if journal_name:
        opf_data = await fetch_opf_policy(journal_name)

    # ── Layer 3: LLM streaming ────────────────────────────────────────────
    client = get_client()

    system = CHAT_SYSTEM_PROMPT_BASE.replace(
        "{KU_KNOWLEDGE}", get_knowledge()
    ).replace(
        "{OPF_DATA}", opf_data if opf_data else "No Open Policy Finder data retrieved for this query."
    )

    if req.language == "arabic":
        system += "\n\nRespond entirely in Arabic."

    messages = [{"role": "system", "content": system}]
    messages += [{"role": m.role, "content": m.content} for m in req.messages]

    def extract_route(text: str):
        """Brace-counting extraction of routing JSON from end of reply."""
        starts = [m.start() for m in re.finditer(r'\{', text)]
        for start in reversed(starts):
            candidate = text[start:]
            depth = 0; end = None
            for i, ch in enumerate(candidate):
                if ch == '{': depth += 1
                elif ch == '}':
                    depth -= 1
                    if depth == 0: end = i + 1; break
            if end is None: continue
            try:
                parsed = json.loads(candidate[:end])
                if "route" in parsed:
                    return text[:start].strip(), parsed.get("route"), parsed.get("prefill", {})
            except Exception:
                continue
        return text.strip(), None, {}

    # Create streaming response outside generator to catch connection errors early
    try:
        openai_stream = await client.chat.completions.create(
            model=MODEL,
            messages=messages,
            max_tokens=600,
            temperature=0.3,
            stream=True,
        )
    except Exception as e:
        logger.error(f"[Chat] OpenAI call failed: {e}")
        raise HTTPException(500, f"AI service error: {str(e)}")

    async def stream_response() -> AsyncGenerator[str, None]:
        full_text = ""
        try:
            async for chunk in openai_stream:
                token = chunk.choices[0].delta.content if chunk.choices else None
                if token:
                    full_text += token
                    yield f'data: {json.dumps({"token": token, "done": False})}\n\n'

            reply, route, prefill = extract_route(full_text)
            if route not in ("journals", "license", "data", None):
                route = None

            done_payload = json.dumps({
                "token": "", "done": True,
                "reply": reply,
                "route": route,
                "prefill": prefill or {},
            })
            yield f"data: {done_payload}\n\n"

        except Exception as e:
            logger.error(f"[Chat stream] {e}")
            err = json.dumps({"token": "⚠️ Something went wrong. Please try again.", "done": True, "route": None, "prefill": {}})
            yield f"data: {err}\n\n"

    return StreamingResponse(
        stream_response(),
        media_type="text/event-stream",
        headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
    )


# ── Journal meta lookup — multi-suggestion with ISSN auto-fill ───────────
@app.get("/api/lookup-journal-meta")
async def lookup_journal_meta(request: Request, name: str):
    """
    Multi-suggestion journal lookup:
    Layer 1a: SCImago — up to 4 ranked title matches (name + ISSN + publisher)
    Layer 1b: OpenAlex — up to 4 results, merged + deduplicated with SCImago
    Layer 2:  LLM — only if nothing found; returns up to 3 name suggestions only
              (NO ISSN from LLM — those come from re-running Layer 1 on acceptance)
    """
    ip = get_client_ip(request)
    if not check_rate_limit(ip, "global"):
        raise HTTPException(429, "Too many requests.")

    name = name.strip()
    if not name or len(name) < 3:
        return {"suggestions": [], "llm_suggestions": []}

    name_lower = name.lower()
    words      = [w for w in name_lower.split() if len(w) > 2]

    def score_title(title: str) -> float:
        """Score a title against the query — higher = better match."""
        t = title.lower()
        if t == name_lower:
            return 1.0
        # Word overlap score
        overlap = sum(1 for w in words if w in t)
        # Bonus for starts-with
        starts  = 0.2 if t.startswith(name_lower[:6]) else 0
        # Penalty for very different lengths
        len_ratio = min(len(name_lower), len(t)) / max(len(name_lower), len(t))
        return (overlap / max(len(words), 1)) * 0.6 + starts + len_ratio * 0.2

    # ── Layer 1a: SCImago — collect top 4 scored matches ─────────────────
    rows = load_subject_index()
    scored = []
    for r in rows:
        s = score_title(r["title"])
        if s > 0.35:   # threshold — avoid noise
            scored.append((s, r))
    scored.sort(key=lambda x: -x[0])
    top_scimago = scored[:4]

    suggestions = []
    seen_titles = set()

    for score, r in top_scimago:
        t = r["title"]
        if t.lower() in seen_titles:
            continue
        seen_titles.add(t.lower())
        ni = norm_issn(r.get("issn", ""))
        suggestions.append({
            "title":      t,
            "issn":       ni or r.get("issn", ""),
            "publisher":  r.get("publisher", ""),
            "source":     "SCImago",
            "confidence": "high" if score > 0.8 else "medium",
            "score":      round(score, 3),
        })

    # ── Layer 1b: OpenAlex — up to 4, merge with SCImago results ─────────
    try:
        oa_client = openalex._get_client()
        r = await oa_client.get(
            "https://api.openalex.org/sources",
            params={"search": name, "per_page": 4, "filter": "type:journal"},
            timeout=6.0,
        )
        if r.is_success:
            for s in r.json().get("results", []):
                title     = s.get("display_name", "")
                if not title or title.lower() in seen_titles:
                    continue
                issns     = s.get("issn", [])
                issn_l    = s.get("issn_l", "")
                publisher = s.get("host_organization_name", "")
                best_issn = issn_l or (issns[0] if issns else "")
                ni        = norm_issn(best_issn)
                sc        = score_title(title)
                seen_titles.add(title.lower())
                suggestions.append({
                    "title":      title,
                    "issn":       ni or best_issn,
                    "publisher":  publisher,
                    "source":     "OpenAlex",
                    "confidence": "high" if sc > 0.8 else "medium",
                    "score":      round(sc, 3),
                })
    except Exception as e:
        logger.warning(f"[LookupJournal] OpenAlex error: {e}")

    # Re-sort merged list by score, keep top 4
    suggestions.sort(key=lambda x: -x.get("score", 0))
    suggestions = suggestions[:4]

    # If we have good suggestions, return them
    if suggestions:
        return {"suggestions": suggestions, "llm_suggestions": []}

    # ── Layer 2: LLM — name suggestions only, NO ISSN ────────────────────
    llm_suggestions = []
    try:
        client = get_client()
        resp = await client.chat.completions.create(
            model=MODEL,
            response_format={"type": "json_object"},
            messages=[
                {
                    "role": "system",
                    "content": (
                        "You are an academic journal name expert. "
                        "The user has typed a journal name that may be misspelled or abbreviated. "
                        "Return ONLY valid JSON: "
                        "{suggestions: [{name: string, confidence: high|medium|low}]} "
                        "Return up to 3 most likely correct journal names, ranked by confidence. "
                        "Only include journals you are confident exist. "
                        "Do NOT return ISSN, publisher, or any other fields. "
                        "Return JSON only."
                    ),
                },
                {"role": "user", "content": f"Journal name typed by user: {name}"},
            ],
            max_tokens=150,
            temperature=0.1,
        )
        raw    = resp.choices[0].message.content or "{}"
        parsed = json.loads(raw)
        for s in parsed.get("suggestions", [])[:3]:
            n = s.get("name", "").strip()
            c = s.get("confidence", "low")
            if n and n.lower() != name_lower and c in ("high", "medium"):
                llm_suggestions.append({"name": n, "confidence": c})
    except Exception as e:
        logger.warning(f"[LookupJournal] LLM error: {e}")

    return {"suggestions": [], "llm_suggestions": llm_suggestions}


# ── Static frontend ────────────────────────────────────────────────────────
@app.get("/")
async def root():
    if os.path.exists("static/index.html"):
        return FileResponse("static/index.html")
    return {"status": "ResearchBee API running", "docs": "/docs"}


if os.path.exists("static"):
    if os.path.exists("static/css"):
        app.mount("/css", StaticFiles(directory="static/css"), name="css")
    if os.path.exists("static/js"):
        app.mount("/js", StaticFiles(directory="static/js"), name="js")