src/streamlit_app.py

import os
import json
import uuid
import gzip
import tempfile
import datetime as dt
import math
import hashlib
import base64
import hmac
from typing import List, Dict, Any, Optional, Tuple, Set
from pathlib import Path

import pandas as pd
import streamlit as st
from huggingface_hub import hf_hub_download, HfApi

# =========================
# ENV CONFIG (HF Space Variables / Secrets)
# =========================
PUBLICATIONS_REPO = os.environ.get("PUBLICATIONS_REPO", "").strip()
REVIEWS_REPO = os.environ.get("REVIEWS_REPO", "").strip()
HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HF_API_TOKEN")
ALLOW_CREATE_REVIEWS_REPO = (os.environ.get("ALLOW_CREATE_REVIEWS_REPO", "0").strip().lower() in ("1","true","yes","y","on"))
REVIEWS_PRIVATE = (os.environ.get("REVIEWS_PRIVATE", "1").strip().lower() in ("1","true","yes","y","on"))

# =========================
# AUTH (простая роль/логин)
# =========================
USERS_REPO = os.environ.get("USERS_REPO", "").strip()  # приватный dataset с users.json
USERS_FILE_PATH = os.environ.get("USERS_FILE_PATH", "users.json").strip()
DEMO_LOGIN = os.environ.get("DEMO_LOGIN", "demo").strip()
DEMO_PASSWORD = os.environ.get("DEMO_PASSWORD", "demo")


# Надёжность по умолчанию: сразу пишем каждый review
BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "1"))

PUB_DIR_REGISTRY_PATH = os.environ.get("PUB_DIR_REGISTRY_PATH", "dir_registry.json")
PUB_CANDIDATES_PREFIX = os.environ.get("PUB_CANDIDATES_PREFIX", "candidates")
REVIEWS_LOG_PREFIX = os.environ.get("REVIEWS_LOG_PREFIX", "reviews_log")

# Паттерны поиска candidates файлов (поддержка разных схем данных)
# Можно переопределить в Space Variables:
#   PUB_CANDIDATES_PATTERNS="{prefix}/{dir_id}_top500.jsonl|{prefix}/{dir_id}.jsonl|manual_candidates/{dir_id}.jsonl|manual_candidates/{dir_id}_top500.jsonl"
PUB_CANDIDATES_PATTERNS = os.environ.get(
    "PUB_CANDIDATES_PATTERNS",
    "{prefix}/{dir_id}_top500.jsonl|{prefix}/{dir_id}_top500.jsonl.gz|{prefix}/{dir_id}.jsonl|{prefix}/{dir_id}.jsonl.gz|manual_candidates/{dir_id}.jsonl|manual_candidates/{dir_id}.jsonl.gz"
).strip()


api = HfApi(token=HF_TOKEN)

st.set_page_config(page_title="Скоринг публикаций", layout="wide")


st.markdown(
    """
<style>
/* Global typography tuning (Streamlit) */
h1 { font-size: 1.75rem; margin-bottom: 0.4rem; }
h2 { font-size: 1.35rem; margin-top: 1.0rem; margin-bottom: 0.35rem; }
h3 { font-size: 1.1rem; margin-top: 0.9rem; margin-bottom: 0.25rem; }
.small-meta { font-size: 0.85rem; opacity: 0.85; }
.dir-card { padding: 0.75rem 0.9rem; border: 1px solid rgba(49,51,63,0.2); border-radius: 12px; background: rgba(49,51,63,0.04); }
.kpi-line { font-size: 0.9rem; opacity: 0.9; }
.kpi-line b { opacity: 1.0; }
</style>
    """,
    unsafe_allow_html=True,
)


# =========================
# Helpers
# =========================

def check_dataset_repo(repo_id: str, token: Optional[str]) -> Tuple[bool, str]:
    """
    Проверяет существование датасет-репозитория (repo_type="dataset").
    Возвращает (ok, message). Если ok=False, message содержит причину.
    """
    try:
        api_local = HfApi(token=token) if token else HfApi()
        _ = api_local.repo_info(repo_id=repo_id, repo_type="dataset")
        return True, "OK"
    except Exception as e:
        msg = str(e)
        # huggingface_hub кидает 404/Repository Not Found
        return False, msg

def maybe_create_reviews_repo(repo_id: str) -> Tuple[bool, str]:
    """
    Пытается создать reviews dataset repo, если включён ALLOW_CREATE_REVIEWS_REPO.
    Возвращает (ok, message).
    """
    if not ALLOW_CREATE_REVIEWS_REPO:
        return False, "ALLOW_CREATE_REVIEWS_REPO=0"
    if not HF_TOKEN:
        return False, "HF_TOKEN отсутствует (нужен для создания репозитория)"
    try:
        api.create_repo(
            repo_id=repo_id,
            repo_type="dataset",
            private=bool(REVIEWS_PRIVATE),
            exist_ok=True,
        )
        return True, "created_or_exists"
    except Exception as e:
        return False, str(e)

def require_env(name: str, value: str) -> None:
    if not value:
        st.error(f"Не задано **{name}**. Укажи в Space Settings → Variables.")
        st.stop()

def safe_int(x) -> Optional[int]:
    try:
        if x is None or pd.isna(x):
            return None
        return int(float(x))
    except Exception:
        return None

def safe_float(x) -> Optional[float]:
    try:
        if x is None or pd.isna(x):
            return None
        return float(x)
    except Exception:
        return None

def doi_url(doi: Optional[str]) -> Optional[str]:
    if not doi:
        return None
    doi = str(doi).strip()
    if doi.startswith("http://") or doi.startswith("https://"):
        return doi
    return f"https://doi.org/{doi}"

def normalize_dir_id(dir_id: str, pad2: bool) -> str:
    """
    Поддержка форматов DIR1..DIR14 и DIR01..DIR14.
    pad2=True  -> DIR01
    pad2=False -> DIR1
    """
    if not isinstance(dir_id, str):
        return str(dir_id)
    s = dir_id.strip().upper()
    if not s.startswith("DIR"):
        return dir_id.strip()
    tail = s[3:]
    try:
        n = int(tail)
    except Exception:
        # DIRXX (не число) — как есть
        return dir_id.strip()
    return f"DIR{n:02d}" if pad2 else f"DIR{n}"

def dir_variants(dir_id: str) -> List[str]:
    """
    Список вариантов идентификатора DIR, чтобы подхватить разные имена файлов/папок.
    """
    if not isinstance(dir_id, str):
        return [str(dir_id)]
    a = dir_id.strip()
    b = normalize_dir_id(a, pad2=True)
    c = normalize_dir_id(a, pad2=False)
    out = []
    for x in [a, b, c]:
        if x and x not in out:
            out.append(x)
    return out

def dir_no(dir_id: str) -> str:
    if isinstance(dir_id, str) and dir_id.upper().startswith("DIR"):
        return dir_id[3:]
    return str(dir_id)

def join_terms(term_list: Any) -> str:
    if not term_list:
        return "—"
    out = []
    for x in term_list:
        if isinstance(x, dict):
            t = x.get("t")
            if t:
                out.append(str(t))
        else:
            out.append(str(x))
    return "; ".join(out) if out else "—"

def topics_line(topics: Any) -> str:
    if not topics:
        return "—"
    out = []
    for t in (topics[:3] if isinstance(topics, list) else []):
        if isinstance(t, dict):
            tid = t.get("topic_id") or t.get("id")
            name = t.get("topic_name") or t.get("display_name")
            if tid and name:
                out.append(f"{tid} — {name}")
            elif name:
                out.append(str(name))
            elif tid:
                out.append(str(tid))
        else:
            out.append(str(t))
    return "; ".join(out) if out else "—"

def _b64u(b: bytes) -> str:
    return base64.urlsafe_b64encode(b).decode("utf-8").rstrip("=")

def _b64u_dec(s: str) -> bytes:
    pad = "=" * (-len(s) % 4)
    return base64.urlsafe_b64decode((s + pad).encode("utf-8"))

def hash_password_pbkdf2(password: str, *, iterations: int = 200_000) -> str:
    salt = os.urandom(16)
    dk = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations)
    return f"pbkdf2_sha256${iterations}${_b64u(salt)}${_b64u(dk)}"

def verify_password(record: Dict[str, Any], password: str) -> bool:
    # 1) PBKDF2 string
    ph = record.get("password_hash")
    if isinstance(ph, str) and ph.startswith("pbkdf2_sha256$"):
        try:
            _, it_s, salt_s, hash_s = ph.split("$", 3)
            it = int(it_s)
            salt = _b64u_dec(salt_s)
            expected = _b64u_dec(hash_s)
            got = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, it)
            return hmac.compare_digest(got, expected)
        except Exception:
            return False

    # 2) structured hash
    if isinstance(ph, dict) and ph.get("algo") == "pbkdf2_sha256":
        try:
            it = int(ph.get("iterations") or 200_000)
            salt = _b64u_dec(str(ph.get("salt") or ""))
            expected = _b64u_dec(str(ph.get("hash") or ""))
            got = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, it)
            return hmac.compare_digest(got, expected)
        except Exception:
            return False

    # 3) plaintext (не рекомендуется, но поддерживаем для простого старта)
    pw = record.get("password")
    if isinstance(pw, str):
        return hmac.compare_digest(pw, password)

    return False

@st.cache_data(show_spinner=False)
def load_users(repo_id: str, relpath: str) -> Dict[str, Dict[str, Any]]:
    if not repo_id:
        return {}
    try:
        path = hf_hub_download(repo_id=repo_id, filename=relpath, repo_type="dataset", token=HF_TOKEN)
        data = json.loads(Path(path).read_text(encoding="utf-8"))
    except Exception:
        return {}
    users: Dict[str, Dict[str, Any]] = {}
    items = data.get("users") if isinstance(data, dict) else None
    if isinstance(items, list):
        for u in items:
            if not isinstance(u, dict):
                continue
            login = str(u.get("login") or "").strip()
            if login:
                users[login] = u
    return users

def ensure_auth() -> Dict[str, str]:
    """
    Возвращает dict {login, role}. Если не авторизован — показывает форму входа и останавливает выполнение.
    """
    if isinstance(st.session_state.get("auth"), dict):
        return st.session_state["auth"]

    users = load_users(USERS_REPO, USERS_FILE_PATH)

    st.markdown("### Вход")

    with st.form("login_form", clear_on_submit=False):
        login = st.text_input("Логин").strip()
        password = st.text_input("Пароль", type="password")
        ok = st.form_submit_button("Войти")

    if ok:
        # Demo (только через ввод логина/пароля, без отдельной кнопки)
        if login == DEMO_LOGIN and hmac.compare_digest(password, str(DEMO_PASSWORD)):
            st.session_state["auth"] = {"login": DEMO_LOGIN, "role": "demo"}
            st.rerun()

        rec = users.get(login)
        if rec and verify_password(rec, password):
            role = str(rec.get("role") or "reviewer").strip().lower()
            if role not in ("admin", "reviewer", "demo"):
                role = "reviewer"
            st.session_state["auth"] = {"login": login, "role": role}
            st.rerun()

        st.error("Неверный логин или пароль.")

    st.stop()


def decode_abstract(abstract_inverted_index: Optional[dict]) -> str:
    if not abstract_inverted_index or not isinstance(abstract_inverted_index, dict):
        return ""
    pos_to_word = {}
    for w, positions in abstract_inverted_index.items():
        if not isinstance(positions, list):
            continue
        for p in positions:
            pos_to_word[p] = w
    return " ".join(pos_to_word[p] for p in sorted(pos_to_word.keys())) if pos_to_word else ""

def get_authors_and_abstract(row: Dict[str, Any]) -> Tuple[str, str]:
    """
    Offline-first: берём авторов/аннотацию ТОЛЬКО из candidates JSONL.
    Никаких обращений к OpenAlex API.
    """
    # authors
    authors = row.get('authors') or row.get('author_names') or row.get('authors_str')
    if not authors:
        # Иногда candidates могут содержать OpenAlex-подобный authroships, но уже на входе (без сети)
        auths = row.get('authorships')
        if isinstance(auths, list):
            names = []
            for a in auths:
                if not isinstance(a, dict):
                    continue
                an = ((a.get('author') or {}).get('display_name') or '').strip()
                if an:
                    names.append(an)
            if names:
                authors = ', '.join(names[:12]) + (', и др.' if len(names) > 12 else '')
    if isinstance(authors, list):
        authors = ', '.join([str(a) for a in authors if a])
    authors_str = authors.strip() if isinstance(authors, str) and authors.strip() else ''

    # abstract
    abstract = row.get('abstract') or row.get('abstract_text')
    if not abstract and isinstance(row.get('abstract_inverted_index'), dict):
        abstract = decode_abstract(row.get('abstract_inverted_index'))
    abstract_str = abstract.strip() if isinstance(abstract, str) and abstract.strip() else ''

    return authors_str or '—', abstract_str or '—'

def open_jsonl_any(path: str) -> List[Dict[str, Any]]:
    rows = []
    if path.endswith(".gz"):
        with gzip.open(path, "rt", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    rows.append(json.loads(line))
    else:
        with open(path, "r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    rows.append(json.loads(line))
    return rows

def normalize_candidate_row(obj: Dict[str, Any], selected_dir: str) -> Dict[str, Any]:
    """
    Приводит разные схемы данных к единому набору полей, используемых UI.
    Поддерживаем:
    - старый формат candidates: work_id/year/cited_by/dir_score/match_score
    - новый формат паспорта: source_id/openalex_work_id/publication_year/cited_by_count/dir_native_components/quality_weight
    """
    out = dict(obj)

    # --- IDs ---
    source_id = out.get("source_id") or out.get("Source ID")
    if isinstance(source_id, str) and source_id.strip():
        out["source_id"] = source_id.strip()

    work_id = out.get("work_id")
    if not isinstance(work_id, str) or not work_id.strip():
        # паспорта
        wid = out.get("openalex_work_id") or out.get("OpenAlex work ID") or out.get("dup_work_id")
        if isinstance(wid, str) and wid.strip():
            work_id = wid.strip()
    out["work_id"] = work_id if isinstance(work_id, str) else None

    # --- Title ---
    title = out.get("title") or out.get("display_name") or out.get("Title")
    out["title"] = str(title).strip() if title else "—"

    # --- DOI ---
    doi = out.get("doi") or out.get("DOI")
    out["doi"] = str(doi).strip() if doi else None

    # --- Abstract (offline) ---
    if out.get("abstract") is None:
        ab = out.get("abstract_text")
        if isinstance(ab, str) and ab.strip():
            out["abstract"] = ab.strip()
        elif isinstance(out.get("abstract_inverted_index"), dict):
            out["abstract"] = decode_abstract(out.get("abstract_inverted_index")) or None


    # --- Year / cited_by ---
    year = out.get("year")
    if year is None:
        year = out.get("publication_year") or out.get("Publication year")
    out["year"] = safe_int(year)

    cited_by = out.get("cited_by")
    if cited_by is None:
        cited_by = out.get("cited_by_count") or out.get("Cited by count")
    out["cited_by"] = safe_int(cited_by)

    # --- Scores ---
    # dir_score: приоритет
    # 1) явное поле dir_score (старый candidates)
    ds = out.get("dir_score")
    # 2) паспорта: dir_native_components.score (если selected_dir совпадает с родным)
    if ds is None:
        comp = out.get("dir_native_components") or {}
        if isinstance(comp, dict) and comp.get("score") is not None:
            ds = comp.get("score")
    # 3) паспорта: dir_scores[selected_dir]
    if ds is None:
        dsd = out.get("dir_scores") or {}
        if isinstance(dsd, dict):
            # пробуем разные варианты dir_id
            for dv in dir_variants(selected_dir):
                if dv in dsd:
                    ds = dsd.get(dv)
                    break
    out["dir_score"] = safe_float(ds)

    # match_score: старое поле; если его нет — используем delta или quality_weight*100 для “второй метрики”
    ms = out.get("match_score")
    if ms is None:
        comp = out.get("dir_native_components") or {}
        if isinstance(comp, dict) and comp.get("delta") is not None:
            ms = comp.get("delta")
        elif out.get("quality_weight") is not None:
            try:
                ms = float(out.get("quality_weight")) * 100.0
            except Exception:
                ms = None
    out["match_score"] = safe_float(ms)

    # quality_weight / components
    qw = out.get("quality_weight")
    out["quality_weight"] = safe_float(qw)
    comp = out.get("dir_native_components")
    out["dir_native_components"] = comp if isinstance(comp, dict) else {}

    # links: pdf/primary location
    out["pdf_url"] = out.get("pdf_url") or out.get("PDF URL")
    out["primary_location_url"] = out.get("primary_location_url") or out.get("Primary location URL")

    return out

def review_key(row: Dict[str, Any]) -> str:
    """
    Ключ "уже оценено":
    - если есть source_id — используем его (если вы теперь опираетесь на source_id)
    - иначе work_id
    """
    sid = row.get("source_id")
    if isinstance(sid, str) and sid.strip():
        return f"SRC::{sid.strip()}"
    wid = row.get("work_id")
    if isinstance(wid, str) and wid.strip():
        return f"W::{wid.strip()}"
    return ""


# =========================
# Publications (read)
# =========================
@st.cache_data(show_spinner=False)
def load_dir_registry(repo_id: str, filename: str) -> List[Dict[str, Any]]:
    path = hf_hub_download(repo_id=repo_id, filename=filename, repo_type="dataset", token=HF_TOKEN)
    with open(path, "r", encoding="utf-8") as f:
        return json.load(f)

@st.cache_data(show_spinner=False)
def load_candidates(repo_id: str, dir_id: str, prefix: str) -> pd.DataFrame:
    """
    Поддержка нескольких паттернов и вариантов DIR id (DIR1 vs DIR01).
    """
    patterns = [p.strip() for p in PUB_CANDIDATES_PATTERNS.split("|") if p.strip()]
    tried = []
    rows = None

    for dv in dir_variants(dir_id):
        for pat in patterns:
            fname = pat.format(prefix=prefix, dir_id=dv)
            tried.append(fname)
            try:
                path = hf_hub_download(repo_id=repo_id, filename=fname, repo_type="dataset", token=HF_TOKEN)
                rows = open_jsonl_any(path)
                if rows:
                    break
                # если файл пустой — тоже считаем найденным
                if rows == []:
                    break
            except Exception:
                continue
        if rows is not None:
            break

    if rows is None:
        raise FileNotFoundError("Не найден candidates файл. Пробовали:\n" + "\n".join(tried[:30]) + ("\n..." if len(tried) > 30 else ""))

    # normalize schema
    normed = [normalize_candidate_row(r, dir_id) for r in rows if isinstance(r, dict)]
    df = pd.DataFrame(normed)

    # numeric columns
    for col in ["dir_score", "match_score", "cited_by", "year", "quality_weight"]:
        if col in df.columns:
            df[col] = pd.to_numeric(df[col], errors="coerce")

    return df


# =========================
# Reviews (read on-demand)
# =========================
@st.cache_data(show_spinner=False)
def list_review_files(repo_id: str, dir_id: str, prefix: str) -> List[str]:
    files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
    # в reviews repo папка может быть DIR1 или DIR01 — подхватываем оба
    needles = [f"/{dv}/" for dv in dir_variants(dir_id)]
    out = []
    for p in files:
        if not (p.startswith(prefix + "/") and p.endswith(".jsonl")):
            continue
        if any(n in p for n in needles):
            out.append(p)
    return out

@st.cache_data(show_spinner=False)
def load_reviewed_keys(repo_id: str, dir_id: str, prefix: str, reviewer: Optional[str] = None) -> Set[str]:
    """
    Поддержка миграции:
    - старые review содержали только work_id -> помечаем как W::<id>
    - новые могут содержать source_id -> помечаем как SRC::<id>
    """
    try:
        files = list_review_files(repo_id, dir_id, prefix)
    except Exception:
        return set()

    reviewed: Set[str] = set()
    for relpath in files:
        try:
            path = hf_hub_download(repo_id=repo_id, filename=relpath, repo_type="dataset", token=HF_TOKEN)
            with open(path, "r", encoding="utf-8") as f:
                for line in f:
                    line = line.strip()
                    if not line:
                        continue
                    obj = json.loads(line)

                    if reviewer and str(obj.get('reviewer') or '').strip() != reviewer:
                        continue

                    sid = obj.get("source_id")
                    if isinstance(sid, str) and sid.strip():
                        reviewed.add(f"SRC::{sid.strip()}")

                    wid = obj.get("work_id")
                    if isinstance(wid, str) and wid.strip():
                        reviewed.add(f"W::{wid.strip()}")
        except Exception:
            continue
    
    return reviewed
@st.cache_data(show_spinner=False)
def load_review_index(repo_id: str, dir_id: str, prefix: str, reviewer: Optional[str] = None) -> Dict[str, Dict[str, Any]]:
    """
    Индекс последних оценок по публикации (для предзаполнения score/comment).
    Возвращает mapping: key -> last_review_obj, где key — SRC::<source_id> или W::<work_id>.
    Если reviewer задан — берём только его записи (для разделения прав).
    """
    try:
        files = list_review_files(repo_id, dir_id, prefix)
    except Exception:
        return {}

    index: Dict[str, Dict[str, Any]] = {}
    ts_index: Dict[str, str] = {}

    def pick_key(obj: Dict[str, Any]) -> Optional[str]:
        sid = obj.get("source_id")
        if isinstance(sid, str) and sid.strip():
            return f"SRC::{sid.strip()}"
        wid = obj.get("work_id")
        if isinstance(wid, str) and wid.strip():
            return f"W::{wid.strip()}"
        return None

    for relpath in files:
        try:
            path = hf_hub_download(repo_id=repo_id, filename=relpath, repo_type="dataset", token=HF_TOKEN)
            with open(path, "r", encoding="utf-8") as f:
                for line in f:
                    line = line.strip()
                    if not line:
                        continue
                    obj = json.loads(line)
                    if reviewer and str(obj.get("reviewer") or "").strip() != reviewer:
                        continue
                    k = pick_key(obj)
                    if not k:
                        continue
                    t = str(obj.get("ts_utc") or obj.get("ts") or "")
                    if t >= ts_index.get(k, ""):
                        ts_index[k] = t
                        index[k] = obj
        except Exception:
            continue
    return index


# =========================
# Reviews (write)
# =========================
def push_batch_to_reviews_repo(repo_id: str, dir_id: str, prefix: str, batch: List[Dict[str, Any]]) -> None:
    if not batch:
        return
    if not HF_TOKEN:
        raise RuntimeError("Нет HF_TOKEN (Secret) — нельзя записывать в reviews dataset.")
    today = dt.date.today().isoformat()
    batch_id = str(uuid.uuid4())

    # Пишем в dir_id как есть (но если dir_id может быть DIR01, а у вас в БД DIR1 — нормализуйте в UI выбором)
    canonical_dir = normalize_dir_id(dir_id, pad2=False)
    path_in_repo = f"{prefix}/{today}/{canonical_dir}/{batch_id}.jsonl"

    with tempfile.NamedTemporaryFile("w", delete=False, encoding="utf-8", suffix=".jsonl") as tmp:
        for row in batch:
            tmp.write(json.dumps(row, ensure_ascii=False) + "\n")
        tmp_path = tmp.name

    api.upload_file(
        path_or_fileobj=tmp_path,
        path_in_repo=path_in_repo,
        repo_id=repo_id,
        repo_type="dataset",
        commit_message=f"add reviews batch {dir_id} {today} ({len(batch)})",
    )


# =========================
# =========================
# MAIN APP (offline-first)
# =========================
auth = ensure_auth()
login = auth.get("login", "anonymous")
role = auth.get("role", "reviewer")

# Sidebar: только пользователь/админ-инструменты
flush_now = False
clear_cache = False

with st.sidebar:
    st.markdown(f"**Пользователь:** {login}<br>**Роль:** {role}", unsafe_allow_html=True)
    if st.button("Выйти", use_container_width=True):
        if "auth" in st.session_state:
            del st.session_state["auth"]
        st.rerun()


    if role != "demo":
        with st.expander("Сервис", expanded=False):
            if st.session_state.get("batch"):
                flush_now = st.button("⬆️ Синхронизировать pending", use_container_width=True)
    if role == "admin":
        with st.expander("Админ", expanded=False):
            clear_cache = st.button("🧹 Сбросить кэш", use_container_width=True)

# Preflight: окружение/доступ к репозиториям
require_env("PUBLICATIONS_REPO", PUBLICATIONS_REPO)
if role != "demo":
    require_env("REVIEWS_REPO", REVIEWS_REPO)

pub_ok, pub_msg = check_dataset_repo(PUBLICATIONS_REPO, HF_TOKEN)
if not pub_ok:
    st.error("PUBLICATIONS_REPO недоступен как dataset repo. Проверь repo_id и доступ.\n\n" + pub_msg)
    st.stop()

if role != "demo":
    rev_ok, rev_msg = check_dataset_repo(REVIEWS_REPO, HF_TOKEN)
    if not rev_ok:
        # создание допускаем только администратору и только если явно разрешено
        if role == "admin":
            created, _ = maybe_create_reviews_repo(REVIEWS_REPO)
            if created:
                rev_ok2, rev_msg2 = check_dataset_repo(REVIEWS_REPO, HF_TOKEN)
                if not rev_ok2:
                    st.error("REVIEWS_REPO недоступен после create_repo.\n\n" + rev_msg2)
                    st.stop()
                else:
                    st.success("REVIEWS_REPO создан/доступен.")
            else:
                st.error("REVIEWS_REPO недоступен (404/нет доступа).\n\n" + rev_msg)
                st.stop()
        else:
            st.error("REVIEWS_REPO недоступен (404/нет доступа).\n\n" + rev_msg)
            st.stop()

# reviewer берём из учётной записи
reviewer = login
reviewer_filter = None if role == 'admin' else reviewer
dirs = load_dir_registry(PUBLICATIONS_REPO, PUB_DIR_REGISTRY_PATH)
dir_ids = [d.get("dir_id") for d in dirs if d.get("dir_id")]
if not dir_ids:
    st.error("В dir_registry.json не найдено ни одного dir_id.")
    st.stop()

dir_map = {d.get("dir_id"): d for d in dirs if d.get("dir_id")}

def _fmt_dir(did: str) -> str:
    meta = dir_map.get(did) or {}
    name = meta.get("dir_name") or "—"
    return f"DIR-{str(dir_no(did)).zfill(2)} — {name}"

st.title("Скоринг публикаций")

# Выбор DIR — в основном интерфейсе (не в сайдбаре)
selected_dir = st.selectbox("Направления исследований", dir_ids, index=0, format_func=_fmt_dir)
dir_meta = dir_map.get(selected_dir) or {}

# =========================
if clear_cache:
    load_dir_registry.clear()
    load_candidates.clear()
    if hasattr(list_review_files, "clear"):
        list_review_files.clear()
    if hasattr(load_reviewed_keys, "clear"):
        load_reviewed_keys.clear()
    if hasattr(load_review_index, "clear"):
        load_review_index.clear()
    for k in [
        "reviewed_remote",
        "reviewed_remote_dir",
        "review_index_remote",
        "review_index_local",
        "reviewed_local_committed",
        "reviewed_local_pending",
        "batch",
    ]:
        if k in st.session_state:
            del st.session_state[k]
    st.toast("Кэш очищен.")
    st.rerun()

# =========================
# DIR header
# =========================
defaults = (dir_meta.get("defaults") or {})
year_from = defaults.get("year_from")
year_to = defaults.get("year_to")

terms = (dir_meta.get("terms") or {})
anchor_str = join_terms(terms.get("anchor"))
support_str = join_terms(terms.get("support"))
noise_str = join_terms(terms.get("noise"))
topics_str = topics_line(dir_meta.get("topics") or [])

dir_desc = dir_meta.get("dir_description", "—")

st.markdown(
    f"""
<div class="dir-card">
  <div class="small-meta"><b>Краткое описание</b></div>
  <div>{dir_desc}</div>
</div>
""",
    unsafe_allow_html=True,
)

with st.expander("Параметры поиска", expanded=True):
    st.markdown(f"**Временной интервал:** {year_from} – {year_to}")
    st.markdown(f"**Якоря:** {anchor_str}")
    st.markdown(f"**Поддержка:** {support_str}")
    st.markdown(f"**Шум:** {noise_str}")
    st.markdown(f"**Topics:** {topics_str}")

st.divider()

# =========================
# Load candidates + fixed sorting (A: dir_score ↓)
# =========================
try:
    df = load_candidates(PUBLICATIONS_REPO, selected_dir, PUB_CANDIDATES_PREFIX)
except Exception as e:
    st.error(f"Не удалось загрузить кандидатов для {selected_dir}: {e}")
    st.stop()

if df.empty:
    st.warning(f"Пустой список кандидатов для {selected_dir}.")
    st.stop()

# Fixed order: dir_score ↓ (stable ties)
sort_cols: List[str] = []
sort_asc: List[bool] = []
if "dir_score" in df.columns:
    sort_cols.append("dir_score"); sort_asc.append(False)
if "quality_weight" in df.columns:
    sort_cols.append("quality_weight"); sort_asc.append(False)
if "match_score" in df.columns:
    sort_cols.append("match_score"); sort_asc.append(False)
if "year" in df.columns:
    sort_cols.append("year"); sort_asc.append(False)
if sort_cols:
    df = df.sort_values(sort_cols, ascending=sort_asc)

df = df.reset_index(drop=True)
total = len(df)
if total == 0:
    st.warning("Пустой список кандидатов.")
    st.stop()

# =========================
# State: batch + local/remote indexes
# =========================
def ensure_state():
    if "batch" not in st.session_state:
        st.session_state["batch"] = []
    if "reviewed_local_committed" not in st.session_state:
        st.session_state["reviewed_local_committed"] = set()
    if "reviewed_local_pending" not in st.session_state:
        st.session_state["reviewed_local_pending"] = set()
    if "reviewed_remote" not in st.session_state:
        st.session_state["reviewed_remote"] = set()
    if "reviewed_remote_dir" not in st.session_state:
        st.session_state["reviewed_remote_dir"] = None
    if "review_index_remote" not in st.session_state:
        st.session_state["review_index_remote"] = {}
    if "review_index_local" not in st.session_state:
        st.session_state["review_index_local"] = {}

ensure_state()

def row_keys_all(r: Dict[str, Any]) -> Set[str]:
    keys = set()
    sid = r.get("source_id")
    if isinstance(sid, str) and sid.strip():
        keys.add(f"SRC::{sid.strip()}")
    wid = r.get("work_id")
    if isinstance(wid, str) and wid.strip():
        keys.add(f"W::{wid.strip()}")
    return keys

canonical_dir = normalize_dir_id(selected_dir, pad2=False)

# Загрузка сохранённых оценок (тихо; remote-статусы не показываем)
need_reload = (st.session_state["reviewed_remote_dir"] != canonical_dir)
if need_reload:
    if role != "demo":
        # Тихо подгружаем только оценки текущего пользователя (или все, если admin)
        st.session_state["reviewed_remote"] = load_reviewed_keys(REVIEWS_REPO, selected_dir, REVIEWS_LOG_PREFIX, reviewer=reviewer_filter)
        st.session_state["review_index_remote"] = load_review_index(REVIEWS_REPO, selected_dir, REVIEWS_LOG_PREFIX, reviewer=reviewer_filter)
    else:
        st.session_state["reviewed_remote"] = set()
        st.session_state["review_index_remote"] = {}
    st.session_state["reviewed_remote_dir"] = canonical_dir

reviewed_committed = set(st.session_state["reviewed_remote"]) | set(st.session_state["reviewed_local_committed"])
reviewed_pending = set(st.session_state["reviewed_local_pending"])
reviewed_effective = reviewed_committed | reviewed_pending

def status_of_row(r: Dict[str, Any]) -> str:
    keys = row_keys_all(r)
    if keys and any(k in reviewed_committed for k in keys):
        return "✅"
    if keys and any(k in reviewed_pending for k in keys):
        return "🕓"
    return "🆕"

def is_reviewed_effective(r: Dict[str, Any]) -> bool:
    keys = row_keys_all(r)
    return bool(keys and any(k in reviewed_effective for k in keys))

def last_review_for_row(r: Dict[str, Any]) -> Optional[Dict[str, Any]]:
    """Берём последнюю известную оценку (pending/local -> remote)."""
    keys = list(row_keys_all(r))
    if not keys:
        return None

    best = None
    best_ts = ""
    # local index first
    for k in keys:
        obj = st.session_state["review_index_local"].get(k)
        if isinstance(obj, dict):
            t = str(obj.get("ts_utc") or obj.get("ts") or "")
            if t > best_ts:
                best_ts = t
                best = obj
    # remote index fallback
    for k in keys:
        obj = st.session_state["review_index_remote"].get(k)
        if isinstance(obj, dict):
            t = str(obj.get("ts_utc") or obj.get("ts") or "")
            if t > best_ts:
                best_ts = t
                best = obj
    return best

# =========================
# Sidebar: statuses (без remote)
# =========================
def split_key_set(keys: Set[str]) -> Tuple[Set[str], Set[str]]:
    src_ids: Set[str] = set()
    w_ids: Set[str] = set()
    for k in keys:
        if isinstance(k, str) and k.startswith("SRC::"):
            src_ids.add(k[5:])
        elif isinstance(k, str) and k.startswith("W::"):
            w_ids.add(k[3:])
    return src_ids, w_ids

comm_src_ids, comm_w_ids = split_key_set(reviewed_committed)
pend_src_ids, pend_w_ids = split_key_set(reviewed_pending)

src_series = df["source_id"].fillna("") if "source_id" in df.columns else pd.Series([""] * len(df))
wid_series = df["work_id"].fillna("") if "work_id" in df.columns else pd.Series([""] * len(df))

mask_committed = src_series.astype(str).isin(comm_src_ids) | wid_series.astype(str).isin(comm_w_ids)
mask_pending = src_series.astype(str).isin(pend_src_ids) | wid_series.astype(str).isin(pend_w_ids)
mask_reviewed = mask_committed | mask_pending

committed_pub_count = int(mask_committed.sum())
pending_pub_count = int(mask_pending.sum())
unreviewed_pub_count = int((~mask_reviewed).sum())


# =========================
# Commit helper
# =========================
def commit_batch() -> Tuple[bool, str]:
    batch = st.session_state.get("batch") or []
    if not batch:
        return False, "batch_empty"
    try:
        push_batch_to_reviews_repo(REVIEWS_REPO, selected_dir, REVIEWS_LOG_PREFIX, batch)
    except Exception as e:
        return False, str(e)

    # Update local sets: pending -> committed (keys)
    for obj in batch:
        sid = obj.get("source_id")
        wid = obj.get("work_id")
        if isinstance(sid, str) and sid.strip():
            k = f"SRC::{sid.strip()}"
            st.session_state["reviewed_local_committed"].add(k)
            st.session_state["reviewed_local_pending"].discard(k)
        if isinstance(wid, str) and wid.strip():
            k = f"W::{wid.strip()}"
            st.session_state["reviewed_local_committed"].add(k)
            st.session_state["reviewed_local_pending"].discard(k)

    st.session_state["batch"] = []
    return True, "ok"

# manual flush
if flush_now and st.session_state.get("batch"):
    with st.spinner("Синхронизирую pending батч…"):
        ok, msg = commit_batch()
    if ok:
        st.success("Pending батч синхронизирован.")
        st.rerun()
    else:
        st.error(f"Не удалось синхронизировать: {msg}")

# recompute after possible commit
reviewed_committed = set(st.session_state["reviewed_remote"]) | set(st.session_state["reviewed_local_committed"])
reviewed_pending = set(st.session_state["reviewed_local_pending"])
reviewed_effective = reviewed_committed | reviewed_pending

# =========================
# Navigation window + layout
# =========================
idx_key = f"idx_{canonical_dir}"
if idx_key not in st.session_state:
    st.session_state[idx_key] = 0

current_idx = int(st.session_state[idx_key])
current_idx = max(0, min(total - 1, current_idx))
st.session_state[idx_key] = current_idx

window_size = 50
window_start = int(current_idx // window_size) * int(window_size)
window_end = min(window_start + int(window_size), total)
page_indices = list(range(window_start, window_end))

def fmt_idx(i: int) -> str:
    r = df.iloc[i].to_dict()
    ico = status_of_row(r)
    year_val = safe_int(r.get("year"))
    year = str(year_val) if year_val is not None else "—"
    title = (str(r.get("title")) if r.get("title") is not None else "—")
    title_short = title if len(title) <= 80 else title[:77] + "…"
    return f"{ico} {i+1:03d} | {year} | {title_short}"

def next_unreviewed(from_idx: int) -> int:
    i = max(0, from_idx)
    while i < total:
        r = df.iloc[i].to_dict()
        if is_reviewed_effective(r):
            i += 1
            continue
        return i
    return total - 1

# main 2-column layout: left=list+scoring, right=publication
col_left, col_right = st.columns([1, 2], gap="large")

with col_left:
    st.markdown("### Список публикаций")
    st.caption(f"Окно: {window_start+1}–{window_end} из {total} • Неоценено: {unreviewed_pub_count}")

    sel = st.selectbox(
        "Публикации в текущем окне",
        options=page_indices,
        index=page_indices.index(current_idx) if current_idx in page_indices else 0,
        format_func=fmt_idx,
        label_visibility="collapsed",
    )

    if int(sel) != current_idx:
        st.session_state[idx_key] = int(sel)
        st.rerun()

    # актуальная строка
    current_idx = int(st.session_state[idx_key])
    row = df.iloc[current_idx].to_dict()

    # предзаполнение score/comment: если есть оценка -> она, иначе 0/пусто
    existing = last_review_for_row(row) or {}
    existing_score = safe_int(existing.get("score"))
    if existing_score is None:
        existing_score = 0
    existing_comment = str(existing.get("comment") or "")

    score_key = f"score_{canonical_dir}_{current_idx}"
    comment_key = f"com_{canonical_dir}_{current_idx}"

    if score_key not in st.session_state:
        st.session_state[score_key] = int(existing_score)
    # защита от старых/некорректных значений
    if int(st.session_state[score_key]) not in (-2, -1, 0, 1, 2):
        st.session_state[score_key] = 0

    if comment_key not in st.session_state:
        st.session_state[comment_key] = existing_comment

  
    label_map = {
        -2: "точно нет",
        -1: "скорее нет",
        0: "не знаю",
        1: "скорее да",
        2: "точно да",
    }

    score = st.radio(
        "Оценка",
        options=[-2, -1, 0, 1, 2],
        format_func=lambda v: f"{v:+d} — {label_map.get(v, '')}".replace("+0", "0"),
        index=[-2, -1, 0, 1, 2].index(int(st.session_state[score_key])),
        key=score_key,
    )

    comment = st.text_area("Комментарий", height=140, key=comment_key)

    b1, b2, b3 = st.columns([1, 2, 1])

    with b1:
        if st.button("⬅️ Назад", use_container_width=True):
            st.session_state[idx_key] = max(0, current_idx - 1)
            st.rerun()

    with b2:
        if st.button("✅ Сохранить и далее", use_container_width=True, disabled=(role=="demo")):
            # собираем мета для review
            authors, abstract = get_authors_and_abstract(row)

            y = safe_int(row.get("year"))
            cites = safe_int(row.get("cited_by"))
            dir_score = safe_float(row.get("dir_score"))
            delta = safe_float(row.get("match_score"))
            qw = safe_float(row.get("quality_weight"))

            work_id = row.get("work_id") if isinstance(row.get("work_id"), str) else None
            source_id = row.get("source_id") if isinstance(row.get("source_id"), str) else None

            review = {
                "ts_utc": dt.datetime.utcnow().isoformat(timespec="seconds") + "Z",
                "dir_id": selected_dir,
                "dir_id_canonical": canonical_dir,
                "source_id": source_id,
                "work_id": work_id,
                "doi": row.get("doi"),
                "title": row.get("title"),
                "authors": authors,
                "abstract": abstract,
                "year": y,
                "cited_by": cites,
                "auto": {
                    "dir_score": dir_score,
                    "delta": delta,
                    "quality_weight": qw,
                    "components": row.get("dir_native_components") or {},
                    "rank": int(current_idx),
                },
                "score": int(score),
                "comment": comment,
                "reviewer": reviewer,
            }

            st.session_state["batch"].append(review)

            # pending keys + local index for prefill
            for k in row_keys_all(row):
                st.session_state["reviewed_local_pending"].add(k)
                st.session_state["review_index_local"][k] = review

            # auto flush by BATCH_SIZE
            flushed = False
            if len(st.session_state["batch"]) >= BATCH_SIZE:
                with st.spinner("Сохраняю в reviews repo…"):
                    ok, msg = commit_batch()
                if ok:
                    flushed = True
                    st.success("Сохранено.")
                else:
                    st.error(f"Не удалось записать в reviews repo: {msg}")
                    st.warning("Оценка сохранена локально как pending. Скачайте pending reviews или повторите синхронизацию.")

            # переход: следующая НЕоценённая (без тумблера)
            nxt = next_unreviewed(current_idx + 1)
            st.session_state[idx_key] = nxt
            st.rerun()

    with b3:
        if st.button("➡️ Далее", use_container_width=True):
            st.session_state[idx_key] = min(total - 1, current_idx + 1)
            st.rerun()

    

    # --- Статусы/прогресс (внизу левого блока) ---
    total_pub = int(total)
    prog_eval = int(committed_pub_count + pending_pub_count)
    prog_pct = int(round(100.0 * prog_eval / total_pub)) if total_pub > 0 else 0
    st.markdown(
        f'<div class="kpi-line">'
        f'<b>Всего</b>: {total_pub} • '
        f'<b>Оценено</b>: {committed_pub_count} • '
        f'<b>Pending</b>: {pending_pub_count} • '
        f'<b>Осталось</b>: {unreviewed_pub_count} • '
        f'<b>{prog_pct}%</b> • '
        f'<b>#{int(current_idx)+1}/{total_pub}</b>'
        f'</div>',
        unsafe_allow_html=True,
    )

    if role != "demo" and (st.session_state.get("batch") or []):
        pending_jsonl = "".join(json.dumps(x, ensure_ascii=False) + "\\n" for x in (st.session_state.get("batch") or []))
        st.download_button(
            "⬇️ Скачать pending (.jsonl)",
            data=pending_jsonl,
            file_name=f"pending_{canonical_dir}_{dt.datetime.utcnow().strftime('%Y%m%d_%H%M%S')}.jsonl",
            mime="application/jsonl",
            use_container_width=True,
        )

with col_right:
    # карточка публикации занимает максимум пространства справа
    row = df.iloc[int(st.session_state[idx_key])].to_dict()

    title = row.get("title") or "—"
    authors, abstract = get_authors_and_abstract(row)

    y = safe_int(row.get("year"))
    cites = safe_int(row.get("cited_by"))
    dir_score = safe_float(row.get("dir_score"))
    delta = safe_float(row.get("match_score"))
    qw = safe_float(row.get("quality_weight"))

    work_id = row.get("work_id") if isinstance(row.get("work_id"), str) else None
    source_id = row.get("source_id") if isinstance(row.get("source_id"), str) else None

    doi_link = doi_url(row.get("doi"))
    pdf_url = row.get("pdf_url")
    pl_url = row.get("primary_location_url")

    oa = row.get("open_access") if isinstance(row.get("open_access"), dict) else {}
    oa_is = oa.get("is_oa") if isinstance(oa, dict) else None
    oa_status = oa.get("oa_status") if isinstance(oa, dict) else None
    oa_url = oa.get("oa_url") if isinstance(oa, dict) else None

    keys = row_keys_all(row)
    committed = bool(keys and any(k in reviewed_committed for k in keys))
    pending = bool(keys and any(k in reviewed_pending for k in keys))
    status_str = "✅ оценено" if committed else ("🕓 pending" if pending else "🆕 не оценено")

    # badges
    has_abs = (isinstance(abstract, str) and abstract.strip() and abstract.strip() != "—")
    has_pdf = (isinstance(pdf_url, str) and isinstance(pdf_url, str) and pdf_url.strip())
    has_pl = (isinstance(pl_url, str) and pl_url.strip())
    has_doi = bool(doi_link)
    badges = [
        f"Abstract {'✅' if has_abs else '❌'}",
        f"PDF {'✅' if has_pdf else '❌'}",
        f"URL {'✅' if has_pl else '❌'}",
        f"DOI {'✅' if has_doi else '❌'}",
    ]
    if oa_is is not None:
        badges.append(f"OA {'✅' if oa_is else '❌'}{(' (' + str(oa_status) + ')') if oa_status else ''}")

    st.markdown(f"## {title}")
    st.caption(status_str + " • " + " | ".join(badges))

    id_lines = []
    if source_id:
        id_lines.append(f"**Source ID:** `{source_id}`")
    if work_id:
        id_lines.append(f"**Work ID:** `{work_id}`")
    if id_lines:
        st.markdown(" • ".join(id_lines))

    # Метрики
    y_s = str(y) if y is not None else "—"
    c_s = str(cites) if cites is not None else "—"
    ds_s = f"{dir_score:.3f}" if dir_score is not None else "—"
    dlt_s = f"{delta:.4f}" if delta is not None else "—"
    qw_s = f"{qw:.3f}" if qw is not None else "—"
    st.markdown(f"**Год:** {y_s}  |  **Цитаты:** {c_s}  |  **dir_score:** {ds_s}  |  **delta:** {dlt_s}  |  **quality:** {qw_s}")

    # Авторы
    if authors and authors != "—":
        st.markdown("**Авторы:**")
        st.write(authors)
    else:
        st.caption("Авторы: — (в исходных DIR*.jsonl авторы не предоставляются)")

    # Ссылки
    links = []
    if work_id:
        links.append(f"[OpenAlex]({work_id})")
    if doi_link:
        links.append(f"[DOI]({doi_link})")
    if isinstance(pdf_url, str) and pdf_url.strip():
        links.append(f"[PDF]({pdf_url.strip()})")
    if isinstance(pl_url, str) and pl_url.strip():
        links.append(f"[Primary URL]({pl_url.strip()})")
    if isinstance(oa_url, str) and oa_url.strip():
        links.append(f"[OA URL]({oa_url.strip()})")
    st.markdown("**Ссылки:** " + (" | ".join(links) if links else "—"))

    # Аннотация
    st.markdown("### Аннотация")
    st.write(abstract if abstract else "—")

    # Доп. детали автооценки (по желанию)
    with st.expander("Детали автооценки", expanded=True):
        comp = row.get("dir_native_components") or {}
        if isinstance(comp, dict) and comp:
            tr = comp.get("trace"); tp = comp.get("topic"); tx = comp.get("text"); nz = comp.get("noise")
            sc = comp.get("score"); dl = comp.get("delta")
            st.markdown(
                f"**Компоненты (native DIR):** "
                f"trace={tr if tr is not None else '—'} | "
                f"topic={tp if tp is not None else '—'} | "
                f"text={tx if tx is not None else '—'} | "
                f"noise={nz if nz is not None else '—'} | "
                f"score={sc if sc is not None else '—'} | "
                f"delta={dl if dl is not None else '—'}"
            )
        expl = row.get("dir_score_explanation")
        if expl:
            st.markdown("**dir_score_explanation:**")
            st.write(expl)