Spaces:
Runtime error
Runtime error
Update src/streamlit_app.py
Browse files- src/streamlit_app.py +578 -587
src/streamlit_app.py
CHANGED
|
@@ -1,33 +1,10 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
# -*- coding: utf-8 -*-
|
| 3 |
-
"""streamlit_app.py
|
| 4 |
-
|
| 5 |
-
Скоринг публикаций по направлениям исследований DIR01..DIR14.
|
| 6 |
-
Offline-first: приложение работает ТОЛЬКО по готовым JSONL candidates и НЕ делает
|
| 7 |
-
никаких сетевых enrichment (OpenAlex API и т.п.).
|
| 8 |
-
|
| 9 |
-
Хранилища на HuggingFace Hub (repo_type="dataset"):
|
| 10 |
-
- PUBLICATIONS_REPO: содержит dir_registry.json и candidates JSONL
|
| 11 |
-
- REVIEWS_REPO: журнал оценок (JSONL batches)
|
| 12 |
-
|
| 13 |
-
Переменные окружения (Space Settings → Variables/Secrets):
|
| 14 |
-
- PUBLICATIONS_REPO, REVIEWS_REPO (обязательно)
|
| 15 |
-
- HF_TOKEN (нужен для записи; для чтения приватных репо)
|
| 16 |
-
- BATCH_SIZE (по умолчанию 1)
|
| 17 |
-
- PAGE_WINDOW_SIZE (по умолчанию 50)
|
| 18 |
-
- PUB_DIR_REGISTRY_PATH (по умолчанию dir_registry.json)
|
| 19 |
-
- PUB_CANDIDATES_PREFIX (по умолчанию candidates)
|
| 20 |
-
- REVIEWS_LOG_PREFIX (по умолчанию reviews_log)
|
| 21 |
-
- PUB_CANDIDATES_PATTERNS (опционально)
|
| 22 |
-
|
| 23 |
-
"""
|
| 24 |
-
|
| 25 |
import os
|
| 26 |
import json
|
| 27 |
import uuid
|
| 28 |
import gzip
|
| 29 |
import tempfile
|
| 30 |
import datetime as dt
|
|
|
|
| 31 |
from typing import List, Dict, Any, Optional, Tuple, Set
|
| 32 |
|
| 33 |
import pandas as pd
|
|
@@ -35,42 +12,78 @@ import streamlit as st
|
|
| 35 |
from huggingface_hub import hf_hub_download, HfApi
|
| 36 |
|
| 37 |
# =========================
|
| 38 |
-
# ENV CONFIG
|
| 39 |
# =========================
|
| 40 |
PUBLICATIONS_REPO = os.environ.get("PUBLICATIONS_REPO", "").strip()
|
| 41 |
REVIEWS_REPO = os.environ.get("REVIEWS_REPO", "").strip()
|
| 42 |
HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HF_API_TOKEN")
|
|
|
|
|
|
|
| 43 |
|
| 44 |
-
|
| 45 |
-
REVIEWS_PRIVATE = (os.environ.get("REVIEWS_PRIVATE", "1").strip().lower() in ("1", "true", "yes", "y", "on"))
|
| 46 |
-
|
| 47 |
BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "1"))
|
| 48 |
-
PAGE_WINDOW_SIZE = int(os.environ.get("PAGE_WINDOW_SIZE", "50"))
|
| 49 |
|
| 50 |
-
PUB_DIR_REGISTRY_PATH = os.environ.get("PUB_DIR_REGISTRY_PATH", "dir_registry.json")
|
| 51 |
-
PUB_CANDIDATES_PREFIX = os.environ.get("PUB_CANDIDATES_PREFIX", "candidates")
|
| 52 |
-
REVIEWS_LOG_PREFIX = os.environ.get("REVIEWS_LOG_PREFIX", "reviews_log")
|
| 53 |
|
| 54 |
-
# Паттерны поиска candidates файлов (разны
|
|
|
|
|
|
|
| 55 |
PUB_CANDIDATES_PATTERNS = os.environ.get(
|
| 56 |
"PUB_CANDIDATES_PATTERNS",
|
| 57 |
-
"{prefix}/{dir_id}_top500.jsonl|{prefix}/{dir_id}_top500.jsonl.gz|{prefix}/{dir_id}.jsonl|{prefix}/{dir_id}.jsonl.gz|manual_candidates/{dir_id}.jsonl|manual_candidates/{dir_id}.jsonl.gz"
|
| 58 |
).strip()
|
| 59 |
|
|
|
|
| 60 |
api = HfApi(token=HF_TOKEN)
|
| 61 |
|
| 62 |
-
st.set_page_config(page_title="
|
|
|
|
| 63 |
|
| 64 |
# =========================
|
| 65 |
# Helpers
|
| 66 |
# =========================
|
| 67 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
def require_env(name: str, value: str) -> None:
|
| 69 |
if not value:
|
| 70 |
st.error(f"Не задано **{name}**. Укажи в Space Settings → Variables.")
|
| 71 |
st.stop()
|
| 72 |
|
| 73 |
-
|
| 74 |
def safe_int(x) -> Optional[int]:
|
| 75 |
try:
|
| 76 |
if x is None or pd.isna(x):
|
|
@@ -79,7 +92,6 @@ def safe_int(x) -> Optional[int]:
|
|
| 79 |
except Exception:
|
| 80 |
return None
|
| 81 |
|
| 82 |
-
|
| 83 |
def safe_float(x) -> Optional[float]:
|
| 84 |
try:
|
| 85 |
if x is None or pd.isna(x):
|
|
@@ -88,20 +100,20 @@ def safe_float(x) -> Optional[float]:
|
|
| 88 |
except Exception:
|
| 89 |
return None
|
| 90 |
|
| 91 |
-
|
| 92 |
def doi_url(doi: Optional[str]) -> Optional[str]:
|
| 93 |
if not doi:
|
| 94 |
return None
|
| 95 |
doi = str(doi).strip()
|
| 96 |
-
if not doi:
|
| 97 |
-
return None
|
| 98 |
if doi.startswith("http://") or doi.startswith("https://"):
|
| 99 |
return doi
|
| 100 |
return f"https://doi.org/{doi}"
|
| 101 |
|
| 102 |
-
|
| 103 |
def normalize_dir_id(dir_id: str, pad2: bool) -> str:
|
| 104 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
if not isinstance(dir_id, str):
|
| 106 |
return str(dir_id)
|
| 107 |
s = dir_id.strip().upper()
|
|
@@ -111,35 +123,34 @@ def normalize_dir_id(dir_id: str, pad2: bool) -> str:
|
|
| 111 |
try:
|
| 112 |
n = int(tail)
|
| 113 |
except Exception:
|
|
|
|
| 114 |
return dir_id.strip()
|
| 115 |
return f"DIR{n:02d}" if pad2 else f"DIR{n}"
|
| 116 |
|
| 117 |
-
|
| 118 |
def dir_variants(dir_id: str) -> List[str]:
|
|
|
|
|
|
|
|
|
|
| 119 |
if not isinstance(dir_id, str):
|
| 120 |
return [str(dir_id)]
|
| 121 |
a = dir_id.strip()
|
| 122 |
b = normalize_dir_id(a, pad2=True)
|
| 123 |
c = normalize_dir_id(a, pad2=False)
|
| 124 |
-
out
|
| 125 |
for x in [a, b, c]:
|
| 126 |
if x and x not in out:
|
| 127 |
out.append(x)
|
| 128 |
return out
|
| 129 |
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
if d2.upper().startswith("DIR") and len(d2) >= 5:
|
| 135 |
-
return f"DIR-{d2[3:]}"
|
| 136 |
-
return d2
|
| 137 |
-
|
| 138 |
|
| 139 |
def join_terms(term_list: Any) -> str:
|
| 140 |
if not term_list:
|
| 141 |
return "—"
|
| 142 |
-
out
|
| 143 |
for x in term_list:
|
| 144 |
if isinstance(x, dict):
|
| 145 |
t = x.get("t")
|
|
@@ -149,72 +160,74 @@ def join_terms(term_list: Any) -> str:
|
|
| 149 |
out.append(str(x))
|
| 150 |
return "; ".join(out) if out else "—"
|
| 151 |
|
| 152 |
-
|
| 153 |
def topics_line(topics: Any) -> str:
|
| 154 |
if not topics:
|
| 155 |
return "—"
|
| 156 |
-
out
|
| 157 |
-
if isinstance(topics, list):
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
out.append(str(t))
|
| 170 |
return "; ".join(out) if out else "—"
|
| 171 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
|
| 173 |
def decode_abstract(abstract_inverted_index: Optional[dict]) -> str:
|
| 174 |
if not abstract_inverted_index or not isinstance(abstract_inverted_index, dict):
|
| 175 |
return ""
|
| 176 |
-
pos_to_word
|
| 177 |
for w, positions in abstract_inverted_index.items():
|
| 178 |
if not isinstance(positions, list):
|
| 179 |
continue
|
| 180 |
for p in positions:
|
| 181 |
-
|
| 182 |
-
pos_to_word[int(p)] = str(w)
|
| 183 |
-
except Exception:
|
| 184 |
-
continue
|
| 185 |
return " ".join(pos_to_word[p] for p in sorted(pos_to_word.keys())) if pos_to_word else ""
|
| 186 |
|
| 187 |
-
|
| 188 |
def get_authors_and_abstract(row: Dict[str, Any]) -> Tuple[str, str]:
|
| 189 |
-
"""
|
| 190 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
if not authors:
|
| 192 |
-
|
|
|
|
| 193 |
if isinstance(auths, list):
|
| 194 |
-
names
|
| 195 |
for a in auths:
|
| 196 |
if not isinstance(a, dict):
|
| 197 |
continue
|
| 198 |
-
an = ((a.get(
|
| 199 |
if an:
|
| 200 |
names.append(an)
|
| 201 |
if names:
|
| 202 |
-
authors =
|
| 203 |
-
|
| 204 |
if isinstance(authors, list):
|
| 205 |
-
authors =
|
| 206 |
-
authors_str = authors.strip() if isinstance(authors, str) and authors.strip() else
|
| 207 |
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
return authors_str or "—", abstract_str or "—"
|
| 214 |
|
|
|
|
| 215 |
|
| 216 |
def open_jsonl_any(path: str) -> List[Dict[str, Any]]:
|
| 217 |
-
rows
|
| 218 |
if path.endswith(".gz"):
|
| 219 |
with gzip.open(path, "rt", encoding="utf-8") as f:
|
| 220 |
for line in f:
|
|
@@ -229,32 +242,37 @@ def open_jsonl_any(path: str) -> List[Dict[str, Any]]:
|
|
| 229 |
rows.append(json.loads(line))
|
| 230 |
return rows
|
| 231 |
|
| 232 |
-
|
| 233 |
def normalize_candidate_row(obj: Dict[str, Any], selected_dir: str) -> Dict[str, Any]:
|
| 234 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
out = dict(obj)
|
| 236 |
|
| 237 |
-
# IDs
|
| 238 |
source_id = out.get("source_id") or out.get("Source ID")
|
| 239 |
if isinstance(source_id, str) and source_id.strip():
|
| 240 |
out["source_id"] = source_id.strip()
|
| 241 |
|
| 242 |
work_id = out.get("work_id")
|
| 243 |
if not isinstance(work_id, str) or not work_id.strip():
|
|
|
|
| 244 |
wid = out.get("openalex_work_id") or out.get("OpenAlex work ID") or out.get("dup_work_id")
|
| 245 |
if isinstance(wid, str) and wid.strip():
|
| 246 |
work_id = wid.strip()
|
| 247 |
-
out["work_id"] = work_id if isinstance(work_id, str)
|
| 248 |
|
| 249 |
-
#
|
| 250 |
title = out.get("title") or out.get("display_name") or out.get("Title")
|
| 251 |
out["title"] = str(title).strip() if title else "—"
|
| 252 |
|
| 253 |
-
#
|
| 254 |
doi = out.get("doi") or out.get("DOI")
|
| 255 |
out["doi"] = str(doi).strip() if doi else None
|
| 256 |
|
| 257 |
-
#
|
| 258 |
if out.get("abstract") is None:
|
| 259 |
ab = out.get("abstract_text")
|
| 260 |
if isinstance(ab, str) and ab.strip():
|
|
@@ -262,7 +280,8 @@ def normalize_candidate_row(obj: Dict[str, Any], selected_dir: str) -> Dict[str,
|
|
| 262 |
elif isinstance(out.get("abstract_inverted_index"), dict):
|
| 263 |
out["abstract"] = decode_abstract(out.get("abstract_inverted_index")) or None
|
| 264 |
|
| 265 |
-
|
|
|
|
| 266 |
year = out.get("year")
|
| 267 |
if year is None:
|
| 268 |
year = out.get("publication_year") or out.get("Publication year")
|
|
@@ -273,21 +292,27 @@ def normalize_candidate_row(obj: Dict[str, Any], selected_dir: str) -> Dict[str,
|
|
| 273 |
cited_by = out.get("cited_by_count") or out.get("Cited by count")
|
| 274 |
out["cited_by"] = safe_int(cited_by)
|
| 275 |
|
| 276 |
-
#
|
|
|
|
|
|
|
| 277 |
ds = out.get("dir_score")
|
|
|
|
| 278 |
if ds is None:
|
| 279 |
comp = out.get("dir_native_components") or {}
|
| 280 |
if isinstance(comp, dict) and comp.get("score") is not None:
|
| 281 |
ds = comp.get("score")
|
|
|
|
| 282 |
if ds is None:
|
| 283 |
dsd = out.get("dir_scores") or {}
|
| 284 |
if isinstance(dsd, dict):
|
|
|
|
| 285 |
for dv in dir_variants(selected_dir):
|
| 286 |
if dv in dsd:
|
| 287 |
ds = dsd.get(dv)
|
| 288 |
break
|
| 289 |
out["dir_score"] = safe_float(ds)
|
| 290 |
|
|
|
|
| 291 |
ms = out.get("match_score")
|
| 292 |
if ms is None:
|
| 293 |
comp = out.get("dir_native_components") or {}
|
|
@@ -300,64 +325,50 @@ def normalize_candidate_row(obj: Dict[str, Any], selected_dir: str) -> Dict[str,
|
|
| 300 |
ms = None
|
| 301 |
out["match_score"] = safe_float(ms)
|
| 302 |
|
| 303 |
-
|
| 304 |
-
|
|
|
|
|
|
|
|
|
|
| 305 |
|
| 306 |
-
# links
|
| 307 |
out["pdf_url"] = out.get("pdf_url") or out.get("PDF URL")
|
| 308 |
out["primary_location_url"] = out.get("primary_location_url") or out.get("Primary location URL")
|
| 309 |
|
| 310 |
return out
|
| 311 |
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
|
|
|
|
|
|
|
|
|
| 316 |
if isinstance(sid, str) and sid.strip():
|
| 317 |
-
|
| 318 |
-
wid =
|
| 319 |
if isinstance(wid, str) and wid.strip():
|
| 320 |
-
|
| 321 |
-
return
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
def check_dataset_repo(repo_id: str, token: Optional[str]) -> Tuple[bool, str]:
|
| 325 |
-
try:
|
| 326 |
-
api_local = HfApi(token=token) if token else HfApi()
|
| 327 |
-
_ = api_local.repo_info(repo_id=repo_id, repo_type="dataset")
|
| 328 |
-
return True, "OK"
|
| 329 |
-
except Exception as e:
|
| 330 |
-
return False, str(e)
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
def maybe_create_reviews_repo(repo_id: str) -> Tuple[bool, str]:
|
| 334 |
-
if not ALLOW_CREATE_REVIEWS_REPO:
|
| 335 |
-
return False, "ALLOW_CREATE_REVIEWS_REPO=0"
|
| 336 |
-
if not HF_TOKEN:
|
| 337 |
-
return False, "HF_TOKEN отсутствует (нужен для создания репозитория)"
|
| 338 |
-
try:
|
| 339 |
-
api.create_repo(repo_id=repo_id, repo_type="dataset", private=bool(REVIEWS_PRIVATE), exist_ok=True)
|
| 340 |
-
return True, "created_or_exists"
|
| 341 |
-
except Exception as e:
|
| 342 |
-
return False, str(e)
|
| 343 |
|
| 344 |
|
| 345 |
# =========================
|
| 346 |
-
#
|
| 347 |
# =========================
|
| 348 |
-
|
| 349 |
@st.cache_data(show_spinner=False)
|
| 350 |
def load_dir_registry(repo_id: str, filename: str) -> List[Dict[str, Any]]:
|
| 351 |
path = hf_hub_download(repo_id=repo_id, filename=filename, repo_type="dataset", token=HF_TOKEN)
|
| 352 |
with open(path, "r", encoding="utf-8") as f:
|
| 353 |
return json.load(f)
|
| 354 |
|
| 355 |
-
|
| 356 |
@st.cache_data(show_spinner=False)
|
| 357 |
def load_candidates(repo_id: str, dir_id: str, prefix: str) -> pd.DataFrame:
|
|
|
|
|
|
|
|
|
|
| 358 |
patterns = [p.strip() for p in PUB_CANDIDATES_PATTERNS.split("|") if p.strip()]
|
| 359 |
-
tried
|
| 360 |
-
rows
|
| 361 |
|
| 362 |
for dv in dir_variants(dir_id):
|
| 363 |
for pat in patterns:
|
|
@@ -366,7 +377,11 @@ def load_candidates(repo_id: str, dir_id: str, prefix: str) -> pd.DataFrame:
|
|
| 366 |
try:
|
| 367 |
path = hf_hub_download(repo_id=repo_id, filename=fname, repo_type="dataset", token=HF_TOKEN)
|
| 368 |
rows = open_jsonl_any(path)
|
| 369 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 370 |
except Exception:
|
| 371 |
continue
|
| 372 |
if rows is not None:
|
|
@@ -375,9 +390,11 @@ def load_candidates(repo_id: str, dir_id: str, prefix: str) -> pd.DataFrame:
|
|
| 375 |
if rows is None:
|
| 376 |
raise FileNotFoundError("Не найден candidates файл. Пробовали:\n" + "\n".join(tried[:30]) + ("\n..." if len(tried) > 30 else ""))
|
| 377 |
|
|
|
|
| 378 |
normed = [normalize_candidate_row(r, dir_id) for r in rows if isinstance(r, dict)]
|
| 379 |
df = pd.DataFrame(normed)
|
| 380 |
|
|
|
|
| 381 |
for col in ["dir_score", "match_score", "cited_by", "year", "quality_weight"]:
|
| 382 |
if col in df.columns:
|
| 383 |
df[col] = pd.to_numeric(df[col], errors="coerce")
|
|
@@ -386,14 +403,14 @@ def load_candidates(repo_id: str, dir_id: str, prefix: str) -> pd.DataFrame:
|
|
| 386 |
|
| 387 |
|
| 388 |
# =========================
|
| 389 |
-
#
|
| 390 |
# =========================
|
| 391 |
-
|
| 392 |
@st.cache_data(show_spinner=False)
|
| 393 |
def list_review_files(repo_id: str, dir_id: str, prefix: str) -> List[str]:
|
| 394 |
files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
|
|
|
|
| 395 |
needles = [f"/{dv}/" for dv in dir_variants(dir_id)]
|
| 396 |
-
out
|
| 397 |
for p in files:
|
| 398 |
if not (p.startswith(prefix + "/") and p.endswith(".jsonl")):
|
| 399 |
continue
|
|
@@ -401,25 +418,19 @@ def list_review_files(repo_id: str, dir_id: str, prefix: str) -> List[str]:
|
|
| 401 |
out.append(p)
|
| 402 |
return out
|
| 403 |
|
| 404 |
-
|
| 405 |
@st.cache_data(show_spinner=False)
|
| 406 |
-
def
|
| 407 |
-
"""
|
| 408 |
-
|
| 409 |
-
- с
|
| 410 |
-
-
|
| 411 |
-
|
| 412 |
-
Важно: это НЕ enrichment; это чтение ваших собственных сохранённых оценок.
|
| 413 |
"""
|
| 414 |
try:
|
| 415 |
files = list_review_files(repo_id, dir_id, prefix)
|
| 416 |
except Exception:
|
| 417 |
-
return set()
|
| 418 |
-
|
| 419 |
-
keys: Set[str] = set()
|
| 420 |
-
by_src: Dict[str, Dict[str, Any]] = {}
|
| 421 |
-
by_w: Dict[str, Dict[str, Any]] = {}
|
| 422 |
|
|
|
|
| 423 |
for relpath in files:
|
| 424 |
try:
|
| 425 |
path = hf_hub_download(repo_id=repo_id, filename=relpath, repo_type="dataset", token=HF_TOKEN)
|
|
@@ -431,34 +442,29 @@ def load_reviews_index(repo_id: str, dir_id: str, prefix: str) -> Tuple[Set[str]
|
|
| 431 |
obj = json.loads(line)
|
| 432 |
|
| 433 |
sid = obj.get("source_id")
|
| 434 |
-
wid = obj.get("work_id")
|
| 435 |
-
|
| 436 |
if isinstance(sid, str) and sid.strip():
|
| 437 |
-
sid
|
| 438 |
-
|
| 439 |
-
|
| 440 |
if isinstance(wid, str) and wid.strip():
|
| 441 |
-
wid
|
| 442 |
-
keys.add(f"W::{wid}")
|
| 443 |
-
by_w[wid] = obj
|
| 444 |
except Exception:
|
| 445 |
continue
|
| 446 |
-
|
| 447 |
-
return keys, by_src, by_w
|
| 448 |
|
| 449 |
|
| 450 |
# =========================
|
| 451 |
-
#
|
| 452 |
# =========================
|
| 453 |
-
|
| 454 |
def push_batch_to_reviews_repo(repo_id: str, dir_id: str, prefix: str, batch: List[Dict[str, Any]]) -> None:
|
| 455 |
if not batch:
|
| 456 |
return
|
| 457 |
if not HF_TOKEN:
|
| 458 |
raise RuntimeError("Нет HF_TOKEN (Secret) — нельзя записывать в reviews dataset.")
|
| 459 |
-
|
| 460 |
today = dt.date.today().isoformat()
|
| 461 |
batch_id = str(uuid.uuid4())
|
|
|
|
|
|
|
| 462 |
canonical_dir = normalize_dir_id(dir_id, pad2=False)
|
| 463 |
path_in_repo = f"{prefix}/{today}/{canonical_dir}/{batch_id}.jsonl"
|
| 464 |
|
|
@@ -477,64 +483,20 @@ def push_batch_to_reviews_repo(repo_id: str, dir_id: str, prefix: str, batch: Li
|
|
| 477 |
|
| 478 |
|
| 479 |
# =========================
|
| 480 |
-
#
|
| 481 |
# =========================
|
| 482 |
|
| 483 |
-
def ensure_state() -> None:
|
| 484 |
-
if "batch" not in st.session_state:
|
| 485 |
-
st.session_state["batch"] = []
|
| 486 |
-
|
| 487 |
-
if "reviewed_local_committed" not in st.session_state:
|
| 488 |
-
st.session_state["reviewed_local_committed"] = set()
|
| 489 |
-
if "reviewed_local_pending" not in st.session_state:
|
| 490 |
-
st.session_state["reviewed_local_pending"] = set()
|
| 491 |
-
|
| 492 |
-
# локальные значения (чтобы предзаполнять при навигации)
|
| 493 |
-
if "local_by_src" not in st.session_state:
|
| 494 |
-
st.session_state["local_by_src"] = {}
|
| 495 |
-
if "local_by_w" not in st.session_state:
|
| 496 |
-
st.session_state["local_by_w"] = {}
|
| 497 |
-
|
| 498 |
-
# remote кэш (НЕ показываем в UI)
|
| 499 |
-
if "remote_dir" not in st.session_state:
|
| 500 |
-
st.session_state["remote_dir"] = None
|
| 501 |
-
if "remote_keys" not in st.session_state:
|
| 502 |
-
st.session_state["remote_keys"] = set()
|
| 503 |
-
if "remote_by_src" not in st.session_state:
|
| 504 |
-
st.session_state["remote_by_src"] = {}
|
| 505 |
-
if "remote_by_w" not in st.session_state:
|
| 506 |
-
st.session_state["remote_by_w"] = {}
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
def get_saved_review_for_row(row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
| 510 |
-
sid = row.get("source_id") if isinstance(row.get("source_id"), str) else None
|
| 511 |
-
wid = row.get("work_id") if isinstance(row.get("work_id"), str) else None
|
| 512 |
-
|
| 513 |
-
# local first (pending/committed current session)
|
| 514 |
-
if sid and sid in st.session_state.get("local_by_src", {}):
|
| 515 |
-
return st.session_state["local_by_src"].get(sid)
|
| 516 |
-
if wid and wid in st.session_state.get("local_by_w", {}):
|
| 517 |
-
return st.session_state["local_by_w"].get(wid)
|
| 518 |
-
|
| 519 |
-
# remote next
|
| 520 |
-
if sid and sid in st.session_state.get("remote_by_src", {}):
|
| 521 |
-
return st.session_state["remote_by_src"].get(sid)
|
| 522 |
-
if wid and wid in st.session_state.get("remote_by_w", {}):
|
| 523 |
-
return st.session_state["remote_by_w"].get(wid)
|
| 524 |
-
|
| 525 |
-
return None
|
| 526 |
-
|
| 527 |
-
|
| 528 |
# =========================
|
| 529 |
-
#
|
| 530 |
# =========================
|
| 531 |
|
| 532 |
with st.sidebar:
|
| 533 |
-
st.title("Ск
|
| 534 |
|
| 535 |
require_env("PUBLICATIONS_REPO", PUBLICATIONS_REPO)
|
| 536 |
require_env("REVIEWS_REPO", REVIEWS_REPO)
|
| 537 |
|
|
|
|
| 538 |
pub_ok, pub_msg = check_dataset_repo(PUBLICATIONS_REPO, HF_TOKEN)
|
| 539 |
if not pub_ok:
|
| 540 |
st.error("PUBLICATIONS_REPO недоступен как dataset repo. Проверь repo_id и доступ.\n\n" + pub_msg)
|
|
@@ -542,145 +504,122 @@ with st.sidebar:
|
|
| 542 |
|
| 543 |
rev_ok, rev_msg = check_dataset_repo(REVIEWS_REPO, HF_TOKEN)
|
| 544 |
if not rev_ok:
|
| 545 |
-
created,
|
| 546 |
if created:
|
| 547 |
rev_ok2, rev_msg2 = check_dataset_repo(REVIEWS_REPO, HF_TOKEN)
|
| 548 |
if not rev_ok2:
|
| 549 |
st.error("REVIEWS_REPO недоступен после create_repo.\n\n" + rev_msg2)
|
| 550 |
st.stop()
|
|
|
|
|
|
|
| 551 |
else:
|
| 552 |
st.error(
|
| 553 |
"REVIEWS_REPO недоступен (404/нет доступа).\n\n"
|
| 554 |
-
"
|
| 555 |
-
"1) REVIEWS_REPO = <owner>/<dataset_name> (именно dataset).\n"
|
| 556 |
-
"2) Репозиторий существует на Hub.\n"
|
| 557 |
-
"3)
|
|
|
|
| 558 |
+ rev_msg
|
| 559 |
)
|
| 560 |
st.stop()
|
| 561 |
|
| 562 |
-
|
| 563 |
-
reviewer = st.text_input("Рецензент (имя/ник)", value=reviewer_default).strip() or "anonymous"
|
| 564 |
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
|
|
|
|
|
|
|
|
|
|
| 570 |
|
| 571 |
-
|
| 572 |
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
|
| 577 |
-
clear_cache = st.button("🧹 Сбросить кэш (публикации/оценки)", use_container_width=True)
|
| 578 |
-
st.caption(f"BATCH_SIZE={BATCH_SIZE}")
|
| 579 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 580 |
|
| 581 |
# =========================
|
| 582 |
-
#
|
| 583 |
# =========================
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 584 |
|
| 585 |
-
if
|
| 586 |
load_dir_registry.clear()
|
| 587 |
load_candidates.clear()
|
| 588 |
-
list_review_files
|
| 589 |
-
|
| 590 |
-
|
|
|
|
|
|
|
|
|
|
| 591 |
for k in [
|
| 592 |
-
"
|
| 593 |
-
"
|
| 594 |
-
"
|
| 595 |
-
"
|
| 596 |
"reviewed_local_committed",
|
| 597 |
"reviewed_local_pending",
|
| 598 |
-
"local_by_src",
|
| 599 |
-
"local_by_w",
|
| 600 |
"batch",
|
| 601 |
]:
|
| 602 |
if k in st.session_state:
|
| 603 |
del st.session_state[k]
|
| 604 |
-
|
| 605 |
-
st.toast("Кэш очищен")
|
| 606 |
st.rerun()
|
| 607 |
|
| 608 |
-
|
| 609 |
# =========================
|
| 610 |
-
#
|
| 611 |
# =========================
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 626 |
)
|
| 627 |
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
# =========================
|
| 635 |
-
|
| 636 |
-
def _get(d: Dict[str, Any], key: str, default: str = "—") -> str:
|
| 637 |
-
v = d.get(key)
|
| 638 |
-
return v if isinstance(v, str) and v.strip() else default
|
| 639 |
-
|
| 640 |
-
st.markdown(f"### {dir_meta.get('dir_name', '—')}")
|
| 641 |
-
|
| 642 |
-
# выделенный блок, чтобы отличался от основного интерфейса
|
| 643 |
-
with st.container():
|
| 644 |
-
st.markdown(
|
| 645 |
-
"""
|
| 646 |
-
<style>
|
| 647 |
-
.dir-card{border:1px solid rgba(255,255,255,.12); border-radius:14px; padding:14px 16px; background: rgba(255,255,255,.03);}
|
| 648 |
-
.dir-tag{font-size:12px; letter-spacing:.08em; opacity:.75; font-weight:700;}
|
| 649 |
-
.dir-desc-title{font-size:11px; letter-spacing:.08em; opacity:.75; font-weight:800; margin-top:10px;}
|
| 650 |
-
</style>
|
| 651 |
-
""",
|
| 652 |
-
unsafe_allow_html=True,
|
| 653 |
-
)
|
| 654 |
-
st.markdown(f"<div class='dir-card'><div class='dir-tag'>{dir_tag(selected_dir)}</div>", unsafe_allow_html=True)
|
| 655 |
-
st.markdown("<div class='dir-desc-title'>КРАТКОЕ ОПИСАНИЕ</div>", unsafe_allow_html=True)
|
| 656 |
-
st.write(_get(dir_meta, "dir_description"))
|
| 657 |
-
|
| 658 |
-
defaults = dir_meta.get("defaults") or {}
|
| 659 |
-
year_from = defaults.get("year_from")
|
| 660 |
-
year_to = defaults.get("year_to")
|
| 661 |
-
|
| 662 |
-
terms = dir_meta.get("terms") or {}
|
| 663 |
-
anchor_str = join_terms(terms.get("anchor"))
|
| 664 |
-
support_str = join_terms(terms.get("support"))
|
| 665 |
-
noise_str = join_terms(terms.get("noise"))
|
| 666 |
-
topics_str = topics_line(dir_meta.get("topics") or [])
|
| 667 |
-
|
| 668 |
-
with st.expander("Детали DIR", expanded=True):
|
| 669 |
-
st.markdown(f"**Временной интервал:** {year_from} – {year_to}")
|
| 670 |
-
st.markdown(f"**Якоря:** {anchor_str}")
|
| 671 |
-
st.markdown(f"**Поддержка:** {support_str}")
|
| 672 |
-
st.markdown(f"**Шум:** {noise_str}")
|
| 673 |
-
st.markdown(f"**Topics поиска:** {topics_str}")
|
| 674 |
-
|
| 675 |
-
st.markdown("</div>", unsafe_allow_html=True)
|
| 676 |
|
| 677 |
st.divider()
|
| 678 |
|
| 679 |
-
|
| 680 |
# =========================
|
| 681 |
-
# Load candidates
|
| 682 |
# =========================
|
| 683 |
-
|
| 684 |
try:
|
| 685 |
df = load_candidates(PUBLICATIONS_REPO, selected_dir, PUB_CANDIDATES_PREFIX)
|
| 686 |
except Exception as e:
|
|
@@ -688,53 +627,75 @@ except Exception as e:
|
|
| 688 |
st.stop()
|
| 689 |
|
| 690 |
if df.empty:
|
| 691 |
-
st.warning(f"Пустой список кандидатов для {selected_dir}")
|
| 692 |
st.stop()
|
| 693 |
|
| 694 |
-
#
|
| 695 |
-
|
| 696 |
-
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
| 703 |
-
|
| 704 |
-
|
|
|
|
|
|
|
| 705 |
|
| 706 |
df = df.reset_index(drop=True)
|
| 707 |
-
|
| 708 |
-
total =
|
| 709 |
-
|
|
|
|
| 710 |
|
| 711 |
# =========================
|
| 712 |
-
# State + remote
|
| 713 |
# =========================
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 714 |
|
| 715 |
ensure_state()
|
| 716 |
|
| 717 |
-
|
| 718 |
-
|
| 719 |
-
|
| 720 |
-
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
| 727 |
-
reviewed_committed = set(st.session_state.get("remote_keys", set())) | set(st.session_state.get("reviewed_local_committed", set()))
|
| 728 |
-
reviewed_pending = set(st.session_state.get("reviewed_local_pending", set()))
|
| 729 |
-
reviewed_effective = reviewed_committed | reviewed_pending
|
| 730 |
|
|
|
|
| 731 |
|
| 732 |
-
|
| 733 |
-
|
| 734 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 735 |
|
|
|
|
|
|
|
|
|
|
| 736 |
|
| 737 |
-
def
|
| 738 |
keys = row_keys_all(r)
|
| 739 |
if keys and any(k in reviewed_committed for k in keys):
|
| 740 |
return "✅"
|
|
@@ -742,18 +703,57 @@ def status_icon(r: Dict[str, Any]) -> str:
|
|
| 742 |
return "🕓"
|
| 743 |
return "🆕"
|
| 744 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 745 |
|
| 746 |
-
# counts for sidebar (без слова remote)
|
| 747 |
src_series = df["source_id"].fillna("") if "source_id" in df.columns else pd.Series([""] * len(df))
|
| 748 |
wid_series = df["work_id"].fillna("") if "work_id" in df.columns else pd.Series([""] * len(df))
|
| 749 |
|
| 750 |
-
|
| 751 |
-
|
| 752 |
-
pend_src = {k[5:] for k in reviewed_pending if isinstance(k, str) and k.startswith("SRC::")}
|
| 753 |
-
pend_w = {k[3:] for k in reviewed_pending if isinstance(k, str) and k.startswith("W::")}
|
| 754 |
-
|
| 755 |
-
mask_committed = src_series.astype(str).isin(comm_src) | wid_series.astype(str).isin(comm_w)
|
| 756 |
-
mask_pending = src_series.astype(str).isin(pend_src) | wid_series.astype(str).isin(pend_w)
|
| 757 |
mask_reviewed = mask_committed | mask_pending
|
| 758 |
|
| 759 |
committed_pub_count = int(mask_committed.sum())
|
|
@@ -761,9 +761,9 @@ pending_pub_count = int(mask_pending.sum())
|
|
| 761 |
unreviewed_pub_count = int((~mask_reviewed).sum())
|
| 762 |
|
| 763 |
with st.sidebar:
|
| 764 |
-
st.
|
| 765 |
-
|
| 766 |
-
|
| 767 |
|
| 768 |
if st.session_state.get("batch"):
|
| 769 |
pending_jsonl = "".join(json.dumps(x, ensure_ascii=False) + "\n" for x in st.session_state["batch"])
|
|
@@ -775,22 +775,19 @@ with st.sidebar:
|
|
| 775 |
use_container_width=True,
|
| 776 |
)
|
| 777 |
|
| 778 |
-
|
| 779 |
# =========================
|
| 780 |
-
# Commit
|
| 781 |
# =========================
|
| 782 |
-
|
| 783 |
def commit_batch() -> Tuple[bool, str]:
|
| 784 |
batch = st.session_state.get("batch") or []
|
| 785 |
if not batch:
|
| 786 |
return False, "batch_empty"
|
| 787 |
-
|
| 788 |
try:
|
| 789 |
push_batch_to_reviews_repo(REVIEWS_REPO, selected_dir, REVIEWS_LOG_PREFIX, batch)
|
| 790 |
except Exception as e:
|
| 791 |
return False, str(e)
|
| 792 |
|
| 793 |
-
# pending -> committed
|
| 794 |
for obj in batch:
|
| 795 |
sid = obj.get("source_id")
|
| 796 |
wid = obj.get("work_id")
|
|
@@ -806,92 +803,67 @@ def commit_batch() -> Tuple[bool, str]:
|
|
| 806 |
st.session_state["batch"] = []
|
| 807 |
return True, "ok"
|
| 808 |
|
| 809 |
-
|
| 810 |
-
if
|
| 811 |
with st.spinner("Синхронизирую pending батч…"):
|
| 812 |
ok, msg = commit_batch()
|
| 813 |
if ok:
|
| 814 |
-
st.success("Pending батч синхронизирован")
|
| 815 |
st.rerun()
|
| 816 |
else:
|
| 817 |
st.error(f"Не удалось синхронизировать: {msg}")
|
| 818 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 819 |
|
| 820 |
# =========================
|
| 821 |
-
# Navigation
|
| 822 |
# =========================
|
| 823 |
-
|
| 824 |
idx_key = f"idx_{canonical_dir}"
|
| 825 |
if idx_key not in st.session_state:
|
| 826 |
st.session_state[idx_key] = 0
|
| 827 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 828 |
|
| 829 |
-
def
|
| 830 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 831 |
while i < total:
|
| 832 |
-
if not skip_reviewed:
|
| 833 |
-
return i
|
| 834 |
r = df.iloc[i].to_dict()
|
| 835 |
if is_reviewed_effective(r):
|
| 836 |
i += 1
|
| 837 |
continue
|
| 838 |
return i
|
| 839 |
-
return
|
| 840 |
-
|
| 841 |
-
|
| 842 |
-
# при первом входе в DIR — позиционируемся на первом неоценённом
|
| 843 |
-
if st.session_state[idx_key] == 0 and skip_reviewed:
|
| 844 |
-
st.session_state[idx_key] = first_unreviewed_from(0)
|
| 845 |
-
|
| 846 |
-
current_idx = int(st.session_state[idx_key])
|
| 847 |
-
current_idx = max(0, min(current_idx, total - 1))
|
| 848 |
-
|
| 849 |
-
|
| 850 |
-
def next_idx(i: int) -> int:
|
| 851 |
-
j = min(total - 1, int(i) + 1)
|
| 852 |
-
return first_unreviewed_from(j) if skip_reviewed else j
|
| 853 |
-
|
| 854 |
|
| 855 |
-
|
| 856 |
-
|
| 857 |
-
return max(0, int(i) - 1)
|
| 858 |
-
|
| 859 |
-
|
| 860 |
-
# =========================
|
| 861 |
-
# Layout: list always visible + card
|
| 862 |
-
# =========================
|
| 863 |
|
| 864 |
-
|
| 865 |
-
|
| 866 |
-
# current row
|
| 867 |
-
row = df.iloc[current_idx].to_dict()
|
| 868 |
-
|
| 869 |
-
# window indices around current
|
| 870 |
-
half = max(1, PAGE_WINDOW_SIZE // 2)
|
| 871 |
-
window_start = max(0, current_idx - half)
|
| 872 |
-
window_end = min(total, window_start + PAGE_WINDOW_SIZE)
|
| 873 |
-
window_start = max(0, window_end - PAGE_WINDOW_SIZE)
|
| 874 |
-
window_indices = list(range(window_start, window_end))
|
| 875 |
-
|
| 876 |
-
|
| 877 |
-
def fmt_idx(i: int) -> str:
|
| 878 |
-
r = df.iloc[i].to_dict()
|
| 879 |
-
ico = status_icon(r)
|
| 880 |
-
year = safe_int(r.get("year"))
|
| 881 |
-
year_s = str(year) if year is not None else "—"
|
| 882 |
-
title = str(r.get("title") or "—")
|
| 883 |
-
title_short = title if len(title) <= 70 else title[:67] + "…"
|
| 884 |
-
return f"{ico} {i+1:03d} | {year_s} | {title_short}"
|
| 885 |
-
|
| 886 |
-
|
| 887 |
-
with col_list:
|
| 888 |
st.markdown("### Список публикаций")
|
| 889 |
-
st.caption("
|
| 890 |
|
| 891 |
sel = st.selectbox(
|
| 892 |
-
"",
|
| 893 |
-
options=
|
| 894 |
-
index=
|
| 895 |
format_func=fmt_idx,
|
| 896 |
label_visibility="collapsed",
|
| 897 |
)
|
|
@@ -900,156 +872,70 @@ with col_list:
|
|
| 900 |
st.session_state[idx_key] = int(sel)
|
| 901 |
st.rerun()
|
| 902 |
|
| 903 |
-
|
| 904 |
-
|
| 905 |
-
|
| 906 |
-
|
| 907 |
-
|
| 908 |
-
|
| 909 |
-
|
| 910 |
-
|
| 911 |
-
|
| 912 |
-
|
| 913 |
-
|
| 914 |
-
|
| 915 |
-
|
| 916 |
-
|
| 917 |
-
|
| 918 |
-
|
| 919 |
-
#
|
| 920 |
-
|
| 921 |
-
|
| 922 |
-
|
| 923 |
-
|
| 924 |
-
|
| 925 |
-
|
| 926 |
-
|
| 927 |
-
|
| 928 |
-
|
| 929 |
-
|
| 930 |
-
|
| 931 |
-
|
| 932 |
-
|
| 933 |
-
|
| 934 |
-
|
| 935 |
-
|
| 936 |
-
|
| 937 |
-
|
| 938 |
-
|
| 939 |
-
|
| 940 |
-
|
| 941 |
-
|
| 942 |
-
|
| 943 |
-
y = safe_int(row.get("year"))
|
| 944 |
-
cites = safe_int(row.get("cited_by"))
|
| 945 |
-
dir_score = safe_float(row.get("dir_score"))
|
| 946 |
-
delta = safe_float(row.get("match_score"))
|
| 947 |
-
qw = safe_float(row.get("quality_weight"))
|
| 948 |
-
|
| 949 |
-
work_id = row.get("work_id") if isinstance(row.get("work_id"), str) else None
|
| 950 |
-
source_id = row.get("source_id") if isinstance(row.get("source_id"), str) else None
|
| 951 |
-
|
| 952 |
-
pdf_url = row.get("pdf_url")
|
| 953 |
-
pl_url = row.get("primary_location_url")
|
| 954 |
-
doi_link = doi_url(row.get("doi"))
|
| 955 |
-
|
| 956 |
-
# open_access (если есть)
|
| 957 |
-
oa = row.get("open_access") if isinstance(row.get("open_access"), dict) else {}
|
| 958 |
-
oa_is = oa.get("is_oa") if isinstance(oa, dict) else None
|
| 959 |
-
oa_status = oa.get("oa_status") if isinstance(oa, dict) else None
|
| 960 |
-
oa_url = oa.get("oa_url") if isinstance(oa, dict) else None
|
| 961 |
-
|
| 962 |
-
keys = row_keys_all(row)
|
| 963 |
-
committed = bool(keys and any(k in reviewed_committed for k in keys))
|
| 964 |
-
pending = bool(keys and any(k in reviewed_pending for k in keys))
|
| 965 |
-
status_str = "✅ оценено" if committed else ("🕓 pending" if pending else "🆕 не оценено")
|
| 966 |
-
|
| 967 |
-
has_abs = (isinstance(abstract, str) and abstract.strip() and abstract.strip() != "—")
|
| 968 |
-
has_pdf = (isinstance(pdf_url, str) and pdf_url.strip())
|
| 969 |
-
has_pl = (isinstance(pl_url, str) and pl_url.strip())
|
| 970 |
-
has_doi = bool(doi_link)
|
| 971 |
-
|
| 972 |
-
badges = [
|
| 973 |
-
f"Abstract {'✅' if has_abs else '❌'}",
|
| 974 |
-
f"PDF {'✅' if has_pdf else '❌'}",
|
| 975 |
-
f"URL {'✅' if has_pl else '❌'}",
|
| 976 |
-
f"DOI {'✅' if has_doi else '❌'}",
|
| 977 |
-
]
|
| 978 |
-
if oa_is is not None:
|
| 979 |
-
badges.append(f"OA {'✅' if oa_is else '❌'}{(' (' + str(oa_status) + ')') if oa_status else ''}")
|
| 980 |
|
|
|
|
| 981 |
|
| 982 |
-
|
| 983 |
-
info_col, score_col = st.columns([2.2, 1.1], gap="large")
|
| 984 |
|
| 985 |
-
with
|
| 986 |
-
st.
|
|
|
|
|
|
|
| 987 |
|
| 988 |
-
|
| 989 |
-
|
| 990 |
-
|
| 991 |
-
|
| 992 |
|
| 993 |
-
|
| 994 |
-
|
|
|
|
|
|
|
|
|
|
| 995 |
|
| 996 |
-
|
| 997 |
-
|
| 998 |
-
ds_s = f"{dir_score:.3f}" if dir_score is not None else "—"
|
| 999 |
-
dlt_s = f"{delta:.4f}" if delta is not None else "—"
|
| 1000 |
-
qw_s = f"{qw:.3f}" if qw is not None else "—"
|
| 1001 |
-
st.markdown(f"**Год:** {y_s} | **Цитаты:** {c_s} | **dir_score:** {ds_s} | **delta:** {dlt_s} | **quality:** {qw_s}")
|
| 1002 |
-
st.caption(" | ".join(badges))
|
| 1003 |
-
|
| 1004 |
-
with st.expander("Детали авторанжирования (опционально)", expanded=False):
|
| 1005 |
-
comp = row.get("dir_native_components") or {}
|
| 1006 |
-
if isinstance(comp, dict) and comp:
|
| 1007 |
-
tr = safe_float(comp.get("trace"))
|
| 1008 |
-
tp = safe_float(comp.get("topic"))
|
| 1009 |
-
tx = safe_float(comp.get("text"))
|
| 1010 |
-
nz = safe_float(comp.get("noise"))
|
| 1011 |
-
sc = safe_float(comp.get("score"))
|
| 1012 |
-
dl = safe_float(comp.get("delta"))
|
| 1013 |
-
st.markdown(
|
| 1014 |
-
f"**Компоненты (native DIR):** "
|
| 1015 |
-
f"trace={tr if tr is not None else '—'} | "
|
| 1016 |
-
f"topic={tp if tp is not None else '—'} | "
|
| 1017 |
-
f"text={tx if tx is not None else '—'} | "
|
| 1018 |
-
f"noise={nz if nz is not None else '—'} | "
|
| 1019 |
-
f"score={sc if sc is not None else '—'} | "
|
| 1020 |
-
f"delta={dl if dl is not None else '—'}"
|
| 1021 |
-
)
|
| 1022 |
-
expl = row.get("dir_score_explanation")
|
| 1023 |
-
if expl:
|
| 1024 |
-
st.markdown("**dir_score_explanation:**")
|
| 1025 |
-
st.write(expl)
|
| 1026 |
-
|
| 1027 |
-
links = []
|
| 1028 |
-
if work_id:
|
| 1029 |
-
links.append(f"[OpenAlex]({work_id})")
|
| 1030 |
-
if doi_link:
|
| 1031 |
-
links.append(f"[DOI]({doi_link})")
|
| 1032 |
-
if isinstance(pdf_url, str) and pdf_url.strip():
|
| 1033 |
-
links.append(f"[PDF]({pdf_url.strip()})")
|
| 1034 |
-
if isinstance(pl_url, str) and pl_url.strip():
|
| 1035 |
-
links.append(f"[Primary URL]({pl_url.strip()})")
|
| 1036 |
-
if isinstance(oa_url, str) and oa_url.strip():
|
| 1037 |
-
links.append(f"[OA URL]({oa_url.strip()})")
|
| 1038 |
-
st.markdown("**Ссылки:** " + (" | ".join(links) if links else "—"))
|
| 1039 |
-
|
| 1040 |
-
st.markdown("**Аннотация:**")
|
| 1041 |
-
st.write(abstract)
|
| 1042 |
-
|
| 1043 |
-
st.markdown(f"**Статус:** {status_str}")
|
| 1044 |
-
st.markdown(f"**Кандидат {int(st.session_state[idx_key])+1}/{total}**")
|
| 1045 |
-
|
| 1046 |
-
with score_col:
|
| 1047 |
-
st.markdown("### Оценка")
|
| 1048 |
-
|
| 1049 |
-
score_val = st.slider("Баллы (-5..+5)", -5, 5, int(st.session_state[score_key]), key=score_key)
|
| 1050 |
-
comment_val = st.text_area("Комментарий", height=160, key=comment_key)
|
| 1051 |
|
| 1052 |
-
if st.button("✅ Сохранить и далее", use_container_width=True):
|
| 1053 |
review = {
|
| 1054 |
"ts_utc": dt.datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
| 1055 |
"dir_id": selected_dir,
|
|
@@ -1067,44 +953,149 @@ with col_card:
|
|
| 1067 |
"delta": delta,
|
| 1068 |
"quality_weight": qw,
|
| 1069 |
"components": row.get("dir_native_components") or {},
|
| 1070 |
-
"rank": int(
|
| 1071 |
},
|
| 1072 |
-
"score": int(
|
| 1073 |
-
"comment":
|
| 1074 |
"reviewer": reviewer,
|
| 1075 |
}
|
| 1076 |
|
| 1077 |
-
# add to batch
|
| 1078 |
st.session_state["batch"].append(review)
|
| 1079 |
|
| 1080 |
-
#
|
| 1081 |
-
|
| 1082 |
-
st.session_state["
|
| 1083 |
-
|
| 1084 |
-
st.session_state["local_by_w"][work_id] = review
|
| 1085 |
-
|
| 1086 |
-
# pending keys (оба)
|
| 1087 |
-
if source_id:
|
| 1088 |
-
st.session_state["reviewed_local_pending"].add(f"SRC::{source_id.strip()}")
|
| 1089 |
-
if work_id:
|
| 1090 |
-
st.session_state["reviewed_local_pending"].add(f"W::{work_id.strip()}")
|
| 1091 |
|
|
|
|
| 1092 |
flushed = False
|
| 1093 |
if len(st.session_state["batch"]) >= BATCH_SIZE:
|
| 1094 |
with st.spinner("Сохраняю в reviews repo…"):
|
| 1095 |
ok, msg = commit_batch()
|
| 1096 |
if ok:
|
| 1097 |
flushed = True
|
| 1098 |
-
st.success("Сохранено")
|
| 1099 |
else:
|
| 1100 |
st.error(f"Не удалось записать в reviews repo: {msg}")
|
| 1101 |
-
st.warning("Оценка сохранена локально как pending. Скачайте pending reviews или синхронизи
|
| 1102 |
|
| 1103 |
-
|
| 1104 |
-
|
|
|
|
|
|
|
| 1105 |
|
| 1106 |
-
|
| 1107 |
-
|
|
|
|
| 1108 |
st.rerun()
|
| 1109 |
|
| 1110 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import os
|
| 2 |
import json
|
| 3 |
import uuid
|
| 4 |
import gzip
|
| 5 |
import tempfile
|
| 6 |
import datetime as dt
|
| 7 |
+
import math
|
| 8 |
from typing import List, Dict, Any, Optional, Tuple, Set
|
| 9 |
|
| 10 |
import pandas as pd
|
|
|
|
| 12 |
from huggingface_hub import hf_hub_download, HfApi
|
| 13 |
|
| 14 |
# =========================
|
| 15 |
+
# ENV CONFIG (HF Space Variables / Secrets)
|
| 16 |
# =========================
|
| 17 |
PUBLICATIONS_REPO = os.environ.get("PUBLICATIONS_REPO", "").strip()
|
| 18 |
REVIEWS_REPO = os.environ.get("REVIEWS_REPO", "").strip()
|
| 19 |
HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HF_API_TOKEN")
|
| 20 |
+
ALLOW_CREATE_REVIEWS_REPO = (os.environ.get("ALLOW_CREATE_REVIEWS_REPO", "0").strip().lower() in ("1","true","yes","y","on"))
|
| 21 |
+
REVIEWS_PRIVATE = (os.environ.get("REVIEWS_PRIVATE", "1").strip().lower() in ("1","true","yes","y","on"))
|
| 22 |
|
| 23 |
+
# Надёжность по умолчанию: сразу пишем каждый review
|
|
|
|
|
|
|
| 24 |
BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "1"))
|
|
|
|
| 25 |
|
| 26 |
+
PUB_DIR_REGISTRY_PATH = os.environ.get("PUB_DIR_REGISTRY_PATH", "dir_registry.json")
|
| 27 |
+
PUB_CANDIDATES_PREFIX = os.environ.get("PUB_CANDIDATES_PREFIX", "candidates")
|
| 28 |
+
REVIEWS_LOG_PREFIX = os.environ.get("REVIEWS_LOG_PREFIX", "reviews_log")
|
| 29 |
|
| 30 |
+
# Паттерны поиска candidates файлов (поддержка разных схем данных)
|
| 31 |
+
# Можно переопределить в Space Variables:
|
| 32 |
+
# PUB_CANDIDATES_PATTERNS="{prefix}/{dir_id}_top500.jsonl|{prefix}/{dir_id}.jsonl|manual_candidates/{dir_id}.jsonl|manual_candidates/{dir_id}_top500.jsonl"
|
| 33 |
PUB_CANDIDATES_PATTERNS = os.environ.get(
|
| 34 |
"PUB_CANDIDATES_PATTERNS",
|
| 35 |
+
"{prefix}/{dir_id}_top500.jsonl|{prefix}/{dir_id}_top500.jsonl.gz|{prefix}/{dir_id}.jsonl|{prefix}/{dir_id}.jsonl.gz|manual_candidates/{dir_id}.jsonl|manual_candidates/{dir_id}.jsonl.gz"
|
| 36 |
).strip()
|
| 37 |
|
| 38 |
+
|
| 39 |
api = HfApi(token=HF_TOKEN)
|
| 40 |
|
| 41 |
+
st.set_page_config(page_title="ISS-GR Screening", layout="wide")
|
| 42 |
+
|
| 43 |
|
| 44 |
# =========================
|
| 45 |
# Helpers
|
| 46 |
# =========================
|
| 47 |
|
| 48 |
+
def check_dataset_repo(repo_id: str, token: Optional[str]) -> Tuple[bool, str]:
|
| 49 |
+
"""
|
| 50 |
+
Проверяет существование датасет-репозитория (repo_type="dataset").
|
| 51 |
+
Возвращает (ok, message). Если ok=False, message содержит причину.
|
| 52 |
+
"""
|
| 53 |
+
try:
|
| 54 |
+
api_local = HfApi(token=token) if token else HfApi()
|
| 55 |
+
_ = api_local.repo_info(repo_id=repo_id, repo_type="dataset")
|
| 56 |
+
return True, "OK"
|
| 57 |
+
except Exception as e:
|
| 58 |
+
msg = str(e)
|
| 59 |
+
# huggingface_hub кидает 404/Repository Not Found
|
| 60 |
+
return False, msg
|
| 61 |
+
|
| 62 |
+
def maybe_create_reviews_repo(repo_id: str) -> Tuple[bool, str]:
|
| 63 |
+
"""
|
| 64 |
+
Пытается создать reviews dataset repo, если включён ALLOW_CREATE_REVIEWS_REPO.
|
| 65 |
+
Возвращает (ok, message).
|
| 66 |
+
"""
|
| 67 |
+
if not ALLOW_CREATE_REVIEWS_REPO:
|
| 68 |
+
return False, "ALLOW_CREATE_REVIEWS_REPO=0"
|
| 69 |
+
if not HF_TOKEN:
|
| 70 |
+
return False, "HF_TOKEN отсутствует (нужен для создания репозитория)"
|
| 71 |
+
try:
|
| 72 |
+
api.create_repo(
|
| 73 |
+
repo_id=repo_id,
|
| 74 |
+
repo_type="dataset",
|
| 75 |
+
private=bool(REVIEWS_PRIVATE),
|
| 76 |
+
exist_ok=True,
|
| 77 |
+
)
|
| 78 |
+
return True, "created_or_exists"
|
| 79 |
+
except Exception as e:
|
| 80 |
+
return False, str(e)
|
| 81 |
+
|
| 82 |
def require_env(name: str, value: str) -> None:
|
| 83 |
if not value:
|
| 84 |
st.error(f"Не задано **{name}**. Укажи в Space Settings → Variables.")
|
| 85 |
st.stop()
|
| 86 |
|
|
|
|
| 87 |
def safe_int(x) -> Optional[int]:
|
| 88 |
try:
|
| 89 |
if x is None or pd.isna(x):
|
|
|
|
| 92 |
except Exception:
|
| 93 |
return None
|
| 94 |
|
|
|
|
| 95 |
def safe_float(x) -> Optional[float]:
|
| 96 |
try:
|
| 97 |
if x is None or pd.isna(x):
|
|
|
|
| 100 |
except Exception:
|
| 101 |
return None
|
| 102 |
|
|
|
|
| 103 |
def doi_url(doi: Optional[str]) -> Optional[str]:
|
| 104 |
if not doi:
|
| 105 |
return None
|
| 106 |
doi = str(doi).strip()
|
|
|
|
|
|
|
| 107 |
if doi.startswith("http://") or doi.startswith("https://"):
|
| 108 |
return doi
|
| 109 |
return f"https://doi.org/{doi}"
|
| 110 |
|
|
|
|
| 111 |
def normalize_dir_id(dir_id: str, pad2: bool) -> str:
|
| 112 |
+
"""
|
| 113 |
+
Поддержка форматов DIR1..DIR14 и DIR01..DIR14.
|
| 114 |
+
pad2=True -> DIR01
|
| 115 |
+
pad2=False -> DIR1
|
| 116 |
+
"""
|
| 117 |
if not isinstance(dir_id, str):
|
| 118 |
return str(dir_id)
|
| 119 |
s = dir_id.strip().upper()
|
|
|
|
| 123 |
try:
|
| 124 |
n = int(tail)
|
| 125 |
except Exception:
|
| 126 |
+
# DIRXX (не число) — как есть
|
| 127 |
return dir_id.strip()
|
| 128 |
return f"DIR{n:02d}" if pad2 else f"DIR{n}"
|
| 129 |
|
|
|
|
| 130 |
def dir_variants(dir_id: str) -> List[str]:
|
| 131 |
+
"""
|
| 132 |
+
Список вариантов идентификатора DIR, чтобы подхватить разные имена файлов/папок.
|
| 133 |
+
"""
|
| 134 |
if not isinstance(dir_id, str):
|
| 135 |
return [str(dir_id)]
|
| 136 |
a = dir_id.strip()
|
| 137 |
b = normalize_dir_id(a, pad2=True)
|
| 138 |
c = normalize_dir_id(a, pad2=False)
|
| 139 |
+
out = []
|
| 140 |
for x in [a, b, c]:
|
| 141 |
if x and x not in out:
|
| 142 |
out.append(x)
|
| 143 |
return out
|
| 144 |
|
| 145 |
+
def dir_no(dir_id: str) -> str:
|
| 146 |
+
if isinstance(dir_id, str) and dir_id.upper().startswith("DIR"):
|
| 147 |
+
return dir_id[3:]
|
| 148 |
+
return str(dir_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
|
| 150 |
def join_terms(term_list: Any) -> str:
|
| 151 |
if not term_list:
|
| 152 |
return "—"
|
| 153 |
+
out = []
|
| 154 |
for x in term_list:
|
| 155 |
if isinstance(x, dict):
|
| 156 |
t = x.get("t")
|
|
|
|
| 160 |
out.append(str(x))
|
| 161 |
return "; ".join(out) if out else "—"
|
| 162 |
|
|
|
|
| 163 |
def topics_line(topics: Any) -> str:
|
| 164 |
if not topics:
|
| 165 |
return "—"
|
| 166 |
+
out = []
|
| 167 |
+
for t in (topics[:3] if isinstance(topics, list) else []):
|
| 168 |
+
if isinstance(t, dict):
|
| 169 |
+
tid = t.get("topic_id") or t.get("id")
|
| 170 |
+
name = t.get("topic_name") or t.get("display_name")
|
| 171 |
+
if tid and name:
|
| 172 |
+
out.append(f"{tid} — {name}")
|
| 173 |
+
elif name:
|
| 174 |
+
out.append(str(name))
|
| 175 |
+
elif tid:
|
| 176 |
+
out.append(str(tid))
|
| 177 |
+
else:
|
| 178 |
+
out.append(str(t))
|
|
|
|
| 179 |
return "; ".join(out) if out else "—"
|
| 180 |
|
| 181 |
+
def get_reviewer() -> str:
|
| 182 |
+
default = os.environ.get("REVIEWER", "")
|
| 183 |
+
v = st.sidebar.text_input("Рецензент (имя/ник)", value=default).strip()
|
| 184 |
+
return v or "anonymous"
|
| 185 |
|
| 186 |
def decode_abstract(abstract_inverted_index: Optional[dict]) -> str:
|
| 187 |
if not abstract_inverted_index or not isinstance(abstract_inverted_index, dict):
|
| 188 |
return ""
|
| 189 |
+
pos_to_word = {}
|
| 190 |
for w, positions in abstract_inverted_index.items():
|
| 191 |
if not isinstance(positions, list):
|
| 192 |
continue
|
| 193 |
for p in positions:
|
| 194 |
+
pos_to_word[p] = w
|
|
|
|
|
|
|
|
|
|
| 195 |
return " ".join(pos_to_word[p] for p in sorted(pos_to_word.keys())) if pos_to_word else ""
|
| 196 |
|
|
|
|
| 197 |
def get_authors_and_abstract(row: Dict[str, Any]) -> Tuple[str, str]:
|
| 198 |
+
"""
|
| 199 |
+
Offline-first: берём авторов/аннотацию ТОЛЬКО из candidates JSONL.
|
| 200 |
+
Никаких обращений к OpenAlex API.
|
| 201 |
+
"""
|
| 202 |
+
# authors
|
| 203 |
+
authors = row.get('authors') or row.get('author_names') or row.get('authors_str')
|
| 204 |
if not authors:
|
| 205 |
+
# Иногда candidates могут содержать OpenAlex-подобный authroships, но уже на входе (без сети)
|
| 206 |
+
auths = row.get('authorships')
|
| 207 |
if isinstance(auths, list):
|
| 208 |
+
names = []
|
| 209 |
for a in auths:
|
| 210 |
if not isinstance(a, dict):
|
| 211 |
continue
|
| 212 |
+
an = ((a.get('author') or {}).get('display_name') or '').strip()
|
| 213 |
if an:
|
| 214 |
names.append(an)
|
| 215 |
if names:
|
| 216 |
+
authors = ', '.join(names[:12]) + (', и др.' if len(names) > 12 else '')
|
|
|
|
| 217 |
if isinstance(authors, list):
|
| 218 |
+
authors = ', '.join([str(a) for a in authors if a])
|
| 219 |
+
authors_str = authors.strip() if isinstance(authors, str) and authors.strip() else ''
|
| 220 |
|
| 221 |
+
# abstract
|
| 222 |
+
abstract = row.get('abstract') or row.get('abstract_text')
|
| 223 |
+
if not abstract and isinstance(row.get('abstract_inverted_index'), dict):
|
| 224 |
+
abstract = decode_abstract(row.get('abstract_inverted_index'))
|
| 225 |
+
abstract_str = abstract.strip() if isinstance(abstract, str) and abstract.strip() else ''
|
|
|
|
| 226 |
|
| 227 |
+
return authors_str or '—', abstract_str or '—'
|
| 228 |
|
| 229 |
def open_jsonl_any(path: str) -> List[Dict[str, Any]]:
|
| 230 |
+
rows = []
|
| 231 |
if path.endswith(".gz"):
|
| 232 |
with gzip.open(path, "rt", encoding="utf-8") as f:
|
| 233 |
for line in f:
|
|
|
|
| 242 |
rows.append(json.loads(line))
|
| 243 |
return rows
|
| 244 |
|
|
|
|
| 245 |
def normalize_candidate_row(obj: Dict[str, Any], selected_dir: str) -> Dict[str, Any]:
|
| 246 |
+
"""
|
| 247 |
+
Приводит разные схемы данных к единому набору полей, используемых UI.
|
| 248 |
+
Поддерживаем:
|
| 249 |
+
- старый формат candidates: work_id/year/cited_by/dir_score/match_score
|
| 250 |
+
- новый формат паспорта: source_id/openalex_work_id/publication_year/cited_by_count/dir_native_components/quality_weight
|
| 251 |
+
"""
|
| 252 |
out = dict(obj)
|
| 253 |
|
| 254 |
+
# --- IDs ---
|
| 255 |
source_id = out.get("source_id") or out.get("Source ID")
|
| 256 |
if isinstance(source_id, str) and source_id.strip():
|
| 257 |
out["source_id"] = source_id.strip()
|
| 258 |
|
| 259 |
work_id = out.get("work_id")
|
| 260 |
if not isinstance(work_id, str) or not work_id.strip():
|
| 261 |
+
# паспорта
|
| 262 |
wid = out.get("openalex_work_id") or out.get("OpenAlex work ID") or out.get("dup_work_id")
|
| 263 |
if isinstance(wid, str) and wid.strip():
|
| 264 |
work_id = wid.strip()
|
| 265 |
+
out["work_id"] = work_id if isinstance(work_id, str) else None
|
| 266 |
|
| 267 |
+
# --- Title ---
|
| 268 |
title = out.get("title") or out.get("display_name") or out.get("Title")
|
| 269 |
out["title"] = str(title).strip() if title else "—"
|
| 270 |
|
| 271 |
+
# --- DOI ---
|
| 272 |
doi = out.get("doi") or out.get("DOI")
|
| 273 |
out["doi"] = str(doi).strip() if doi else None
|
| 274 |
|
| 275 |
+
# --- Abstract (offline) ---
|
| 276 |
if out.get("abstract") is None:
|
| 277 |
ab = out.get("abstract_text")
|
| 278 |
if isinstance(ab, str) and ab.strip():
|
|
|
|
| 280 |
elif isinstance(out.get("abstract_inverted_index"), dict):
|
| 281 |
out["abstract"] = decode_abstract(out.get("abstract_inverted_index")) or None
|
| 282 |
|
| 283 |
+
|
| 284 |
+
# --- Year / cited_by ---
|
| 285 |
year = out.get("year")
|
| 286 |
if year is None:
|
| 287 |
year = out.get("publication_year") or out.get("Publication year")
|
|
|
|
| 292 |
cited_by = out.get("cited_by_count") or out.get("Cited by count")
|
| 293 |
out["cited_by"] = safe_int(cited_by)
|
| 294 |
|
| 295 |
+
# --- Scores ---
|
| 296 |
+
# dir_score: приоритет
|
| 297 |
+
# 1) явное поле dir_score (старый candidates)
|
| 298 |
ds = out.get("dir_score")
|
| 299 |
+
# 2) паспорта: dir_native_components.score (если selected_dir совпадает с родным)
|
| 300 |
if ds is None:
|
| 301 |
comp = out.get("dir_native_components") or {}
|
| 302 |
if isinstance(comp, dict) and comp.get("score") is not None:
|
| 303 |
ds = comp.get("score")
|
| 304 |
+
# 3) паспорта: dir_scores[selected_dir]
|
| 305 |
if ds is None:
|
| 306 |
dsd = out.get("dir_scores") or {}
|
| 307 |
if isinstance(dsd, dict):
|
| 308 |
+
# пробуем разные варианты dir_id
|
| 309 |
for dv in dir_variants(selected_dir):
|
| 310 |
if dv in dsd:
|
| 311 |
ds = dsd.get(dv)
|
| 312 |
break
|
| 313 |
out["dir_score"] = safe_float(ds)
|
| 314 |
|
| 315 |
+
# match_score: старое поле; если его нет — используем delta или quality_weight*100 для “второй метрики”
|
| 316 |
ms = out.get("match_score")
|
| 317 |
if ms is None:
|
| 318 |
comp = out.get("dir_native_components") or {}
|
|
|
|
| 325 |
ms = None
|
| 326 |
out["match_score"] = safe_float(ms)
|
| 327 |
|
| 328 |
+
# quality_weight / components
|
| 329 |
+
qw = out.get("quality_weight")
|
| 330 |
+
out["quality_weight"] = safe_float(qw)
|
| 331 |
+
comp = out.get("dir_native_components")
|
| 332 |
+
out["dir_native_components"] = comp if isinstance(comp, dict) else {}
|
| 333 |
|
| 334 |
+
# links: pdf/primary location
|
| 335 |
out["pdf_url"] = out.get("pdf_url") or out.get("PDF URL")
|
| 336 |
out["primary_location_url"] = out.get("primary_location_url") or out.get("Primary location URL")
|
| 337 |
|
| 338 |
return out
|
| 339 |
|
| 340 |
+
def review_key(row: Dict[str, Any]) -> str:
|
| 341 |
+
"""
|
| 342 |
+
Ключ "уже оценено":
|
| 343 |
+
- если есть source_id — используем его (если вы теперь опираетесь на source_id)
|
| 344 |
+
- иначе work_id
|
| 345 |
+
"""
|
| 346 |
+
sid = row.get("source_id")
|
| 347 |
if isinstance(sid, str) and sid.strip():
|
| 348 |
+
return f"SRC::{sid.strip()}"
|
| 349 |
+
wid = row.get("work_id")
|
| 350 |
if isinstance(wid, str) and wid.strip():
|
| 351 |
+
return f"W::{wid.strip()}"
|
| 352 |
+
return ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 353 |
|
| 354 |
|
| 355 |
# =========================
|
| 356 |
+
# Publications (read)
|
| 357 |
# =========================
|
|
|
|
| 358 |
@st.cache_data(show_spinner=False)
|
| 359 |
def load_dir_registry(repo_id: str, filename: str) -> List[Dict[str, Any]]:
|
| 360 |
path = hf_hub_download(repo_id=repo_id, filename=filename, repo_type="dataset", token=HF_TOKEN)
|
| 361 |
with open(path, "r", encoding="utf-8") as f:
|
| 362 |
return json.load(f)
|
| 363 |
|
|
|
|
| 364 |
@st.cache_data(show_spinner=False)
|
| 365 |
def load_candidates(repo_id: str, dir_id: str, prefix: str) -> pd.DataFrame:
|
| 366 |
+
"""
|
| 367 |
+
Поддержка нескольких паттернов и вариантов DIR id (DIR1 vs DIR01).
|
| 368 |
+
"""
|
| 369 |
patterns = [p.strip() for p in PUB_CANDIDATES_PATTERNS.split("|") if p.strip()]
|
| 370 |
+
tried = []
|
| 371 |
+
rows = None
|
| 372 |
|
| 373 |
for dv in dir_variants(dir_id):
|
| 374 |
for pat in patterns:
|
|
|
|
| 377 |
try:
|
| 378 |
path = hf_hub_download(repo_id=repo_id, filename=fname, repo_type="dataset", token=HF_TOKEN)
|
| 379 |
rows = open_jsonl_any(path)
|
| 380 |
+
if rows:
|
| 381 |
+
break
|
| 382 |
+
# если файл пустой — тоже считаем найденным
|
| 383 |
+
if rows == []:
|
| 384 |
+
break
|
| 385 |
except Exception:
|
| 386 |
continue
|
| 387 |
if rows is not None:
|
|
|
|
| 390 |
if rows is None:
|
| 391 |
raise FileNotFoundError("Не найден candidates файл. Пробовали:\n" + "\n".join(tried[:30]) + ("\n..." if len(tried) > 30 else ""))
|
| 392 |
|
| 393 |
+
# normalize schema
|
| 394 |
normed = [normalize_candidate_row(r, dir_id) for r in rows if isinstance(r, dict)]
|
| 395 |
df = pd.DataFrame(normed)
|
| 396 |
|
| 397 |
+
# numeric columns
|
| 398 |
for col in ["dir_score", "match_score", "cited_by", "year", "quality_weight"]:
|
| 399 |
if col in df.columns:
|
| 400 |
df[col] = pd.to_numeric(df[col], errors="coerce")
|
|
|
|
| 403 |
|
| 404 |
|
| 405 |
# =========================
|
| 406 |
+
# Reviews (read on-demand)
|
| 407 |
# =========================
|
|
|
|
| 408 |
@st.cache_data(show_spinner=False)
|
| 409 |
def list_review_files(repo_id: str, dir_id: str, prefix: str) -> List[str]:
|
| 410 |
files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
|
| 411 |
+
# в reviews repo папка может быть DIR1 или DIR01 — подхватываем оба
|
| 412 |
needles = [f"/{dv}/" for dv in dir_variants(dir_id)]
|
| 413 |
+
out = []
|
| 414 |
for p in files:
|
| 415 |
if not (p.startswith(prefix + "/") and p.endswith(".jsonl")):
|
| 416 |
continue
|
|
|
|
| 418 |
out.append(p)
|
| 419 |
return out
|
| 420 |
|
|
|
|
| 421 |
@st.cache_data(show_spinner=False)
|
| 422 |
+
def load_reviewed_keys(repo_id: str, dir_id: str, prefix: str) -> Set[str]:
|
| 423 |
+
"""
|
| 424 |
+
Поддержка миграции:
|
| 425 |
+
- старые review содержали только work_id -> помечаем как W::<id>
|
| 426 |
+
- новые могут содержать source_id -> помечаем как SRC::<id>
|
|
|
|
|
|
|
| 427 |
"""
|
| 428 |
try:
|
| 429 |
files = list_review_files(repo_id, dir_id, prefix)
|
| 430 |
except Exception:
|
| 431 |
+
return set()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 432 |
|
| 433 |
+
reviewed: Set[str] = set()
|
| 434 |
for relpath in files:
|
| 435 |
try:
|
| 436 |
path = hf_hub_download(repo_id=repo_id, filename=relpath, repo_type="dataset", token=HF_TOKEN)
|
|
|
|
| 442 |
obj = json.loads(line)
|
| 443 |
|
| 444 |
sid = obj.get("source_id")
|
|
|
|
|
|
|
| 445 |
if isinstance(sid, str) and sid.strip():
|
| 446 |
+
reviewed.add(f"SRC::{sid.strip()}")
|
| 447 |
+
|
| 448 |
+
wid = obj.get("work_id")
|
| 449 |
if isinstance(wid, str) and wid.strip():
|
| 450 |
+
reviewed.add(f"W::{wid.strip()}")
|
|
|
|
|
|
|
| 451 |
except Exception:
|
| 452 |
continue
|
| 453 |
+
return reviewed
|
|
|
|
| 454 |
|
| 455 |
|
| 456 |
# =========================
|
| 457 |
+
# Reviews (write)
|
| 458 |
# =========================
|
|
|
|
| 459 |
def push_batch_to_reviews_repo(repo_id: str, dir_id: str, prefix: str, batch: List[Dict[str, Any]]) -> None:
|
| 460 |
if not batch:
|
| 461 |
return
|
| 462 |
if not HF_TOKEN:
|
| 463 |
raise RuntimeError("Нет HF_TOKEN (Secret) — нельзя записывать в reviews dataset.")
|
|
|
|
| 464 |
today = dt.date.today().isoformat()
|
| 465 |
batch_id = str(uuid.uuid4())
|
| 466 |
+
|
| 467 |
+
# Пишем в dir_id как есть (но если dir_id может быть DIR01, а у вас в БД DIR1 — нормализуйте в UI выбором)
|
| 468 |
canonical_dir = normalize_dir_id(dir_id, pad2=False)
|
| 469 |
path_in_repo = f"{prefix}/{today}/{canonical_dir}/{batch_id}.jsonl"
|
| 470 |
|
|
|
|
| 483 |
|
| 484 |
|
| 485 |
# =========================
|
| 486 |
+
# Sidebar
|
| 487 |
# =========================
|
| 488 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 489 |
# =========================
|
| 490 |
+
# MAIN APP (offline-first)
|
| 491 |
# =========================
|
| 492 |
|
| 493 |
with st.sidebar:
|
| 494 |
+
st.title("ISS-GR Скрининг (offline-first)")
|
| 495 |
|
| 496 |
require_env("PUBLICATIONS_REPO", PUBLICATIONS_REPO)
|
| 497 |
require_env("REVIEWS_REPO", REVIEWS_REPO)
|
| 498 |
|
| 499 |
+
# Проверяем наличие репозиториев (чтобы не ловить 404 при сохранении)
|
| 500 |
pub_ok, pub_msg = check_dataset_repo(PUBLICATIONS_REPO, HF_TOKEN)
|
| 501 |
if not pub_ok:
|
| 502 |
st.error("PUBLICATIONS_REPO недоступен как dataset repo. Проверь repo_id и доступ.\n\n" + pub_msg)
|
|
|
|
| 504 |
|
| 505 |
rev_ok, rev_msg = check_dataset_repo(REVIEWS_REPO, HF_TOKEN)
|
| 506 |
if not rev_ok:
|
| 507 |
+
created, cmsg = maybe_create_reviews_repo(REVIEWS_REPO)
|
| 508 |
if created:
|
| 509 |
rev_ok2, rev_msg2 = check_dataset_repo(REVIEWS_REPO, HF_TOKEN)
|
| 510 |
if not rev_ok2:
|
| 511 |
st.error("REVIEWS_REPO недоступен после create_repo.\n\n" + rev_msg2)
|
| 512 |
st.stop()
|
| 513 |
+
else:
|
| 514 |
+
st.success("REVIEWS_REPO создан/доступен.")
|
| 515 |
else:
|
| 516 |
st.error(
|
| 517 |
"REVIEWS_REPO недоступен (404/нет доступа).\n\n"
|
| 518 |
+
"Что проверить:\n"
|
| 519 |
+
"1) В Space Variables: REVIEWS_REPO = <owner>/<dataset_name> (именно dataset).\n"
|
| 520 |
+
"2) Репозиторий реально существует на Hub.\n"
|
| 521 |
+
"3) Если repo приватный — в Secrets должен быть HF_TOKEN с правами write на этот dataset.\n"
|
| 522 |
+
"4) Если repo ещё не создан — включи ALLOW_CREATE_REVIEWS_REPO=1 (опционально) или создай вручную.\n\n"
|
| 523 |
+ rev_msg
|
| 524 |
)
|
| 525 |
st.stop()
|
| 526 |
|
| 527 |
+
# =========================
|
|
|
|
| 528 |
|
| 529 |
+
# =========================
|
| 530 |
+
# Load DIR registry + UI (offline-first)
|
| 531 |
+
# =========================
|
| 532 |
+
dirs = load_dir_registry(PUBLICATIONS_REPO, PUB_DIR_REGISTRY_PATH)
|
| 533 |
+
dir_ids = [d.get("dir_id") for d in dirs if d.get("dir_id")]
|
| 534 |
+
if not dir_ids:
|
| 535 |
+
st.error("В dir_registry.json не найдено ни одного dir_id.")
|
| 536 |
+
st.stop()
|
| 537 |
|
| 538 |
+
dir_map = {d.get("dir_id"): d for d in dirs if d.get("dir_id")}
|
| 539 |
|
| 540 |
+
def _fmt_dir(did: str) -> str:
|
| 541 |
+
meta = dir_map.get(did) or {}
|
| 542 |
+
name = meta.get("dir_name") or "—"
|
| 543 |
+
return f"DIR-{str(dir_no(did)).zfill(2)} — {name}"
|
|
|
|
|
|
|
| 544 |
|
| 545 |
+
st.title("Скоринг публикаций")
|
| 546 |
+
|
| 547 |
+
# Выбор DIR — в основном интерфейсе (не в сайдбаре)
|
| 548 |
+
selected_dir = st.selectbox("DIR", dir_ids, index=0, format_func=_fmt_dir)
|
| 549 |
+
dir_meta = dir_map.get(selected_dir) or {}
|
| 550 |
|
| 551 |
# =========================
|
| 552 |
+
# Sidebar (минимум)
|
| 553 |
# =========================
|
| 554 |
+
with st.sidebar:
|
| 555 |
+
reviewer = get_reviewer()
|
| 556 |
+
st.caption(f"BATCH_SIZE={BATCH_SIZE} (настройка через переменную окружения)")
|
| 557 |
+
st.divider()
|
| 558 |
+
flush_now = st.button("⬆️ Синхронизировать pending сейчас", use_container_width=True)
|
| 559 |
+
clear_cache = st.button("🧹 Сбросить кэш", use_container_width=True)
|
| 560 |
|
| 561 |
+
if clear_cache:
|
| 562 |
load_dir_registry.clear()
|
| 563 |
load_candidates.clear()
|
| 564 |
+
if hasattr(list_review_files, "clear"):
|
| 565 |
+
list_review_files.clear()
|
| 566 |
+
if hasattr(load_reviewed_keys, "clear"):
|
| 567 |
+
load_reviewed_keys.clear()
|
| 568 |
+
if hasattr(load_review_index, "clear"):
|
| 569 |
+
load_review_index.clear()
|
| 570 |
for k in [
|
| 571 |
+
"reviewed_remote",
|
| 572 |
+
"reviewed_remote_dir",
|
| 573 |
+
"review_index_remote",
|
| 574 |
+
"review_index_local",
|
| 575 |
"reviewed_local_committed",
|
| 576 |
"reviewed_local_pending",
|
|
|
|
|
|
|
| 577 |
"batch",
|
| 578 |
]:
|
| 579 |
if k in st.session_state:
|
| 580 |
del st.session_state[k]
|
| 581 |
+
st.toast("Кэш очищен.")
|
|
|
|
| 582 |
st.rerun()
|
| 583 |
|
|
|
|
| 584 |
# =========================
|
| 585 |
+
# DIR header
|
| 586 |
# =========================
|
| 587 |
+
defaults = (dir_meta.get("defaults") or {})
|
| 588 |
+
year_from = defaults.get("year_from")
|
| 589 |
+
year_to = defaults.get("year_to")
|
| 590 |
+
|
| 591 |
+
terms = (dir_meta.get("terms") or {})
|
| 592 |
+
anchor_str = join_terms(terms.get("anchor"))
|
| 593 |
+
support_str = join_terms(terms.get("support"))
|
| 594 |
+
noise_str = join_terms(terms.get("noise"))
|
| 595 |
+
topics_str = topics_line(dir_meta.get("topics") or [])
|
| 596 |
+
|
| 597 |
+
dir_desc = dir_meta.get("dir_description", "—")
|
| 598 |
+
|
| 599 |
+
st.markdown(
|
| 600 |
+
f"""
|
| 601 |
+
<div style="padding: 14px 16px; border: 1px solid rgba(49,51,63,0.2); border-radius: 12px; background: rgba(49,51,63,0.04);">
|
| 602 |
+
<div style="margin-top: 0px;">
|
| 603 |
+
<div style="font-size: 12px; font-weight: 700; text-transform: uppercase; opacity: 0.7;">Краткое описание</div>
|
| 604 |
+
<div style="margin-top: 4px; font-size: 14px; line-height: 1.35;">{dir_desc}</div>
|
| 605 |
+
</div>
|
| 606 |
+
</div>
|
| 607 |
+
""",
|
| 608 |
+
unsafe_allow_html=True,
|
| 609 |
)
|
| 610 |
|
| 611 |
+
with st.expander("Детали DIR", expanded=True):
|
| 612 |
+
st.markdown(f"**Временной интервал:** {year_from} – {year_to}")
|
| 613 |
+
st.markdown(f"**Якоря:** {anchor_str}")
|
| 614 |
+
st.markdown(f"**Поддержка:** {support_str}")
|
| 615 |
+
st.markdown(f"**Шум:** {noise_str}")
|
| 616 |
+
st.markdown(f"**Topics поиска:** {topics_str}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 617 |
|
| 618 |
st.divider()
|
| 619 |
|
|
|
|
| 620 |
# =========================
|
| 621 |
+
# Load candidates + fixed sorting (A: dir_score ↓)
|
| 622 |
# =========================
|
|
|
|
| 623 |
try:
|
| 624 |
df = load_candidates(PUBLICATIONS_REPO, selected_dir, PUB_CANDIDATES_PREFIX)
|
| 625 |
except Exception as e:
|
|
|
|
| 627 |
st.stop()
|
| 628 |
|
| 629 |
if df.empty:
|
| 630 |
+
st.warning(f"Пустой список кандидатов для {selected_dir}.")
|
| 631 |
st.stop()
|
| 632 |
|
| 633 |
+
# Fixed order: dir_score ↓ (stable ties)
|
| 634 |
+
sort_cols: List[str] = []
|
| 635 |
+
sort_asc: List[bool] = []
|
| 636 |
+
if "dir_score" in df.columns:
|
| 637 |
+
sort_cols.append("dir_score"); sort_asc.append(False)
|
| 638 |
+
if "quality_weight" in df.columns:
|
| 639 |
+
sort_cols.append("quality_weight"); sort_asc.append(False)
|
| 640 |
+
if "match_score" in df.columns:
|
| 641 |
+
sort_cols.append("match_score"); sort_asc.append(False)
|
| 642 |
+
if "year" in df.columns:
|
| 643 |
+
sort_cols.append("year"); sort_asc.append(False)
|
| 644 |
+
if sort_cols:
|
| 645 |
+
df = df.sort_values(sort_cols, ascending=sort_asc)
|
| 646 |
|
| 647 |
df = df.reset_index(drop=True)
|
| 648 |
+
total = len(df)
|
| 649 |
+
if total == 0:
|
| 650 |
+
st.warning("Пустой список кандидатов.")
|
| 651 |
+
st.stop()
|
| 652 |
|
| 653 |
# =========================
|
| 654 |
+
# State: batch + local/remote indexes
|
| 655 |
# =========================
|
| 656 |
+
def ensure_state():
|
| 657 |
+
if "batch" not in st.session_state:
|
| 658 |
+
st.session_state["batch"] = []
|
| 659 |
+
if "reviewed_local_committed" not in st.session_state:
|
| 660 |
+
st.session_state["reviewed_local_committed"] = set()
|
| 661 |
+
if "reviewed_local_pending" not in st.session_state:
|
| 662 |
+
st.session_state["reviewed_local_pending"] = set()
|
| 663 |
+
if "reviewed_remote" not in st.session_state:
|
| 664 |
+
st.session_state["reviewed_remote"] = set()
|
| 665 |
+
if "reviewed_remote_dir" not in st.session_state:
|
| 666 |
+
st.session_state["reviewed_remote_dir"] = None
|
| 667 |
+
if "review_index_remote" not in st.session_state:
|
| 668 |
+
st.session_state["review_index_remote"] = {}
|
| 669 |
+
if "review_index_local" not in st.session_state:
|
| 670 |
+
st.session_state["review_index_local"] = {}
|
| 671 |
|
| 672 |
ensure_state()
|
| 673 |
|
| 674 |
+
def row_keys_all(r: Dict[str, Any]) -> Set[str]:
|
| 675 |
+
keys = set()
|
| 676 |
+
sid = r.get("source_id")
|
| 677 |
+
if isinstance(sid, str) and sid.strip():
|
| 678 |
+
keys.add(f"SRC::{sid.strip()}")
|
| 679 |
+
wid = r.get("work_id")
|
| 680 |
+
if isinstance(wid, str) and wid.strip():
|
| 681 |
+
keys.add(f"W::{wid.strip()}")
|
| 682 |
+
return keys
|
|
|
|
|
|
|
|
|
|
|
|
|
| 683 |
|
| 684 |
+
canonical_dir = normalize_dir_id(selected_dir, pad2=False)
|
| 685 |
|
| 686 |
+
# Загрузка сохранённых оценок (тихо; remote-статусы не показываем)
|
| 687 |
+
need_reload = (st.session_state["reviewed_remote_dir"] != canonical_dir)
|
| 688 |
+
if need_reload:
|
| 689 |
+
# Без спиннера и без UI-вывода: просто подготавливаем пропуск/предзаполнение
|
| 690 |
+
st.session_state["reviewed_remote"] = load_reviewed_keys(REVIEWS_REPO, selected_dir, REVIEWS_LOG_PREFIX)
|
| 691 |
+
st.session_state["review_index_remote"] = load_review_index(REVIEWS_REPO, selected_dir, REVIEWS_LOG_PREFIX)
|
| 692 |
+
st.session_state["reviewed_remote_dir"] = canonical_dir
|
| 693 |
|
| 694 |
+
reviewed_committed = set(st.session_state["reviewed_remote"]) | set(st.session_state["reviewed_local_committed"])
|
| 695 |
+
reviewed_pending = set(st.session_state["reviewed_local_pending"])
|
| 696 |
+
reviewed_effective = reviewed_committed | reviewed_pending
|
| 697 |
|
| 698 |
+
def status_of_row(r: Dict[str, Any]) -> str:
|
| 699 |
keys = row_keys_all(r)
|
| 700 |
if keys and any(k in reviewed_committed for k in keys):
|
| 701 |
return "✅"
|
|
|
|
| 703 |
return "🕓"
|
| 704 |
return "🆕"
|
| 705 |
|
| 706 |
+
def is_reviewed_effective(r: Dict[str, Any]) -> bool:
|
| 707 |
+
keys = row_keys_all(r)
|
| 708 |
+
return bool(keys and any(k in reviewed_effective for k in keys))
|
| 709 |
+
|
| 710 |
+
def last_review_for_row(r: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
| 711 |
+
"""Берём последнюю известную оценку (pending/local -> remote)."""
|
| 712 |
+
keys = list(row_keys_all(r))
|
| 713 |
+
if not keys:
|
| 714 |
+
return None
|
| 715 |
+
|
| 716 |
+
best = None
|
| 717 |
+
best_ts = ""
|
| 718 |
+
# local index first
|
| 719 |
+
for k in keys:
|
| 720 |
+
obj = st.session_state["review_index_local"].get(k)
|
| 721 |
+
if isinstance(obj, dict):
|
| 722 |
+
t = str(obj.get("ts_utc") or obj.get("ts") or "")
|
| 723 |
+
if t > best_ts:
|
| 724 |
+
best_ts = t
|
| 725 |
+
best = obj
|
| 726 |
+
# remote index fallback
|
| 727 |
+
for k in keys:
|
| 728 |
+
obj = st.session_state["review_index_remote"].get(k)
|
| 729 |
+
if isinstance(obj, dict):
|
| 730 |
+
t = str(obj.get("ts_utc") or obj.get("ts") or "")
|
| 731 |
+
if t > best_ts:
|
| 732 |
+
best_ts = t
|
| 733 |
+
best = obj
|
| 734 |
+
return best
|
| 735 |
+
|
| 736 |
+
# =========================
|
| 737 |
+
# Sidebar: statuses (без remote)
|
| 738 |
+
# =========================
|
| 739 |
+
def split_key_set(keys: Set[str]) -> Tuple[Set[str], Set[str]]:
|
| 740 |
+
src_ids: Set[str] = set()
|
| 741 |
+
w_ids: Set[str] = set()
|
| 742 |
+
for k in keys:
|
| 743 |
+
if isinstance(k, str) and k.startswith("SRC::"):
|
| 744 |
+
src_ids.add(k[5:])
|
| 745 |
+
elif isinstance(k, str) and k.startswith("W::"):
|
| 746 |
+
w_ids.add(k[3:])
|
| 747 |
+
return src_ids, w_ids
|
| 748 |
+
|
| 749 |
+
comm_src_ids, comm_w_ids = split_key_set(reviewed_committed)
|
| 750 |
+
pend_src_ids, pend_w_ids = split_key_set(reviewed_pending)
|
| 751 |
|
|
|
|
| 752 |
src_series = df["source_id"].fillna("") if "source_id" in df.columns else pd.Series([""] * len(df))
|
| 753 |
wid_series = df["work_id"].fillna("") if "work_id" in df.columns else pd.Series([""] * len(df))
|
| 754 |
|
| 755 |
+
mask_committed = src_series.astype(str).isin(comm_src_ids) | wid_series.astype(str).isin(comm_w_ids)
|
| 756 |
+
mask_pending = src_series.astype(str).isin(pend_src_ids) | wid_series.astype(str).isin(pend_w_ids)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 757 |
mask_reviewed = mask_committed | mask_pending
|
| 758 |
|
| 759 |
committed_pub_count = int(mask_committed.sum())
|
|
|
|
| 761 |
unreviewed_pub_count = int((~mask_reviewed).sum())
|
| 762 |
|
| 763 |
with st.sidebar:
|
| 764 |
+
with st.expander("Статусы", expanded=False):
|
| 765 |
+
st.write(f"✅ сохранено: {committed_pub_count} публикаций")
|
| 766 |
+
st.write(f"🕓 pending: {pending_pub_count} публикаций")
|
| 767 |
|
| 768 |
if st.session_state.get("batch"):
|
| 769 |
pending_jsonl = "".join(json.dumps(x, ensure_ascii=False) + "\n" for x in st.session_state["batch"])
|
|
|
|
| 775 |
use_container_width=True,
|
| 776 |
)
|
| 777 |
|
|
|
|
| 778 |
# =========================
|
| 779 |
+
# Commit helper
|
| 780 |
# =========================
|
|
|
|
| 781 |
def commit_batch() -> Tuple[bool, str]:
|
| 782 |
batch = st.session_state.get("batch") or []
|
| 783 |
if not batch:
|
| 784 |
return False, "batch_empty"
|
|
|
|
| 785 |
try:
|
| 786 |
push_batch_to_reviews_repo(REVIEWS_REPO, selected_dir, REVIEWS_LOG_PREFIX, batch)
|
| 787 |
except Exception as e:
|
| 788 |
return False, str(e)
|
| 789 |
|
| 790 |
+
# Update local sets: pending -> committed (keys)
|
| 791 |
for obj in batch:
|
| 792 |
sid = obj.get("source_id")
|
| 793 |
wid = obj.get("work_id")
|
|
|
|
| 803 |
st.session_state["batch"] = []
|
| 804 |
return True, "ok"
|
| 805 |
|
| 806 |
+
# manual flush
|
| 807 |
+
if flush_now and st.session_state.get("batch"):
|
| 808 |
with st.spinner("Синхронизирую pending батч…"):
|
| 809 |
ok, msg = commit_batch()
|
| 810 |
if ok:
|
| 811 |
+
st.success("Pending батч синхронизирован.")
|
| 812 |
st.rerun()
|
| 813 |
else:
|
| 814 |
st.error(f"Не удалось синхронизировать: {msg}")
|
| 815 |
|
| 816 |
+
# recompute after possible commit
|
| 817 |
+
reviewed_committed = set(st.session_state["reviewed_remote"]) | set(st.session_state["reviewed_local_committed"])
|
| 818 |
+
reviewed_pending = set(st.session_state["reviewed_local_pending"])
|
| 819 |
+
reviewed_effective = reviewed_committed | reviewed_pending
|
| 820 |
|
| 821 |
# =========================
|
| 822 |
+
# Navigation window + layout
|
| 823 |
# =========================
|
|
|
|
| 824 |
idx_key = f"idx_{canonical_dir}"
|
| 825 |
if idx_key not in st.session_state:
|
| 826 |
st.session_state[idx_key] = 0
|
| 827 |
|
| 828 |
+
current_idx = int(st.session_state[idx_key])
|
| 829 |
+
current_idx = max(0, min(total - 1, current_idx))
|
| 830 |
+
st.session_state[idx_key] = current_idx
|
| 831 |
+
|
| 832 |
+
window_size = int(PAGE_WINDOW_SIZE) if PAGE_WINDOW_SIZE else 50
|
| 833 |
+
window_start = int(current_idx // window_size) * int(window_size)
|
| 834 |
+
window_end = min(window_start + int(window_size), total)
|
| 835 |
+
page_indices = list(range(window_start, window_end))
|
| 836 |
|
| 837 |
+
def fmt_idx(i: int) -> str:
|
| 838 |
+
r = df.iloc[i].to_dict()
|
| 839 |
+
ico = status_of_row(r)
|
| 840 |
+
year_val = safe_int(r.get("year"))
|
| 841 |
+
year = str(year_val) if year_val is not None else "—"
|
| 842 |
+
title = (str(r.get("title")) if r.get("title") is not None else "—")
|
| 843 |
+
title_short = title if len(title) <= 80 else title[:77] + "…"
|
| 844 |
+
return f"{ico} {i+1:03d} | {year} | {title_short}"
|
| 845 |
+
|
| 846 |
+
def next_unreviewed(from_idx: int) -> int:
|
| 847 |
+
i = max(0, from_idx)
|
| 848 |
while i < total:
|
|
|
|
|
|
|
| 849 |
r = df.iloc[i].to_dict()
|
| 850 |
if is_reviewed_effective(r):
|
| 851 |
i += 1
|
| 852 |
continue
|
| 853 |
return i
|
| 854 |
+
return total - 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 855 |
|
| 856 |
+
# main 2-column layout: left=list+scoring, right=publication
|
| 857 |
+
col_left, col_right = st.columns([1, 2], gap="large")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 858 |
|
| 859 |
+
with col_left:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 860 |
st.markdown("### Список публикаций")
|
| 861 |
+
st.caption(f"Окно: {window_start+1}–{window_end} из {total} • Неоценено: {unreviewed_pub_count}")
|
| 862 |
|
| 863 |
sel = st.selectbox(
|
| 864 |
+
"Публикации в текущем окне",
|
| 865 |
+
options=page_indices,
|
| 866 |
+
index=page_indices.index(current_idx) if current_idx in page_indices else 0,
|
| 867 |
format_func=fmt_idx,
|
| 868 |
label_visibility="collapsed",
|
| 869 |
)
|
|
|
|
| 872 |
st.session_state[idx_key] = int(sel)
|
| 873 |
st.rerun()
|
| 874 |
|
| 875 |
+
# актуальная строка
|
| 876 |
+
current_idx = int(st.session_state[idx_key])
|
| 877 |
+
row = df.iloc[current_idx].to_dict()
|
| 878 |
+
|
| 879 |
+
# предзаполнение score/comment: если есть оценка -> она, иначе 0/пусто
|
| 880 |
+
existing = last_review_for_row(row) or {}
|
| 881 |
+
existing_score = safe_int(existing.get("score"))
|
| 882 |
+
if existing_score is None:
|
| 883 |
+
existing_score = 0
|
| 884 |
+
existing_comment = str(existing.get("comment") or "")
|
| 885 |
+
|
| 886 |
+
score_key = f"score_{canonical_dir}_{current_idx}"
|
| 887 |
+
comment_key = f"com_{canonical_dir}_{current_idx}"
|
| 888 |
+
|
| 889 |
+
if score_key not in st.session_state:
|
| 890 |
+
st.session_state[score_key] = int(existing_score)
|
| 891 |
+
# защита от старых/некорректных значений
|
| 892 |
+
if int(st.session_state[score_key]) not in (-2, -1, 0, 1, 2):
|
| 893 |
+
st.session_state[score_key] = 0
|
| 894 |
+
|
| 895 |
+
if comment_key not in st.session_state:
|
| 896 |
+
st.session_state[comment_key] = existing_comment
|
| 897 |
+
|
| 898 |
+
st.markdown("### Оценка")
|
| 899 |
+
|
| 900 |
+
label_map = {
|
| 901 |
+
-2: "точно нет",
|
| 902 |
+
-1: "скорее нет",
|
| 903 |
+
0: "не знаю",
|
| 904 |
+
1: "скорее да",
|
| 905 |
+
2: "точно да",
|
| 906 |
+
}
|
| 907 |
+
|
| 908 |
+
score = st.radio(
|
| 909 |
+
"Оценка",
|
| 910 |
+
options=[-2, -1, 0, 1, 2],
|
| 911 |
+
format_func=lambda v: f"{v:+d} — {label_map.get(v, '')}".replace("+0", "0"),
|
| 912 |
+
index=[-2, -1, 0, 1, 2].index(int(st.session_state[score_key])),
|
| 913 |
+
key=score_key,
|
| 914 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 915 |
|
| 916 |
+
comment = st.text_area("Комментарий", height=140, key=comment_key)
|
| 917 |
|
| 918 |
+
b1, b2, b3 = st.columns([1, 2, 1])
|
|
|
|
| 919 |
|
| 920 |
+
with b1:
|
| 921 |
+
if st.button("⬅️ Назад", use_container_width=True):
|
| 922 |
+
st.session_state[idx_key] = max(0, current_idx - 1)
|
| 923 |
+
st.rerun()
|
| 924 |
|
| 925 |
+
with b2:
|
| 926 |
+
if st.button("✅ Сохрани��ь и далее", use_container_width=True):
|
| 927 |
+
# собираем мета для review
|
| 928 |
+
authors, abstract = get_authors_and_abstract(row)
|
| 929 |
|
| 930 |
+
y = safe_int(row.get("year"))
|
| 931 |
+
cites = safe_int(row.get("cited_by"))
|
| 932 |
+
dir_score = safe_float(row.get("dir_score"))
|
| 933 |
+
delta = safe_float(row.get("match_score"))
|
| 934 |
+
qw = safe_float(row.get("quality_weight"))
|
| 935 |
|
| 936 |
+
work_id = row.get("work_id") if isinstance(row.get("work_id"), str) else None
|
| 937 |
+
source_id = row.get("source_id") if isinstance(row.get("source_id"), str) else None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 938 |
|
|
|
|
| 939 |
review = {
|
| 940 |
"ts_utc": dt.datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
| 941 |
"dir_id": selected_dir,
|
|
|
|
| 953 |
"delta": delta,
|
| 954 |
"quality_weight": qw,
|
| 955 |
"components": row.get("dir_native_components") or {},
|
| 956 |
+
"rank": int(current_idx),
|
| 957 |
},
|
| 958 |
+
"score": int(score),
|
| 959 |
+
"comment": comment,
|
| 960 |
"reviewer": reviewer,
|
| 961 |
}
|
| 962 |
|
|
|
|
| 963 |
st.session_state["batch"].append(review)
|
| 964 |
|
| 965 |
+
# pending keys + local index for prefill
|
| 966 |
+
for k in row_keys_all(row):
|
| 967 |
+
st.session_state["reviewed_local_pending"].add(k)
|
| 968 |
+
st.session_state["review_index_local"][k] = review
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 969 |
|
| 970 |
+
# auto flush by BATCH_SIZE
|
| 971 |
flushed = False
|
| 972 |
if len(st.session_state["batch"]) >= BATCH_SIZE:
|
| 973 |
with st.spinner("Сохраняю в reviews repo…"):
|
| 974 |
ok, msg = commit_batch()
|
| 975 |
if ok:
|
| 976 |
flushed = True
|
| 977 |
+
st.success("Сохранено.")
|
| 978 |
else:
|
| 979 |
st.error(f"Не удалось записать в reviews repo: {msg}")
|
| 980 |
+
st.warning("Оценка сохранена локально как pending. Скачайте pending reviews или повторите синхронизацию.")
|
| 981 |
|
| 982 |
+
# переход: следующая НЕоценённая (без тумблера)
|
| 983 |
+
nxt = next_unreviewed(current_idx + 1)
|
| 984 |
+
st.session_state[idx_key] = nxt
|
| 985 |
+
st.rerun()
|
| 986 |
|
| 987 |
+
with b3:
|
| 988 |
+
if st.button("➡️ Далее", use_container_width=True):
|
| 989 |
+
st.session_state[idx_key] = min(total - 1, current_idx + 1)
|
| 990 |
st.rerun()
|
| 991 |
|
| 992 |
+
st.caption(f"Pending: {len(st.session_state.get('batch') or [])}")
|
| 993 |
+
|
| 994 |
+
with col_right:
|
| 995 |
+
# карточка публикации занимает максимум пространства справа
|
| 996 |
+
row = df.iloc[int(st.session_state[idx_key])].to_dict()
|
| 997 |
+
|
| 998 |
+
title = row.get("title") or "—"
|
| 999 |
+
authors, abstract = get_authors_and_abstract(row)
|
| 1000 |
+
|
| 1001 |
+
y = safe_int(row.get("year"))
|
| 1002 |
+
cites = safe_int(row.get("cited_by"))
|
| 1003 |
+
dir_score = safe_float(row.get("dir_score"))
|
| 1004 |
+
delta = safe_float(row.get("match_score"))
|
| 1005 |
+
qw = safe_float(row.get("quality_weight"))
|
| 1006 |
+
|
| 1007 |
+
work_id = row.get("work_id") if isinstance(row.get("work_id"), str) else None
|
| 1008 |
+
source_id = row.get("source_id") if isinstance(row.get("source_id"), str) else None
|
| 1009 |
+
|
| 1010 |
+
doi_link = doi_url(row.get("doi"))
|
| 1011 |
+
pdf_url = row.get("pdf_url")
|
| 1012 |
+
pl_url = row.get("primary_location_url")
|
| 1013 |
+
|
| 1014 |
+
oa = row.get("open_access") if isinstance(row.get("open_access"), dict) else {}
|
| 1015 |
+
oa_is = oa.get("is_oa") if isinstance(oa, dict) else None
|
| 1016 |
+
oa_status = oa.get("oa_status") if isinstance(oa, dict) else None
|
| 1017 |
+
oa_url = oa.get("oa_url") if isinstance(oa, dict) else None
|
| 1018 |
+
|
| 1019 |
+
keys = row_keys_all(row)
|
| 1020 |
+
committed = bool(keys and any(k in reviewed_committed for k in keys))
|
| 1021 |
+
pending = bool(keys and any(k in reviewed_pending for k in keys))
|
| 1022 |
+
status_str = "✅ оценено" if committed else ("🕓 pending" if pending else "🆕 не оценено")
|
| 1023 |
+
|
| 1024 |
+
# badges
|
| 1025 |
+
has_abs = (isinstance(abstract, str) and abstract.strip() and abstract.strip() != "—")
|
| 1026 |
+
has_pdf = (isinstance(pdf_url, str) and isinstance(pdf_url, str) and pdf_url.strip())
|
| 1027 |
+
has_pl = (isinstance(pl_url, str) and pl_url.strip())
|
| 1028 |
+
has_doi = bool(doi_link)
|
| 1029 |
+
badges = [
|
| 1030 |
+
f"Abstract {'✅' if has_abs else '❌'}",
|
| 1031 |
+
f"PDF {'✅' if has_pdf else '❌'}",
|
| 1032 |
+
f"URL {'✅' if has_pl else '❌'}",
|
| 1033 |
+
f"DOI {'✅' if has_doi else '❌'}",
|
| 1034 |
+
]
|
| 1035 |
+
if oa_is is not None:
|
| 1036 |
+
badges.append(f"OA {'✅' if oa_is else '❌'}{(' (' + str(oa_status) + ')') if oa_status else ''}")
|
| 1037 |
+
|
| 1038 |
+
st.markdown(f"## {title}")
|
| 1039 |
+
st.caption(status_str + " • " + " | ".join(badges))
|
| 1040 |
+
|
| 1041 |
+
id_lines = []
|
| 1042 |
+
if source_id:
|
| 1043 |
+
id_lines.append(f"**Source ID:** `{source_id}`")
|
| 1044 |
+
if work_id:
|
| 1045 |
+
id_lines.append(f"**Work ID:** `{work_id}`")
|
| 1046 |
+
if id_lines:
|
| 1047 |
+
st.markdown(" • ".join(id_lines))
|
| 1048 |
+
|
| 1049 |
+
# Метрики
|
| 1050 |
+
y_s = str(y) if y is not None else "—"
|
| 1051 |
+
c_s = str(cites) if cites is not None else "—"
|
| 1052 |
+
ds_s = f"{dir_score:.3f}" if dir_score is not None else "—"
|
| 1053 |
+
dlt_s = f"{delta:.4f}" if delta is not None else "—"
|
| 1054 |
+
qw_s = f"{qw:.3f}" if qw is not None else "—"
|
| 1055 |
+
st.markdown(f"**Год:** {y_s} | **Цитаты:** {c_s} | **dir_score:** {ds_s} | **delta:** {dlt_s} | **quality:** {qw_s}")
|
| 1056 |
+
|
| 1057 |
+
# Авторы
|
| 1058 |
+
if authors and authors != "—":
|
| 1059 |
+
st.markdown("**Авторы:**")
|
| 1060 |
+
st.write(authors)
|
| 1061 |
+
else:
|
| 1062 |
+
st.caption("Авторы: — (в исходных DIR*.jsonl авторы не предоставляются)")
|
| 1063 |
+
|
| 1064 |
+
# Ссылки
|
| 1065 |
+
links = []
|
| 1066 |
+
if work_id:
|
| 1067 |
+
links.append(f"[OpenAlex]({work_id})")
|
| 1068 |
+
if doi_link:
|
| 1069 |
+
links.append(f"[DOI]({doi_link})")
|
| 1070 |
+
if isinstance(pdf_url, str) and pdf_url.strip():
|
| 1071 |
+
links.append(f"[PDF]({pdf_url.strip()})")
|
| 1072 |
+
if isinstance(pl_url, str) and pl_url.strip():
|
| 1073 |
+
links.append(f"[Primary URL]({pl_url.strip()})")
|
| 1074 |
+
if isinstance(oa_url, str) and oa_url.strip():
|
| 1075 |
+
links.append(f"[OA URL]({oa_url.strip()})")
|
| 1076 |
+
st.markdown("**Ссылки:** " + (" | ".join(links) if links else "—"))
|
| 1077 |
+
|
| 1078 |
+
# Аннотация
|
| 1079 |
+
st.markdown("### Аннотация")
|
| 1080 |
+
st.write(abstract if abstract else "—")
|
| 1081 |
+
|
| 1082 |
+
# Доп. детали автооценки (по желанию)
|
| 1083 |
+
with st.expander("Детали автооценки", expanded=True):
|
| 1084 |
+
comp = row.get("dir_native_components") or {}
|
| 1085 |
+
if isinstance(comp, dict) and comp:
|
| 1086 |
+
tr = comp.get("trace"); tp = comp.get("topic"); tx = comp.get("text"); nz = comp.get("noise")
|
| 1087 |
+
sc = comp.get("score"); dl = comp.get("delta")
|
| 1088 |
+
st.markdown(
|
| 1089 |
+
f"**Компоненты (native DIR):** "
|
| 1090 |
+
f"trace={tr if tr is not None else '—'} | "
|
| 1091 |
+
f"topic={tp if tp is not None else '—'} | "
|
| 1092 |
+
f"text={tx if tx is not None else '—'} | "
|
| 1093 |
+
f"noise={nz if nz is not None else '—'} | "
|
| 1094 |
+
f"score={sc if sc is not None else '—'} | "
|
| 1095 |
+
f"delta={dl if dl is not None else '—'}"
|
| 1096 |
+
)
|
| 1097 |
+
expl = row.get("dir_score_explanation")
|
| 1098 |
+
if expl:
|
| 1099 |
+
st.markdown("**dir_score_explanation:**")
|
| 1100 |
+
st.write(expl)
|
| 1101 |
+
|