yogl commited on
Commit
08477b6
·
verified ·
1 Parent(s): a1854a8

Update src/streamlit_app.py

Browse files
Files changed (1) hide show
  1. src/streamlit_app.py +578 -587
src/streamlit_app.py CHANGED
@@ -1,33 +1,10 @@
1
- #!/usr/bin/env python3
2
- # -*- coding: utf-8 -*-
3
- """streamlit_app.py
4
-
5
- Скоринг публикаций по направлениям исследований DIR01..DIR14.
6
- Offline-first: приложение работает ТОЛЬКО по готовым JSONL candidates и НЕ делает
7
- никаких сетевых enrichment (OpenAlex API и т.п.).
8
-
9
- Хранилища на HuggingFace Hub (repo_type="dataset"):
10
- - PUBLICATIONS_REPO: содержит dir_registry.json и candidates JSONL
11
- - REVIEWS_REPO: журнал оценок (JSONL batches)
12
-
13
- Переменные окружения (Space Settings → Variables/Secrets):
14
- - PUBLICATIONS_REPO, REVIEWS_REPO (обязательно)
15
- - HF_TOKEN (нужен для записи; для чтения приватных репо)
16
- - BATCH_SIZE (по умолчанию 1)
17
- - PAGE_WINDOW_SIZE (по умолчанию 50)
18
- - PUB_DIR_REGISTRY_PATH (по умолчанию dir_registry.json)
19
- - PUB_CANDIDATES_PREFIX (по умолчанию candidates)
20
- - REVIEWS_LOG_PREFIX (по умолчанию reviews_log)
21
- - PUB_CANDIDATES_PATTERNS (опционально)
22
-
23
- """
24
-
25
  import os
26
  import json
27
  import uuid
28
  import gzip
29
  import tempfile
30
  import datetime as dt
 
31
  from typing import List, Dict, Any, Optional, Tuple, Set
32
 
33
  import pandas as pd
@@ -35,42 +12,78 @@ import streamlit as st
35
  from huggingface_hub import hf_hub_download, HfApi
36
 
37
  # =========================
38
- # ENV CONFIG
39
  # =========================
40
  PUBLICATIONS_REPO = os.environ.get("PUBLICATIONS_REPO", "").strip()
41
  REVIEWS_REPO = os.environ.get("REVIEWS_REPO", "").strip()
42
  HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HF_API_TOKEN")
 
 
43
 
44
- ALLOW_CREATE_REVIEWS_REPO = (os.environ.get("ALLOW_CREATE_REVIEWS_REPO", "0").strip().lower() in ("1", "true", "yes", "y", "on"))
45
- REVIEWS_PRIVATE = (os.environ.get("REVIEWS_PRIVATE", "1").strip().lower() in ("1", "true", "yes", "y", "on"))
46
-
47
  BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "1"))
48
- PAGE_WINDOW_SIZE = int(os.environ.get("PAGE_WINDOW_SIZE", "50"))
49
 
50
- PUB_DIR_REGISTRY_PATH = os.environ.get("PUB_DIR_REGISTRY_PATH", "dir_registry.json").strip()
51
- PUB_CANDIDATES_PREFIX = os.environ.get("PUB_CANDIDATES_PREFIX", "candidates").strip()
52
- REVIEWS_LOG_PREFIX = os.environ.get("REVIEWS_LOG_PREFIX", "reviews_log").strip()
53
 
54
- # Паттерны поиска candidates файлов (разные схемы именования)
 
 
55
  PUB_CANDIDATES_PATTERNS = os.environ.get(
56
  "PUB_CANDIDATES_PATTERNS",
57
- "{prefix}/{dir_id}_top500.jsonl|{prefix}/{dir_id}_top500.jsonl.gz|{prefix}/{dir_id}.jsonl|{prefix}/{dir_id}.jsonl.gz|manual_candidates/{dir_id}.jsonl|manual_candidates/{dir_id}.jsonl.gz",
58
  ).strip()
59
 
 
60
  api = HfApi(token=HF_TOKEN)
61
 
62
- st.set_page_config(page_title="Скоринг публикаций", layout="wide")
 
63
 
64
  # =========================
65
  # Helpers
66
  # =========================
67
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
  def require_env(name: str, value: str) -> None:
69
  if not value:
70
  st.error(f"Не задано **{name}**. Укажи в Space Settings → Variables.")
71
  st.stop()
72
 
73
-
74
  def safe_int(x) -> Optional[int]:
75
  try:
76
  if x is None or pd.isna(x):
@@ -79,7 +92,6 @@ def safe_int(x) -> Optional[int]:
79
  except Exception:
80
  return None
81
 
82
-
83
  def safe_float(x) -> Optional[float]:
84
  try:
85
  if x is None or pd.isna(x):
@@ -88,20 +100,20 @@ def safe_float(x) -> Optional[float]:
88
  except Exception:
89
  return None
90
 
91
-
92
  def doi_url(doi: Optional[str]) -> Optional[str]:
93
  if not doi:
94
  return None
95
  doi = str(doi).strip()
96
- if not doi:
97
- return None
98
  if doi.startswith("http://") or doi.startswith("https://"):
99
  return doi
100
  return f"https://doi.org/{doi}"
101
 
102
-
103
  def normalize_dir_id(dir_id: str, pad2: bool) -> str:
104
- """DIR1..DIR14 / DIR01..DIR14"""
 
 
 
 
105
  if not isinstance(dir_id, str):
106
  return str(dir_id)
107
  s = dir_id.strip().upper()
@@ -111,35 +123,34 @@ def normalize_dir_id(dir_id: str, pad2: bool) -> str:
111
  try:
112
  n = int(tail)
113
  except Exception:
 
114
  return dir_id.strip()
115
  return f"DIR{n:02d}" if pad2 else f"DIR{n}"
116
 
117
-
118
  def dir_variants(dir_id: str) -> List[str]:
 
 
 
119
  if not isinstance(dir_id, str):
120
  return [str(dir_id)]
121
  a = dir_id.strip()
122
  b = normalize_dir_id(a, pad2=True)
123
  c = normalize_dir_id(a, pad2=False)
124
- out: List[str] = []
125
  for x in [a, b, c]:
126
  if x and x not in out:
127
  out.append(x)
128
  return out
129
 
130
-
131
- def dir_tag(dir_id: str) -> str:
132
- """DIR-01"""
133
- d2 = normalize_dir_id(dir_id, pad2=True) # DIR01
134
- if d2.upper().startswith("DIR") and len(d2) >= 5:
135
- return f"DIR-{d2[3:]}"
136
- return d2
137
-
138
 
139
  def join_terms(term_list: Any) -> str:
140
  if not term_list:
141
  return "—"
142
- out: List[str] = []
143
  for x in term_list:
144
  if isinstance(x, dict):
145
  t = x.get("t")
@@ -149,72 +160,74 @@ def join_terms(term_list: Any) -> str:
149
  out.append(str(x))
150
  return "; ".join(out) if out else "—"
151
 
152
-
153
  def topics_line(topics: Any) -> str:
154
  if not topics:
155
  return "—"
156
- out: List[str] = []
157
- if isinstance(topics, list):
158
- for t in topics[:3]:
159
- if isinstance(t, dict):
160
- tid = t.get("topic_id") or t.get("id")
161
- name = t.get("topic_name") or t.get("display_name")
162
- if tid and name:
163
- out.append(f"{tid} — {name}")
164
- elif name:
165
- out.append(str(name))
166
- elif tid:
167
- out.append(str(tid))
168
- else:
169
- out.append(str(t))
170
  return "; ".join(out) if out else "—"
171
 
 
 
 
 
172
 
173
  def decode_abstract(abstract_inverted_index: Optional[dict]) -> str:
174
  if not abstract_inverted_index or not isinstance(abstract_inverted_index, dict):
175
  return ""
176
- pos_to_word: Dict[int, str] = {}
177
  for w, positions in abstract_inverted_index.items():
178
  if not isinstance(positions, list):
179
  continue
180
  for p in positions:
181
- try:
182
- pos_to_word[int(p)] = str(w)
183
- except Exception:
184
- continue
185
  return " ".join(pos_to_word[p] for p in sorted(pos_to_word.keys())) if pos_to_word else ""
186
 
187
-
188
  def get_authors_and_abstract(row: Dict[str, Any]) -> Tuple[str, str]:
189
- """Offline-only: показываем только то, что есть в candidates JSONL."""
190
- authors = row.get("authors") or row.get("author_names") or row.get("authors_str")
 
 
 
 
191
  if not authors:
192
- auths = row.get("authorships")
 
193
  if isinstance(auths, list):
194
- names: List[str] = []
195
  for a in auths:
196
  if not isinstance(a, dict):
197
  continue
198
- an = ((a.get("author") or {}).get("display_name") or "").strip()
199
  if an:
200
  names.append(an)
201
  if names:
202
- authors = ", ".join(names[:12]) + (", и др." if len(names) > 12 else "")
203
-
204
  if isinstance(authors, list):
205
- authors = ", ".join([str(a) for a in authors if a])
206
- authors_str = authors.strip() if isinstance(authors, str) and authors.strip() else ""
207
 
208
- abstract = row.get("abstract") or row.get("abstract_text")
209
- if not abstract and isinstance(row.get("abstract_inverted_index"), dict):
210
- abstract = decode_abstract(row.get("abstract_inverted_index"))
211
- abstract_str = abstract.strip() if isinstance(abstract, str) and abstract.strip() else ""
212
-
213
- return authors_str or "—", abstract_str or "—"
214
 
 
215
 
216
  def open_jsonl_any(path: str) -> List[Dict[str, Any]]:
217
- rows: List[Dict[str, Any]] = []
218
  if path.endswith(".gz"):
219
  with gzip.open(path, "rt", encoding="utf-8") as f:
220
  for line in f:
@@ -229,32 +242,37 @@ def open_jsonl_any(path: str) -> List[Dict[str, Any]]:
229
  rows.append(json.loads(line))
230
  return rows
231
 
232
-
233
  def normalize_candidate_row(obj: Dict[str, Any], selected_dir: str) -> Dict[str, Any]:
234
- """Нормализация схем candidates (под разные генераторы)."""
 
 
 
 
 
235
  out = dict(obj)
236
 
237
- # IDs
238
  source_id = out.get("source_id") or out.get("Source ID")
239
  if isinstance(source_id, str) and source_id.strip():
240
  out["source_id"] = source_id.strip()
241
 
242
  work_id = out.get("work_id")
243
  if not isinstance(work_id, str) or not work_id.strip():
 
244
  wid = out.get("openalex_work_id") or out.get("OpenAlex work ID") or out.get("dup_work_id")
245
  if isinstance(wid, str) and wid.strip():
246
  work_id = wid.strip()
247
- out["work_id"] = work_id if isinstance(work_id, str) and work_id.strip() else None
248
 
249
- # title
250
  title = out.get("title") or out.get("display_name") or out.get("Title")
251
  out["title"] = str(title).strip() if title else "—"
252
 
253
- # doi
254
  doi = out.get("doi") or out.get("DOI")
255
  out["doi"] = str(doi).strip() if doi else None
256
 
257
- # abstract (offline)
258
  if out.get("abstract") is None:
259
  ab = out.get("abstract_text")
260
  if isinstance(ab, str) and ab.strip():
@@ -262,7 +280,8 @@ def normalize_candidate_row(obj: Dict[str, Any], selected_dir: str) -> Dict[str,
262
  elif isinstance(out.get("abstract_inverted_index"), dict):
263
  out["abstract"] = decode_abstract(out.get("abstract_inverted_index")) or None
264
 
265
- # year / cited
 
266
  year = out.get("year")
267
  if year is None:
268
  year = out.get("publication_year") or out.get("Publication year")
@@ -273,21 +292,27 @@ def normalize_candidate_row(obj: Dict[str, Any], selected_dir: str) -> Dict[str,
273
  cited_by = out.get("cited_by_count") or out.get("Cited by count")
274
  out["cited_by"] = safe_int(cited_by)
275
 
276
- # scores
 
 
277
  ds = out.get("dir_score")
 
278
  if ds is None:
279
  comp = out.get("dir_native_components") or {}
280
  if isinstance(comp, dict) and comp.get("score") is not None:
281
  ds = comp.get("score")
 
282
  if ds is None:
283
  dsd = out.get("dir_scores") or {}
284
  if isinstance(dsd, dict):
 
285
  for dv in dir_variants(selected_dir):
286
  if dv in dsd:
287
  ds = dsd.get(dv)
288
  break
289
  out["dir_score"] = safe_float(ds)
290
 
 
291
  ms = out.get("match_score")
292
  if ms is None:
293
  comp = out.get("dir_native_components") or {}
@@ -300,64 +325,50 @@ def normalize_candidate_row(obj: Dict[str, Any], selected_dir: str) -> Dict[str,
300
  ms = None
301
  out["match_score"] = safe_float(ms)
302
 
303
- out["quality_weight"] = safe_float(out.get("quality_weight"))
304
- out["dir_native_components"] = out.get("dir_native_components") if isinstance(out.get("dir_native_components"), dict) else {}
 
 
 
305
 
306
- # links
307
  out["pdf_url"] = out.get("pdf_url") or out.get("PDF URL")
308
  out["primary_location_url"] = out.get("primary_location_url") or out.get("Primary location URL")
309
 
310
  return out
311
 
312
-
313
- def row_keys_all(r: Dict[str, Any]) -> Set[str]:
314
- keys: Set[str] = set()
315
- sid = r.get("source_id")
 
 
 
316
  if isinstance(sid, str) and sid.strip():
317
- keys.add(f"SRC::{sid.strip()}")
318
- wid = r.get("work_id")
319
  if isinstance(wid, str) and wid.strip():
320
- keys.add(f"W::{wid.strip()}")
321
- return keys
322
-
323
-
324
- def check_dataset_repo(repo_id: str, token: Optional[str]) -> Tuple[bool, str]:
325
- try:
326
- api_local = HfApi(token=token) if token else HfApi()
327
- _ = api_local.repo_info(repo_id=repo_id, repo_type="dataset")
328
- return True, "OK"
329
- except Exception as e:
330
- return False, str(e)
331
-
332
-
333
- def maybe_create_reviews_repo(repo_id: str) -> Tuple[bool, str]:
334
- if not ALLOW_CREATE_REVIEWS_REPO:
335
- return False, "ALLOW_CREATE_REVIEWS_REPO=0"
336
- if not HF_TOKEN:
337
- return False, "HF_TOKEN отсутствует (нужен для создания репозитория)"
338
- try:
339
- api.create_repo(repo_id=repo_id, repo_type="dataset", private=bool(REVIEWS_PRIVATE), exist_ok=True)
340
- return True, "created_or_exists"
341
- except Exception as e:
342
- return False, str(e)
343
 
344
 
345
  # =========================
346
- # Hub reads
347
  # =========================
348
-
349
  @st.cache_data(show_spinner=False)
350
  def load_dir_registry(repo_id: str, filename: str) -> List[Dict[str, Any]]:
351
  path = hf_hub_download(repo_id=repo_id, filename=filename, repo_type="dataset", token=HF_TOKEN)
352
  with open(path, "r", encoding="utf-8") as f:
353
  return json.load(f)
354
 
355
-
356
  @st.cache_data(show_spinner=False)
357
  def load_candidates(repo_id: str, dir_id: str, prefix: str) -> pd.DataFrame:
 
 
 
358
  patterns = [p.strip() for p in PUB_CANDIDATES_PATTERNS.split("|") if p.strip()]
359
- tried: List[str] = []
360
- rows: Optional[List[Dict[str, Any]]] = None
361
 
362
  for dv in dir_variants(dir_id):
363
  for pat in patterns:
@@ -366,7 +377,11 @@ def load_candidates(repo_id: str, dir_id: str, prefix: str) -> pd.DataFrame:
366
  try:
367
  path = hf_hub_download(repo_id=repo_id, filename=fname, repo_type="dataset", token=HF_TOKEN)
368
  rows = open_jsonl_any(path)
369
- break
 
 
 
 
370
  except Exception:
371
  continue
372
  if rows is not None:
@@ -375,9 +390,11 @@ def load_candidates(repo_id: str, dir_id: str, prefix: str) -> pd.DataFrame:
375
  if rows is None:
376
  raise FileNotFoundError("Не найден candidates файл. Пробовали:\n" + "\n".join(tried[:30]) + ("\n..." if len(tried) > 30 else ""))
377
 
 
378
  normed = [normalize_candidate_row(r, dir_id) for r in rows if isinstance(r, dict)]
379
  df = pd.DataFrame(normed)
380
 
 
381
  for col in ["dir_score", "match_score", "cited_by", "year", "quality_weight"]:
382
  if col in df.columns:
383
  df[col] = pd.to_numeric(df[col], errors="coerce")
@@ -386,14 +403,14 @@ def load_candidates(repo_id: str, dir_id: str, prefix: str) -> pd.DataFrame:
386
 
387
 
388
  # =========================
389
- # Hub reviews index (read)
390
  # =========================
391
-
392
  @st.cache_data(show_spinner=False)
393
  def list_review_files(repo_id: str, dir_id: str, prefix: str) -> List[str]:
394
  files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
 
395
  needles = [f"/{dv}/" for dv in dir_variants(dir_id)]
396
- out: List[str] = []
397
  for p in files:
398
  if not (p.startswith(prefix + "/") and p.endswith(".jsonl")):
399
  continue
@@ -401,25 +418,19 @@ def list_review_files(repo_id: str, dir_id: str, prefix: str) -> List[str]:
401
  out.append(p)
402
  return out
403
 
404
-
405
  @st.cache_data(show_spinner=False)
406
- def load_reviews_index(repo_id: str, dir_id: str, prefix: str) -> Tuple[Set[str], Dict[str, Dict[str, Any]], Dict[str, Dict[str, Any]]]:
407
- """Возвращает:
408
- - множество ключей (SRC:: / W::) уже оценённых публикаций
409
- - словарь последних review по source_id
410
- - словарь последних review по work_id
411
-
412
- Важно: это НЕ enrichment; это чтение ваших собственных сохранённых оценок.
413
  """
414
  try:
415
  files = list_review_files(repo_id, dir_id, prefix)
416
  except Exception:
417
- return set(), {}, {}
418
-
419
- keys: Set[str] = set()
420
- by_src: Dict[str, Dict[str, Any]] = {}
421
- by_w: Dict[str, Dict[str, Any]] = {}
422
 
 
423
  for relpath in files:
424
  try:
425
  path = hf_hub_download(repo_id=repo_id, filename=relpath, repo_type="dataset", token=HF_TOKEN)
@@ -431,34 +442,29 @@ def load_reviews_index(repo_id: str, dir_id: str, prefix: str) -> Tuple[Set[str]
431
  obj = json.loads(line)
432
 
433
  sid = obj.get("source_id")
434
- wid = obj.get("work_id")
435
-
436
  if isinstance(sid, str) and sid.strip():
437
- sid = sid.strip()
438
- keys.add(f"SRC::{sid}")
439
- by_src[sid] = obj
440
  if isinstance(wid, str) and wid.strip():
441
- wid = wid.strip()
442
- keys.add(f"W::{wid}")
443
- by_w[wid] = obj
444
  except Exception:
445
  continue
446
-
447
- return keys, by_src, by_w
448
 
449
 
450
  # =========================
451
- # Hub write
452
  # =========================
453
-
454
  def push_batch_to_reviews_repo(repo_id: str, dir_id: str, prefix: str, batch: List[Dict[str, Any]]) -> None:
455
  if not batch:
456
  return
457
  if not HF_TOKEN:
458
  raise RuntimeError("Нет HF_TOKEN (Secret) — нельзя записывать в reviews dataset.")
459
-
460
  today = dt.date.today().isoformat()
461
  batch_id = str(uuid.uuid4())
 
 
462
  canonical_dir = normalize_dir_id(dir_id, pad2=False)
463
  path_in_repo = f"{prefix}/{today}/{canonical_dir}/{batch_id}.jsonl"
464
 
@@ -477,64 +483,20 @@ def push_batch_to_reviews_repo(repo_id: str, dir_id: str, prefix: str, batch: Li
477
 
478
 
479
  # =========================
480
- # Session state
481
  # =========================
482
 
483
- def ensure_state() -> None:
484
- if "batch" not in st.session_state:
485
- st.session_state["batch"] = []
486
-
487
- if "reviewed_local_committed" not in st.session_state:
488
- st.session_state["reviewed_local_committed"] = set()
489
- if "reviewed_local_pending" not in st.session_state:
490
- st.session_state["reviewed_local_pending"] = set()
491
-
492
- # локальные значения (чтобы предзаполнять при навигации)
493
- if "local_by_src" not in st.session_state:
494
- st.session_state["local_by_src"] = {}
495
- if "local_by_w" not in st.session_state:
496
- st.session_state["local_by_w"] = {}
497
-
498
- # remote кэш (НЕ показываем в UI)
499
- if "remote_dir" not in st.session_state:
500
- st.session_state["remote_dir"] = None
501
- if "remote_keys" not in st.session_state:
502
- st.session_state["remote_keys"] = set()
503
- if "remote_by_src" not in st.session_state:
504
- st.session_state["remote_by_src"] = {}
505
- if "remote_by_w" not in st.session_state:
506
- st.session_state["remote_by_w"] = {}
507
-
508
-
509
- def get_saved_review_for_row(row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
510
- sid = row.get("source_id") if isinstance(row.get("source_id"), str) else None
511
- wid = row.get("work_id") if isinstance(row.get("work_id"), str) else None
512
-
513
- # local first (pending/committed current session)
514
- if sid and sid in st.session_state.get("local_by_src", {}):
515
- return st.session_state["local_by_src"].get(sid)
516
- if wid and wid in st.session_state.get("local_by_w", {}):
517
- return st.session_state["local_by_w"].get(wid)
518
-
519
- # remote next
520
- if sid and sid in st.session_state.get("remote_by_src", {}):
521
- return st.session_state["remote_by_src"].get(sid)
522
- if wid and wid in st.session_state.get("remote_by_w", {}):
523
- return st.session_state["remote_by_w"].get(wid)
524
-
525
- return None
526
-
527
-
528
  # =========================
529
- # Sidebar: диагностика + управление
530
  # =========================
531
 
532
  with st.sidebar:
533
- st.title("Скоринг публикаций")
534
 
535
  require_env("PUBLICATIONS_REPO", PUBLICATIONS_REPO)
536
  require_env("REVIEWS_REPO", REVIEWS_REPO)
537
 
 
538
  pub_ok, pub_msg = check_dataset_repo(PUBLICATIONS_REPO, HF_TOKEN)
539
  if not pub_ok:
540
  st.error("PUBLICATIONS_REPO недоступен как dataset repo. Проверь repo_id и доступ.\n\n" + pub_msg)
@@ -542,145 +504,122 @@ with st.sidebar:
542
 
543
  rev_ok, rev_msg = check_dataset_repo(REVIEWS_REPO, HF_TOKEN)
544
  if not rev_ok:
545
- created, _ = maybe_create_reviews_repo(REVIEWS_REPO)
546
  if created:
547
  rev_ok2, rev_msg2 = check_dataset_repo(REVIEWS_REPO, HF_TOKEN)
548
  if not rev_ok2:
549
  st.error("REVIEWS_REPO недоступен после create_repo.\n\n" + rev_msg2)
550
  st.stop()
 
 
551
  else:
552
  st.error(
553
  "REVIEWS_REPO недоступен (404/нет доступа).\n\n"
554
- "Проверь:\n"
555
- "1) REVIEWS_REPO = <owner>/<dataset_name> (именно dataset).\n"
556
- "2) Репозиторий существует на Hub.\n"
557
- "3) Для приватного repo нужен HF_TOKEN с доступом.\n\n"
 
558
  + rev_msg
559
  )
560
  st.stop()
561
 
562
- reviewer_default = os.environ.get("REVIEWER", "")
563
- reviewer = st.text_input("Рецензент (имя/ник)", value=reviewer_default).strip() or "anonymous"
564
 
565
- sort_mode = st.selectbox(
566
- "Сортировка",
567
- ["dir_score ↓", "delta ↓", "quality ↓", "год ↓", "cited_by ↓"],
568
- index=0,
569
- )
 
 
 
570
 
571
- skip_reviewed = st.checkbox("Пропускать уже оценённые", value=True)
572
 
573
- st.divider()
574
- # сервисные кнопки
575
- with st.expander("Сервис", expanded=False):
576
- flush_now = st.button("⬆️ Синхронизировать pending сейчас", use_container_width=True)
577
- clear_cache = st.button("🧹 Сбросить кэш (публикации/оценки)", use_container_width=True)
578
- st.caption(f"BATCH_SIZE={BATCH_SIZE}")
579
 
 
 
 
 
 
580
 
581
  # =========================
582
- # Cache reset
583
  # =========================
 
 
 
 
 
 
584
 
585
- if "clear_cache" in locals() and clear_cache:
586
  load_dir_registry.clear()
587
  load_candidates.clear()
588
- list_review_files.clear()
589
- load_reviews_index.clear()
590
-
 
 
 
591
  for k in [
592
- "remote_dir",
593
- "remote_keys",
594
- "remote_by_src",
595
- "remote_by_w",
596
  "reviewed_local_committed",
597
  "reviewed_local_pending",
598
- "local_by_src",
599
- "local_by_w",
600
  "batch",
601
  ]:
602
  if k in st.session_state:
603
  del st.session_state[k]
604
-
605
- st.toast("Кэш очищен")
606
  st.rerun()
607
 
608
-
609
  # =========================
610
- # Load DIR registry + main DIR select
611
  # =========================
612
-
613
- dirs = load_dir_registry(PUBLICATIONS_REPO, PUB_DIR_REGISTRY_PATH)
614
- _dir_map = {d.get("dir_id"): d for d in dirs if d.get("dir_id")}
615
- _dir_ids = [d for d in _dir_map.keys()]
616
- if not _dir_ids:
617
- st.error("В dir_registry.json не найдено ни одного dir_id")
618
- st.stop()
619
-
620
- # top selector
621
- selected_dir = st.selectbox(
622
- "DIR",
623
- options=_dir_ids,
624
- index=0,
625
- format_func=lambda x: f"{dir_tag(x)} — {_dir_map.get(x, {}).get('dir_name', '—')}",
 
 
 
 
 
 
 
 
626
  )
627
 
628
- dir_meta = _dir_map[selected_dir]
629
- canonical_dir = normalize_dir_id(selected_dir, pad2=False)
630
-
631
-
632
- # =========================
633
- # DIR header (card)
634
- # =========================
635
-
636
- def _get(d: Dict[str, Any], key: str, default: str = "—") -> str:
637
- v = d.get(key)
638
- return v if isinstance(v, str) and v.strip() else default
639
-
640
- st.markdown(f"### {dir_meta.get('dir_name', '—')}")
641
-
642
- # выделенный блок, чтобы отличался от основного интерфейса
643
- with st.container():
644
- st.markdown(
645
- """
646
- <style>
647
- .dir-card{border:1px solid rgba(255,255,255,.12); border-radius:14px; padding:14px 16px; background: rgba(255,255,255,.03);}
648
- .dir-tag{font-size:12px; letter-spacing:.08em; opacity:.75; font-weight:700;}
649
- .dir-desc-title{font-size:11px; letter-spacing:.08em; opacity:.75; font-weight:800; margin-top:10px;}
650
- </style>
651
- """,
652
- unsafe_allow_html=True,
653
- )
654
- st.markdown(f"<div class='dir-card'><div class='dir-tag'>{dir_tag(selected_dir)}</div>", unsafe_allow_html=True)
655
- st.markdown("<div class='dir-desc-title'>КРАТКОЕ ОПИСАНИЕ</div>", unsafe_allow_html=True)
656
- st.write(_get(dir_meta, "dir_description"))
657
-
658
- defaults = dir_meta.get("defaults") or {}
659
- year_from = defaults.get("year_from")
660
- year_to = defaults.get("year_to")
661
-
662
- terms = dir_meta.get("terms") or {}
663
- anchor_str = join_terms(terms.get("anchor"))
664
- support_str = join_terms(terms.get("support"))
665
- noise_str = join_terms(terms.get("noise"))
666
- topics_str = topics_line(dir_meta.get("topics") or [])
667
-
668
- with st.expander("Детали DIR", expanded=True):
669
- st.markdown(f"**Временной интервал:** {year_from} – {year_to}")
670
- st.markdown(f"**Якоря:** {anchor_str}")
671
- st.markdown(f"**Поддержка:** {support_str}")
672
- st.markdown(f"**Шум:** {noise_str}")
673
- st.markdown(f"**Topics поиска:** {topics_str}")
674
-
675
- st.markdown("</div>", unsafe_allow_html=True)
676
 
677
  st.divider()
678
 
679
-
680
  # =========================
681
- # Load candidates
682
  # =========================
683
-
684
  try:
685
  df = load_candidates(PUBLICATIONS_REPO, selected_dir, PUB_CANDIDATES_PREFIX)
686
  except Exception as e:
@@ -688,53 +627,75 @@ except Exception as e:
688
  st.stop()
689
 
690
  if df.empty:
691
- st.warning(f"Пустой список кандидатов для {selected_dir}")
692
  st.stop()
693
 
694
- # sort
695
- if sort_mode == "dir_score ↓" and "dir_score" in df.columns:
696
- df = df.sort_values(["dir_score", "quality_weight", "match_score"], ascending=[False, False, False])
697
- elif sort_mode == "delta ↓" and "match_score" in df.columns:
698
- df = df.sort_values(["match_score", "dir_score", "quality_weight"], ascending=[False, False, False])
699
- elif sort_mode == "quality ↓" and "quality_weight" in df.columns:
700
- df = df.sort_values(["quality_weight", "dir_score"], ascending=[False, False])
701
- elif sort_mode == "год ↓" and "year" in df.columns:
702
- df = df.sort_values(["year", "dir_score"], ascending=[False, False])
703
- elif sort_mode == "cited_by ↓" and "cited_by" in df.columns:
704
- df = df.sort_values(["cited_by", "dir_score"], ascending=[False, False])
 
 
705
 
706
  df = df.reset_index(drop=True)
707
-
708
- total = int(len(df))
709
-
 
710
 
711
  # =========================
712
- # State + remote reviews index (silent)
713
  # =========================
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
714
 
715
  ensure_state()
716
 
717
- if st.session_state.get("remote_dir") != canonical_dir:
718
- # Тихо подгружаем индекс оценок, чтобы:
719
- # - корректно пропускать уже оценённые
720
- # - предзаполнять score/comment при навигации
721
- r_keys, r_by_src, r_by_w = load_reviews_index(REVIEWS_REPO, selected_dir, REVIEWS_LOG_PREFIX)
722
- st.session_state["remote_dir"] = canonical_dir
723
- st.session_state["remote_keys"] = set(r_keys)
724
- st.session_state["remote_by_src"] = dict(r_by_src)
725
- st.session_state["remote_by_w"] = dict(r_by_w)
726
-
727
- reviewed_committed = set(st.session_state.get("remote_keys", set())) | set(st.session_state.get("reviewed_local_committed", set()))
728
- reviewed_pending = set(st.session_state.get("reviewed_local_pending", set()))
729
- reviewed_effective = reviewed_committed | reviewed_pending
730
 
 
731
 
732
- def is_reviewed_effective(r: Dict[str, Any]) -> bool:
733
- keys = row_keys_all(r)
734
- return bool(keys and any(k in reviewed_effective for k in keys))
 
 
 
 
735
 
 
 
 
736
 
737
- def status_icon(r: Dict[str, Any]) -> str:
738
  keys = row_keys_all(r)
739
  if keys and any(k in reviewed_committed for k in keys):
740
  return "✅"
@@ -742,18 +703,57 @@ def status_icon(r: Dict[str, Any]) -> str:
742
  return "🕓"
743
  return "🆕"
744
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
745
 
746
- # counts for sidebar (без слова remote)
747
  src_series = df["source_id"].fillna("") if "source_id" in df.columns else pd.Series([""] * len(df))
748
  wid_series = df["work_id"].fillna("") if "work_id" in df.columns else pd.Series([""] * len(df))
749
 
750
- comm_src = {k[5:] for k in reviewed_committed if isinstance(k, str) and k.startswith("SRC::")}
751
- comm_w = {k[3:] for k in reviewed_committed if isinstance(k, str) and k.startswith("W::")}
752
- pend_src = {k[5:] for k in reviewed_pending if isinstance(k, str) and k.startswith("SRC::")}
753
- pend_w = {k[3:] for k in reviewed_pending if isinstance(k, str) and k.startswith("W::")}
754
-
755
- mask_committed = src_series.astype(str).isin(comm_src) | wid_series.astype(str).isin(comm_w)
756
- mask_pending = src_series.astype(str).isin(pend_src) | wid_series.astype(str).isin(pend_w)
757
  mask_reviewed = mask_committed | mask_pending
758
 
759
  committed_pub_count = int(mask_committed.sum())
@@ -761,9 +761,9 @@ pending_pub_count = int(mask_pending.sum())
761
  unreviewed_pub_count = int((~mask_reviewed).sum())
762
 
763
  with st.sidebar:
764
- st.markdown("### Статусы")
765
- st.write(f"✅ сохранено: {committed_pub_count}")
766
- st.write(f"🕓 pending: {pending_pub_count}")
767
 
768
  if st.session_state.get("batch"):
769
  pending_jsonl = "".join(json.dumps(x, ensure_ascii=False) + "\n" for x in st.session_state["batch"])
@@ -775,22 +775,19 @@ with st.sidebar:
775
  use_container_width=True,
776
  )
777
 
778
-
779
  # =========================
780
- # Commit helpers
781
  # =========================
782
-
783
  def commit_batch() -> Tuple[bool, str]:
784
  batch = st.session_state.get("batch") or []
785
  if not batch:
786
  return False, "batch_empty"
787
-
788
  try:
789
  push_batch_to_reviews_repo(REVIEWS_REPO, selected_dir, REVIEWS_LOG_PREFIX, batch)
790
  except Exception as e:
791
  return False, str(e)
792
 
793
- # pending -> committed
794
  for obj in batch:
795
  sid = obj.get("source_id")
796
  wid = obj.get("work_id")
@@ -806,92 +803,67 @@ def commit_batch() -> Tuple[bool, str]:
806
  st.session_state["batch"] = []
807
  return True, "ok"
808
 
809
-
810
- if "flush_now" in locals() and flush_now and st.session_state.get("batch"):
811
  with st.spinner("Синхронизирую pending батч…"):
812
  ok, msg = commit_batch()
813
  if ok:
814
- st.success("Pending батч синхронизирован")
815
  st.rerun()
816
  else:
817
  st.error(f"Не удалось синхронизировать: {msg}")
818
 
 
 
 
 
819
 
820
  # =========================
821
- # Navigation state
822
  # =========================
823
-
824
  idx_key = f"idx_{canonical_dir}"
825
  if idx_key not in st.session_state:
826
  st.session_state[idx_key] = 0
827
 
 
 
 
 
 
 
 
 
828
 
829
- def first_unreviewed_from(i0: int) -> int:
830
- i = max(0, int(i0))
 
 
 
 
 
 
 
 
 
831
  while i < total:
832
- if not skip_reviewed:
833
- return i
834
  r = df.iloc[i].to_dict()
835
  if is_reviewed_effective(r):
836
  i += 1
837
  continue
838
  return i
839
- return max(0, total - 1)
840
-
841
-
842
- # при первом входе в DIR — позиционируемся на первом неоценённом
843
- if st.session_state[idx_key] == 0 and skip_reviewed:
844
- st.session_state[idx_key] = first_unreviewed_from(0)
845
-
846
- current_idx = int(st.session_state[idx_key])
847
- current_idx = max(0, min(current_idx, total - 1))
848
-
849
-
850
- def next_idx(i: int) -> int:
851
- j = min(total - 1, int(i) + 1)
852
- return first_unreviewed_from(j) if skip_reviewed else j
853
-
854
 
855
- def prev_idx(i: int) -> int:
856
- # назад НЕ пропускаем оценённые: эксперт часто хочет вернуться
857
- return max(0, int(i) - 1)
858
-
859
-
860
- # =========================
861
- # Layout: list always visible + card
862
- # =========================
863
 
864
- col_list, col_card = st.columns([1.35, 2.65], gap="large")
865
-
866
- # current row
867
- row = df.iloc[current_idx].to_dict()
868
-
869
- # window indices around current
870
- half = max(1, PAGE_WINDOW_SIZE // 2)
871
- window_start = max(0, current_idx - half)
872
- window_end = min(total, window_start + PAGE_WINDOW_SIZE)
873
- window_start = max(0, window_end - PAGE_WINDOW_SIZE)
874
- window_indices = list(range(window_start, window_end))
875
-
876
-
877
- def fmt_idx(i: int) -> str:
878
- r = df.iloc[i].to_dict()
879
- ico = status_icon(r)
880
- year = safe_int(r.get("year"))
881
- year_s = str(year) if year is not None else "—"
882
- title = str(r.get("title") or "—")
883
- title_short = title if len(title) <= 70 else title[:67] + "…"
884
- return f"{ico} {i+1:03d} | {year_s} | {title_short}"
885
-
886
-
887
- with col_list:
888
  st.markdown("### Список публикаций")
889
- st.caption("Публикации в текущем окне")
890
 
891
  sel = st.selectbox(
892
- "",
893
- options=window_indices,
894
- index=window_indices.index(current_idx) if current_idx in window_indices else 0,
895
  format_func=fmt_idx,
896
  label_visibility="collapsed",
897
  )
@@ -900,156 +872,70 @@ with col_list:
900
  st.session_state[idx_key] = int(sel)
901
  st.rerun()
902
 
903
- nav1, nav2, nav3 = st.columns(3)
904
- with nav1:
905
- if st.button("⬅️ Назад", use_container_width=True):
906
- st.session_state[idx_key] = prev_idx(current_idx)
907
- st.rerun()
908
- with nav2:
909
- if st.button("➡️ Далее", use_container_width=True):
910
- st.session_state[idx_key] = next_idx(current_idx)
911
- st.rerun()
912
- with nav3:
913
- st.write(f"**Осталось:** {unreviewed_pub_count}")
914
-
915
- st.caption(f"Всего: {total} · Показаны: {window_start+1}–{window_end}")
916
-
917
-
918
- # =========================
919
- # Publication card + scoring
920
- # =========================
921
-
922
- row = df.iloc[int(st.session_state[idx_key])].to_dict()
923
-
924
- saved = get_saved_review_for_row(row) or {}
925
- saved_score = safe_int(saved.get("score")) if isinstance(saved, dict) else None
926
- saved_comment = saved.get("comment") if isinstance(saved, dict) else None
927
-
928
- default_score = int(saved_score) if saved_score is not None else 0
929
- default_comment = str(saved_comment) if isinstance(saved_comment, str) else ""
930
-
931
- score_key = f"score_{canonical_dir}_{int(st.session_state[idx_key])}"
932
- comment_key = f"com_{canonical_dir}_{int(st.session_state[idx_key])}"
933
-
934
- # предзаполняем при первом появлении виджета
935
- if score_key not in st.session_state:
936
- st.session_state[score_key] = default_score
937
- if comment_key not in st.session_state:
938
- st.session_state[comment_key] = default_comment
939
-
940
-
941
- authors, abstract = get_authors_and_abstract(row)
942
-
943
- y = safe_int(row.get("year"))
944
- cites = safe_int(row.get("cited_by"))
945
- dir_score = safe_float(row.get("dir_score"))
946
- delta = safe_float(row.get("match_score"))
947
- qw = safe_float(row.get("quality_weight"))
948
-
949
- work_id = row.get("work_id") if isinstance(row.get("work_id"), str) else None
950
- source_id = row.get("source_id") if isinstance(row.get("source_id"), str) else None
951
-
952
- pdf_url = row.get("pdf_url")
953
- pl_url = row.get("primary_location_url")
954
- doi_link = doi_url(row.get("doi"))
955
-
956
- # open_access (если есть)
957
- oa = row.get("open_access") if isinstance(row.get("open_access"), dict) else {}
958
- oa_is = oa.get("is_oa") if isinstance(oa, dict) else None
959
- oa_status = oa.get("oa_status") if isinstance(oa, dict) else None
960
- oa_url = oa.get("oa_url") if isinstance(oa, dict) else None
961
-
962
- keys = row_keys_all(row)
963
- committed = bool(keys and any(k in reviewed_committed for k in keys))
964
- pending = bool(keys and any(k in reviewed_pending for k in keys))
965
- status_str = "✅ оценено" if committed else ("🕓 pending" if pending else "🆕 не оценено")
966
-
967
- has_abs = (isinstance(abstract, str) and abstract.strip() and abstract.strip() != "—")
968
- has_pdf = (isinstance(pdf_url, str) and pdf_url.strip())
969
- has_pl = (isinstance(pl_url, str) and pl_url.strip())
970
- has_doi = bool(doi_link)
971
-
972
- badges = [
973
- f"Abstract {'✅' if has_abs else '❌'}",
974
- f"PDF {'✅' if has_pdf else '❌'}",
975
- f"URL {'✅' if has_pl else '❌'}",
976
- f"DOI {'✅' if has_doi else '❌'}",
977
- ]
978
- if oa_is is not None:
979
- badges.append(f"OA {'✅' if oa_is else '❌'}{(' (' + str(oa_status) + ')') if oa_status else ''}")
980
 
 
981
 
982
- with col_card:
983
- info_col, score_col = st.columns([2.2, 1.1], gap="large")
984
 
985
- with info_col:
986
- st.markdown(f"### {row.get('title') or '—'}")
 
 
987
 
988
- if source_id:
989
- st.markdown(f"**Source ID:** `{source_id}`")
990
- if work_id:
991
- st.markdown(f"**Work ID:** `{work_id}`")
992
 
993
- st.markdown("**Авторы:**")
994
- st.write(authors)
 
 
 
995
 
996
- y_s = str(y) if y is not None else "—"
997
- c_s = str(cites) if cites is not None else "—"
998
- ds_s = f"{dir_score:.3f}" if dir_score is not None else "—"
999
- dlt_s = f"{delta:.4f}" if delta is not None else "—"
1000
- qw_s = f"{qw:.3f}" if qw is not None else "—"
1001
- st.markdown(f"**Год:** {y_s} | **Цитаты:** {c_s} | **dir_score:** {ds_s} | **delta:** {dlt_s} | **quality:** {qw_s}")
1002
- st.caption(" | ".join(badges))
1003
-
1004
- with st.expander("Детали авторанжирования (опционально)", expanded=False):
1005
- comp = row.get("dir_native_components") or {}
1006
- if isinstance(comp, dict) and comp:
1007
- tr = safe_float(comp.get("trace"))
1008
- tp = safe_float(comp.get("topic"))
1009
- tx = safe_float(comp.get("text"))
1010
- nz = safe_float(comp.get("noise"))
1011
- sc = safe_float(comp.get("score"))
1012
- dl = safe_float(comp.get("delta"))
1013
- st.markdown(
1014
- f"**Компоненты (native DIR):** "
1015
- f"trace={tr if tr is not None else '—'} | "
1016
- f"topic={tp if tp is not None else '—'} | "
1017
- f"text={tx if tx is not None else '—'} | "
1018
- f"noise={nz if nz is not None else '—'} | "
1019
- f"score={sc if sc is not None else '—'} | "
1020
- f"delta={dl if dl is not None else '—'}"
1021
- )
1022
- expl = row.get("dir_score_explanation")
1023
- if expl:
1024
- st.markdown("**dir_score_explanation:**")
1025
- st.write(expl)
1026
-
1027
- links = []
1028
- if work_id:
1029
- links.append(f"[OpenAlex]({work_id})")
1030
- if doi_link:
1031
- links.append(f"[DOI]({doi_link})")
1032
- if isinstance(pdf_url, str) and pdf_url.strip():
1033
- links.append(f"[PDF]({pdf_url.strip()})")
1034
- if isinstance(pl_url, str) and pl_url.strip():
1035
- links.append(f"[Primary URL]({pl_url.strip()})")
1036
- if isinstance(oa_url, str) and oa_url.strip():
1037
- links.append(f"[OA URL]({oa_url.strip()})")
1038
- st.markdown("**Ссылки:** " + (" | ".join(links) if links else "—"))
1039
-
1040
- st.markdown("**Аннотация:**")
1041
- st.write(abstract)
1042
-
1043
- st.markdown(f"**Статус:** {status_str}")
1044
- st.markdown(f"**Кандидат {int(st.session_state[idx_key])+1}/{total}**")
1045
-
1046
- with score_col:
1047
- st.markdown("### Оценка")
1048
-
1049
- score_val = st.slider("Баллы (-5..+5)", -5, 5, int(st.session_state[score_key]), key=score_key)
1050
- comment_val = st.text_area("Комментарий", height=160, key=comment_key)
1051
 
1052
- if st.button("✅ Сохранить и далее", use_container_width=True):
1053
  review = {
1054
  "ts_utc": dt.datetime.utcnow().isoformat(timespec="seconds") + "Z",
1055
  "dir_id": selected_dir,
@@ -1067,44 +953,149 @@ with col_card:
1067
  "delta": delta,
1068
  "quality_weight": qw,
1069
  "components": row.get("dir_native_components") or {},
1070
- "rank": int(st.session_state[idx_key]),
1071
  },
1072
- "score": int(score_val),
1073
- "comment": str(comment_val or ""),
1074
  "reviewer": reviewer,
1075
  }
1076
 
1077
- # add to batch
1078
  st.session_state["batch"].append(review)
1079
 
1080
- # update local maps (для предзаполнения)
1081
- if source_id:
1082
- st.session_state["local_by_src"][source_id] = review
1083
- if work_id:
1084
- st.session_state["local_by_w"][work_id] = review
1085
-
1086
- # pending keys (оба)
1087
- if source_id:
1088
- st.session_state["reviewed_local_pending"].add(f"SRC::{source_id.strip()}")
1089
- if work_id:
1090
- st.session_state["reviewed_local_pending"].add(f"W::{work_id.strip()}")
1091
 
 
1092
  flushed = False
1093
  if len(st.session_state["batch"]) >= BATCH_SIZE:
1094
  with st.spinner("Сохраняю в reviews repo…"):
1095
  ok, msg = commit_batch()
1096
  if ok:
1097
  flushed = True
1098
- st.success("Сохранено")
1099
  else:
1100
  st.error(f"Не удалось записать в reviews repo: {msg}")
1101
- st.warning("Оценка сохранена локально как pending. Скачайте pending reviews или синхронизируйте позже.")
1102
 
1103
- if not flushed:
1104
- st.info(f"Добавлено в pending: {len(st.session_state['batch'])}/{BATCH_SIZE}")
 
 
1105
 
1106
- # move next
1107
- st.session_state[idx_key] = next_idx(int(st.session_state[idx_key]))
 
1108
  st.rerun()
1109
 
1110
- st.caption(f"Pending (несинхронизировано): {len(st.session_state.get('batch') or [])}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import os
2
  import json
3
  import uuid
4
  import gzip
5
  import tempfile
6
  import datetime as dt
7
+ import math
8
  from typing import List, Dict, Any, Optional, Tuple, Set
9
 
10
  import pandas as pd
 
12
  from huggingface_hub import hf_hub_download, HfApi
13
 
14
  # =========================
15
+ # ENV CONFIG (HF Space Variables / Secrets)
16
  # =========================
17
  PUBLICATIONS_REPO = os.environ.get("PUBLICATIONS_REPO", "").strip()
18
  REVIEWS_REPO = os.environ.get("REVIEWS_REPO", "").strip()
19
  HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HF_API_TOKEN")
20
+ ALLOW_CREATE_REVIEWS_REPO = (os.environ.get("ALLOW_CREATE_REVIEWS_REPO", "0").strip().lower() in ("1","true","yes","y","on"))
21
+ REVIEWS_PRIVATE = (os.environ.get("REVIEWS_PRIVATE", "1").strip().lower() in ("1","true","yes","y","on"))
22
 
23
+ # Надёжность по умолчанию: сразу пишем каждый review
 
 
24
  BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "1"))
 
25
 
26
+ PUB_DIR_REGISTRY_PATH = os.environ.get("PUB_DIR_REGISTRY_PATH", "dir_registry.json")
27
+ PUB_CANDIDATES_PREFIX = os.environ.get("PUB_CANDIDATES_PREFIX", "candidates")
28
+ REVIEWS_LOG_PREFIX = os.environ.get("REVIEWS_LOG_PREFIX", "reviews_log")
29
 
30
+ # Паттерны поиска candidates файлов (поддержка разных схем данных)
31
+ # Можно переопределить в Space Variables:
32
+ # PUB_CANDIDATES_PATTERNS="{prefix}/{dir_id}_top500.jsonl|{prefix}/{dir_id}.jsonl|manual_candidates/{dir_id}.jsonl|manual_candidates/{dir_id}_top500.jsonl"
33
  PUB_CANDIDATES_PATTERNS = os.environ.get(
34
  "PUB_CANDIDATES_PATTERNS",
35
+ "{prefix}/{dir_id}_top500.jsonl|{prefix}/{dir_id}_top500.jsonl.gz|{prefix}/{dir_id}.jsonl|{prefix}/{dir_id}.jsonl.gz|manual_candidates/{dir_id}.jsonl|manual_candidates/{dir_id}.jsonl.gz"
36
  ).strip()
37
 
38
+
39
  api = HfApi(token=HF_TOKEN)
40
 
41
+ st.set_page_config(page_title="ISS-GR Screening", layout="wide")
42
+
43
 
44
  # =========================
45
  # Helpers
46
  # =========================
47
 
48
+ def check_dataset_repo(repo_id: str, token: Optional[str]) -> Tuple[bool, str]:
49
+ """
50
+ Проверяет существование датасет-репозитория (repo_type="dataset").
51
+ Возвращает (ok, message). Если ok=False, message содержит причину.
52
+ """
53
+ try:
54
+ api_local = HfApi(token=token) if token else HfApi()
55
+ _ = api_local.repo_info(repo_id=repo_id, repo_type="dataset")
56
+ return True, "OK"
57
+ except Exception as e:
58
+ msg = str(e)
59
+ # huggingface_hub кидает 404/Repository Not Found
60
+ return False, msg
61
+
62
+ def maybe_create_reviews_repo(repo_id: str) -> Tuple[bool, str]:
63
+ """
64
+ Пытается создать reviews dataset repo, если включён ALLOW_CREATE_REVIEWS_REPO.
65
+ Возвращает (ok, message).
66
+ """
67
+ if not ALLOW_CREATE_REVIEWS_REPO:
68
+ return False, "ALLOW_CREATE_REVIEWS_REPO=0"
69
+ if not HF_TOKEN:
70
+ return False, "HF_TOKEN отсутствует (нужен для создания репозитория)"
71
+ try:
72
+ api.create_repo(
73
+ repo_id=repo_id,
74
+ repo_type="dataset",
75
+ private=bool(REVIEWS_PRIVATE),
76
+ exist_ok=True,
77
+ )
78
+ return True, "created_or_exists"
79
+ except Exception as e:
80
+ return False, str(e)
81
+
82
  def require_env(name: str, value: str) -> None:
83
  if not value:
84
  st.error(f"Не задано **{name}**. Укажи в Space Settings → Variables.")
85
  st.stop()
86
 
 
87
  def safe_int(x) -> Optional[int]:
88
  try:
89
  if x is None or pd.isna(x):
 
92
  except Exception:
93
  return None
94
 
 
95
  def safe_float(x) -> Optional[float]:
96
  try:
97
  if x is None or pd.isna(x):
 
100
  except Exception:
101
  return None
102
 
 
103
  def doi_url(doi: Optional[str]) -> Optional[str]:
104
  if not doi:
105
  return None
106
  doi = str(doi).strip()
 
 
107
  if doi.startswith("http://") or doi.startswith("https://"):
108
  return doi
109
  return f"https://doi.org/{doi}"
110
 
 
111
  def normalize_dir_id(dir_id: str, pad2: bool) -> str:
112
+ """
113
+ Поддержка форматов DIR1..DIR14 и DIR01..DIR14.
114
+ pad2=True -> DIR01
115
+ pad2=False -> DIR1
116
+ """
117
  if not isinstance(dir_id, str):
118
  return str(dir_id)
119
  s = dir_id.strip().upper()
 
123
  try:
124
  n = int(tail)
125
  except Exception:
126
+ # DIRXX (не число) — как есть
127
  return dir_id.strip()
128
  return f"DIR{n:02d}" if pad2 else f"DIR{n}"
129
 
 
130
  def dir_variants(dir_id: str) -> List[str]:
131
+ """
132
+ Список вариантов идентификатора DIR, чтобы подхватить разные имена файлов/папок.
133
+ """
134
  if not isinstance(dir_id, str):
135
  return [str(dir_id)]
136
  a = dir_id.strip()
137
  b = normalize_dir_id(a, pad2=True)
138
  c = normalize_dir_id(a, pad2=False)
139
+ out = []
140
  for x in [a, b, c]:
141
  if x and x not in out:
142
  out.append(x)
143
  return out
144
 
145
+ def dir_no(dir_id: str) -> str:
146
+ if isinstance(dir_id, str) and dir_id.upper().startswith("DIR"):
147
+ return dir_id[3:]
148
+ return str(dir_id)
 
 
 
 
149
 
150
  def join_terms(term_list: Any) -> str:
151
  if not term_list:
152
  return "—"
153
+ out = []
154
  for x in term_list:
155
  if isinstance(x, dict):
156
  t = x.get("t")
 
160
  out.append(str(x))
161
  return "; ".join(out) if out else "—"
162
 
 
163
  def topics_line(topics: Any) -> str:
164
  if not topics:
165
  return "—"
166
+ out = []
167
+ for t in (topics[:3] if isinstance(topics, list) else []):
168
+ if isinstance(t, dict):
169
+ tid = t.get("topic_id") or t.get("id")
170
+ name = t.get("topic_name") or t.get("display_name")
171
+ if tid and name:
172
+ out.append(f"{tid} {name}")
173
+ elif name:
174
+ out.append(str(name))
175
+ elif tid:
176
+ out.append(str(tid))
177
+ else:
178
+ out.append(str(t))
 
179
  return "; ".join(out) if out else "—"
180
 
181
+ def get_reviewer() -> str:
182
+ default = os.environ.get("REVIEWER", "")
183
+ v = st.sidebar.text_input("Рецензент (имя/ник)", value=default).strip()
184
+ return v or "anonymous"
185
 
186
  def decode_abstract(abstract_inverted_index: Optional[dict]) -> str:
187
  if not abstract_inverted_index or not isinstance(abstract_inverted_index, dict):
188
  return ""
189
+ pos_to_word = {}
190
  for w, positions in abstract_inverted_index.items():
191
  if not isinstance(positions, list):
192
  continue
193
  for p in positions:
194
+ pos_to_word[p] = w
 
 
 
195
  return " ".join(pos_to_word[p] for p in sorted(pos_to_word.keys())) if pos_to_word else ""
196
 
 
197
  def get_authors_and_abstract(row: Dict[str, Any]) -> Tuple[str, str]:
198
+ """
199
+ Offline-first: берём авторов/аннотацию ТОЛЬКО из candidates JSONL.
200
+ Никаких обращений к OpenAlex API.
201
+ """
202
+ # authors
203
+ authors = row.get('authors') or row.get('author_names') or row.get('authors_str')
204
  if not authors:
205
+ # Иногда candidates могут содержать OpenAlex-подобный authroships, но уже на входе (без сети)
206
+ auths = row.get('authorships')
207
  if isinstance(auths, list):
208
+ names = []
209
  for a in auths:
210
  if not isinstance(a, dict):
211
  continue
212
+ an = ((a.get('author') or {}).get('display_name') or '').strip()
213
  if an:
214
  names.append(an)
215
  if names:
216
+ authors = ', '.join(names[:12]) + (', и др.' if len(names) > 12 else '')
 
217
  if isinstance(authors, list):
218
+ authors = ', '.join([str(a) for a in authors if a])
219
+ authors_str = authors.strip() if isinstance(authors, str) and authors.strip() else ''
220
 
221
+ # abstract
222
+ abstract = row.get('abstract') or row.get('abstract_text')
223
+ if not abstract and isinstance(row.get('abstract_inverted_index'), dict):
224
+ abstract = decode_abstract(row.get('abstract_inverted_index'))
225
+ abstract_str = abstract.strip() if isinstance(abstract, str) and abstract.strip() else ''
 
226
 
227
+ return authors_str or '—', abstract_str or '—'
228
 
229
  def open_jsonl_any(path: str) -> List[Dict[str, Any]]:
230
+ rows = []
231
  if path.endswith(".gz"):
232
  with gzip.open(path, "rt", encoding="utf-8") as f:
233
  for line in f:
 
242
  rows.append(json.loads(line))
243
  return rows
244
 
 
245
  def normalize_candidate_row(obj: Dict[str, Any], selected_dir: str) -> Dict[str, Any]:
246
+ """
247
+ Приводит разные схемы данных к единому набору полей, используемых UI.
248
+ Поддерживаем:
249
+ - старый формат candidates: work_id/year/cited_by/dir_score/match_score
250
+ - новый формат паспорта: source_id/openalex_work_id/publication_year/cited_by_count/dir_native_components/quality_weight
251
+ """
252
  out = dict(obj)
253
 
254
+ # --- IDs ---
255
  source_id = out.get("source_id") or out.get("Source ID")
256
  if isinstance(source_id, str) and source_id.strip():
257
  out["source_id"] = source_id.strip()
258
 
259
  work_id = out.get("work_id")
260
  if not isinstance(work_id, str) or not work_id.strip():
261
+ # паспорта
262
  wid = out.get("openalex_work_id") or out.get("OpenAlex work ID") or out.get("dup_work_id")
263
  if isinstance(wid, str) and wid.strip():
264
  work_id = wid.strip()
265
+ out["work_id"] = work_id if isinstance(work_id, str) else None
266
 
267
+ # --- Title ---
268
  title = out.get("title") or out.get("display_name") or out.get("Title")
269
  out["title"] = str(title).strip() if title else "—"
270
 
271
+ # --- DOI ---
272
  doi = out.get("doi") or out.get("DOI")
273
  out["doi"] = str(doi).strip() if doi else None
274
 
275
+ # --- Abstract (offline) ---
276
  if out.get("abstract") is None:
277
  ab = out.get("abstract_text")
278
  if isinstance(ab, str) and ab.strip():
 
280
  elif isinstance(out.get("abstract_inverted_index"), dict):
281
  out["abstract"] = decode_abstract(out.get("abstract_inverted_index")) or None
282
 
283
+
284
+ # --- Year / cited_by ---
285
  year = out.get("year")
286
  if year is None:
287
  year = out.get("publication_year") or out.get("Publication year")
 
292
  cited_by = out.get("cited_by_count") or out.get("Cited by count")
293
  out["cited_by"] = safe_int(cited_by)
294
 
295
+ # --- Scores ---
296
+ # dir_score: приоритет
297
+ # 1) явное поле dir_score (старый candidates)
298
  ds = out.get("dir_score")
299
+ # 2) паспорта: dir_native_components.score (если selected_dir совпадает с родным)
300
  if ds is None:
301
  comp = out.get("dir_native_components") or {}
302
  if isinstance(comp, dict) and comp.get("score") is not None:
303
  ds = comp.get("score")
304
+ # 3) паспорта: dir_scores[selected_dir]
305
  if ds is None:
306
  dsd = out.get("dir_scores") or {}
307
  if isinstance(dsd, dict):
308
+ # пробуем разные варианты dir_id
309
  for dv in dir_variants(selected_dir):
310
  if dv in dsd:
311
  ds = dsd.get(dv)
312
  break
313
  out["dir_score"] = safe_float(ds)
314
 
315
+ # match_score: старое поле; если его нет — используем delta или quality_weight*100 для “второй метрики”
316
  ms = out.get("match_score")
317
  if ms is None:
318
  comp = out.get("dir_native_components") or {}
 
325
  ms = None
326
  out["match_score"] = safe_float(ms)
327
 
328
+ # quality_weight / components
329
+ qw = out.get("quality_weight")
330
+ out["quality_weight"] = safe_float(qw)
331
+ comp = out.get("dir_native_components")
332
+ out["dir_native_components"] = comp if isinstance(comp, dict) else {}
333
 
334
+ # links: pdf/primary location
335
  out["pdf_url"] = out.get("pdf_url") or out.get("PDF URL")
336
  out["primary_location_url"] = out.get("primary_location_url") or out.get("Primary location URL")
337
 
338
  return out
339
 
340
+ def review_key(row: Dict[str, Any]) -> str:
341
+ """
342
+ Ключ "уже оценено":
343
+ - если есть source_id — используем его (если вы теперь опираетесь на source_id)
344
+ - иначе work_id
345
+ """
346
+ sid = row.get("source_id")
347
  if isinstance(sid, str) and sid.strip():
348
+ return f"SRC::{sid.strip()}"
349
+ wid = row.get("work_id")
350
  if isinstance(wid, str) and wid.strip():
351
+ return f"W::{wid.strip()}"
352
+ return ""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
353
 
354
 
355
  # =========================
356
+ # Publications (read)
357
  # =========================
 
358
  @st.cache_data(show_spinner=False)
359
  def load_dir_registry(repo_id: str, filename: str) -> List[Dict[str, Any]]:
360
  path = hf_hub_download(repo_id=repo_id, filename=filename, repo_type="dataset", token=HF_TOKEN)
361
  with open(path, "r", encoding="utf-8") as f:
362
  return json.load(f)
363
 
 
364
  @st.cache_data(show_spinner=False)
365
  def load_candidates(repo_id: str, dir_id: str, prefix: str) -> pd.DataFrame:
366
+ """
367
+ Поддержка нескольких паттернов и вариантов DIR id (DIR1 vs DIR01).
368
+ """
369
  patterns = [p.strip() for p in PUB_CANDIDATES_PATTERNS.split("|") if p.strip()]
370
+ tried = []
371
+ rows = None
372
 
373
  for dv in dir_variants(dir_id):
374
  for pat in patterns:
 
377
  try:
378
  path = hf_hub_download(repo_id=repo_id, filename=fname, repo_type="dataset", token=HF_TOKEN)
379
  rows = open_jsonl_any(path)
380
+ if rows:
381
+ break
382
+ # если файл пустой — тоже считаем найденным
383
+ if rows == []:
384
+ break
385
  except Exception:
386
  continue
387
  if rows is not None:
 
390
  if rows is None:
391
  raise FileNotFoundError("Не найден candidates файл. Пробовали:\n" + "\n".join(tried[:30]) + ("\n..." if len(tried) > 30 else ""))
392
 
393
+ # normalize schema
394
  normed = [normalize_candidate_row(r, dir_id) for r in rows if isinstance(r, dict)]
395
  df = pd.DataFrame(normed)
396
 
397
+ # numeric columns
398
  for col in ["dir_score", "match_score", "cited_by", "year", "quality_weight"]:
399
  if col in df.columns:
400
  df[col] = pd.to_numeric(df[col], errors="coerce")
 
403
 
404
 
405
  # =========================
406
+ # Reviews (read on-demand)
407
  # =========================
 
408
  @st.cache_data(show_spinner=False)
409
  def list_review_files(repo_id: str, dir_id: str, prefix: str) -> List[str]:
410
  files = api.list_repo_files(repo_id=repo_id, repo_type="dataset")
411
+ # в reviews repo папка может быть DIR1 или DIR01 — подхватываем оба
412
  needles = [f"/{dv}/" for dv in dir_variants(dir_id)]
413
+ out = []
414
  for p in files:
415
  if not (p.startswith(prefix + "/") and p.endswith(".jsonl")):
416
  continue
 
418
  out.append(p)
419
  return out
420
 
 
421
  @st.cache_data(show_spinner=False)
422
+ def load_reviewed_keys(repo_id: str, dir_id: str, prefix: str) -> Set[str]:
423
+ """
424
+ Поддержка миграции:
425
+ - старые review содержали только work_id -> помечаем как W::<id>
426
+ - новые могут содержать source_id -> помечаем как SRC::<id>
 
 
427
  """
428
  try:
429
  files = list_review_files(repo_id, dir_id, prefix)
430
  except Exception:
431
+ return set()
 
 
 
 
432
 
433
+ reviewed: Set[str] = set()
434
  for relpath in files:
435
  try:
436
  path = hf_hub_download(repo_id=repo_id, filename=relpath, repo_type="dataset", token=HF_TOKEN)
 
442
  obj = json.loads(line)
443
 
444
  sid = obj.get("source_id")
 
 
445
  if isinstance(sid, str) and sid.strip():
446
+ reviewed.add(f"SRC::{sid.strip()}")
447
+
448
+ wid = obj.get("work_id")
449
  if isinstance(wid, str) and wid.strip():
450
+ reviewed.add(f"W::{wid.strip()}")
 
 
451
  except Exception:
452
  continue
453
+ return reviewed
 
454
 
455
 
456
  # =========================
457
+ # Reviews (write)
458
  # =========================
 
459
  def push_batch_to_reviews_repo(repo_id: str, dir_id: str, prefix: str, batch: List[Dict[str, Any]]) -> None:
460
  if not batch:
461
  return
462
  if not HF_TOKEN:
463
  raise RuntimeError("Нет HF_TOKEN (Secret) — нельзя записывать в reviews dataset.")
 
464
  today = dt.date.today().isoformat()
465
  batch_id = str(uuid.uuid4())
466
+
467
+ # Пишем в dir_id как есть (но если dir_id может быть DIR01, а у вас в БД DIR1 — нормализуйте в UI выбором)
468
  canonical_dir = normalize_dir_id(dir_id, pad2=False)
469
  path_in_repo = f"{prefix}/{today}/{canonical_dir}/{batch_id}.jsonl"
470
 
 
483
 
484
 
485
  # =========================
486
+ # Sidebar
487
  # =========================
488
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
489
  # =========================
490
+ # MAIN APP (offline-first)
491
  # =========================
492
 
493
  with st.sidebar:
494
+ st.title("ISS-GR Скрининг (offline-first)")
495
 
496
  require_env("PUBLICATIONS_REPO", PUBLICATIONS_REPO)
497
  require_env("REVIEWS_REPO", REVIEWS_REPO)
498
 
499
+ # Проверяем наличие репозиториев (чтобы не ловить 404 при сохранении)
500
  pub_ok, pub_msg = check_dataset_repo(PUBLICATIONS_REPO, HF_TOKEN)
501
  if not pub_ok:
502
  st.error("PUBLICATIONS_REPO недоступен как dataset repo. Проверь repo_id и доступ.\n\n" + pub_msg)
 
504
 
505
  rev_ok, rev_msg = check_dataset_repo(REVIEWS_REPO, HF_TOKEN)
506
  if not rev_ok:
507
+ created, cmsg = maybe_create_reviews_repo(REVIEWS_REPO)
508
  if created:
509
  rev_ok2, rev_msg2 = check_dataset_repo(REVIEWS_REPO, HF_TOKEN)
510
  if not rev_ok2:
511
  st.error("REVIEWS_REPO недоступен после create_repo.\n\n" + rev_msg2)
512
  st.stop()
513
+ else:
514
+ st.success("REVIEWS_REPO создан/доступен.")
515
  else:
516
  st.error(
517
  "REVIEWS_REPO недоступен (404/нет доступа).\n\n"
518
+ "Что проверить:\n"
519
+ "1) В Space Variables: REVIEWS_REPO = <owner>/<dataset_name> (именно dataset).\n"
520
+ "2) Репозиторий реально существует на Hub.\n"
521
+ "3) Если repo приватный в Secrets должен быть HF_TOKEN с правами write на этот dataset.\n"
522
+ "4) Если repo ещё не создан — включи ALLOW_CREATE_REVIEWS_REPO=1 (опционально) или создай вручную.\n\n"
523
  + rev_msg
524
  )
525
  st.stop()
526
 
527
+ # =========================
 
528
 
529
+ # =========================
530
+ # Load DIR registry + UI (offline-first)
531
+ # =========================
532
+ dirs = load_dir_registry(PUBLICATIONS_REPO, PUB_DIR_REGISTRY_PATH)
533
+ dir_ids = [d.get("dir_id") for d in dirs if d.get("dir_id")]
534
+ if not dir_ids:
535
+ st.error("В dir_registry.json не найдено ни одного dir_id.")
536
+ st.stop()
537
 
538
+ dir_map = {d.get("dir_id"): d for d in dirs if d.get("dir_id")}
539
 
540
+ def _fmt_dir(did: str) -> str:
541
+ meta = dir_map.get(did) or {}
542
+ name = meta.get("dir_name") or "—"
543
+ return f"DIR-{str(dir_no(did)).zfill(2)} {name}"
 
 
544
 
545
+ st.title("Скоринг публикаций")
546
+
547
+ # Выбор DIR — в основном интерфейсе (не в сайдбаре)
548
+ selected_dir = st.selectbox("DIR", dir_ids, index=0, format_func=_fmt_dir)
549
+ dir_meta = dir_map.get(selected_dir) or {}
550
 
551
  # =========================
552
+ # Sidebar (минимум)
553
  # =========================
554
+ with st.sidebar:
555
+ reviewer = get_reviewer()
556
+ st.caption(f"BATCH_SIZE={BATCH_SIZE} (настройка через переменную окружения)")
557
+ st.divider()
558
+ flush_now = st.button("⬆️ Синхронизировать pending сейчас", use_container_width=True)
559
+ clear_cache = st.button("🧹 Сбросить кэш", use_container_width=True)
560
 
561
+ if clear_cache:
562
  load_dir_registry.clear()
563
  load_candidates.clear()
564
+ if hasattr(list_review_files, "clear"):
565
+ list_review_files.clear()
566
+ if hasattr(load_reviewed_keys, "clear"):
567
+ load_reviewed_keys.clear()
568
+ if hasattr(load_review_index, "clear"):
569
+ load_review_index.clear()
570
  for k in [
571
+ "reviewed_remote",
572
+ "reviewed_remote_dir",
573
+ "review_index_remote",
574
+ "review_index_local",
575
  "reviewed_local_committed",
576
  "reviewed_local_pending",
 
 
577
  "batch",
578
  ]:
579
  if k in st.session_state:
580
  del st.session_state[k]
581
+ st.toast("Кэш очищен.")
 
582
  st.rerun()
583
 
 
584
  # =========================
585
+ # DIR header
586
  # =========================
587
+ defaults = (dir_meta.get("defaults") or {})
588
+ year_from = defaults.get("year_from")
589
+ year_to = defaults.get("year_to")
590
+
591
+ terms = (dir_meta.get("terms") or {})
592
+ anchor_str = join_terms(terms.get("anchor"))
593
+ support_str = join_terms(terms.get("support"))
594
+ noise_str = join_terms(terms.get("noise"))
595
+ topics_str = topics_line(dir_meta.get("topics") or [])
596
+
597
+ dir_desc = dir_meta.get("dir_description", "—")
598
+
599
+ st.markdown(
600
+ f"""
601
+ <div style="padding: 14px 16px; border: 1px solid rgba(49,51,63,0.2); border-radius: 12px; background: rgba(49,51,63,0.04);">
602
+ <div style="margin-top: 0px;">
603
+ <div style="font-size: 12px; font-weight: 700; text-transform: uppercase; opacity: 0.7;">Краткое описание</div>
604
+ <div style="margin-top: 4px; font-size: 14px; line-height: 1.35;">{dir_desc}</div>
605
+ </div>
606
+ </div>
607
+ """,
608
+ unsafe_allow_html=True,
609
  )
610
 
611
+ with st.expander("Детали DIR", expanded=True):
612
+ st.markdown(f"**Временной интервал:** {year_from} – {year_to}")
613
+ st.markdown(f"**Якоря:** {anchor_str}")
614
+ st.markdown(f"**Поддержка:** {support_str}")
615
+ st.markdown(f"**Шум:** {noise_str}")
616
+ st.markdown(f"**Topics поиска:** {topics_str}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
617
 
618
  st.divider()
619
 
 
620
  # =========================
621
+ # Load candidates + fixed sorting (A: dir_score ↓)
622
  # =========================
 
623
  try:
624
  df = load_candidates(PUBLICATIONS_REPO, selected_dir, PUB_CANDIDATES_PREFIX)
625
  except Exception as e:
 
627
  st.stop()
628
 
629
  if df.empty:
630
+ st.warning(f"Пустой список кандидатов для {selected_dir}.")
631
  st.stop()
632
 
633
+ # Fixed order: dir_score ↓ (stable ties)
634
+ sort_cols: List[str] = []
635
+ sort_asc: List[bool] = []
636
+ if "dir_score" in df.columns:
637
+ sort_cols.append("dir_score"); sort_asc.append(False)
638
+ if "quality_weight" in df.columns:
639
+ sort_cols.append("quality_weight"); sort_asc.append(False)
640
+ if "match_score" in df.columns:
641
+ sort_cols.append("match_score"); sort_asc.append(False)
642
+ if "year" in df.columns:
643
+ sort_cols.append("year"); sort_asc.append(False)
644
+ if sort_cols:
645
+ df = df.sort_values(sort_cols, ascending=sort_asc)
646
 
647
  df = df.reset_index(drop=True)
648
+ total = len(df)
649
+ if total == 0:
650
+ st.warning("Пустой список кандидатов.")
651
+ st.stop()
652
 
653
  # =========================
654
+ # State: batch + local/remote indexes
655
  # =========================
656
+ def ensure_state():
657
+ if "batch" not in st.session_state:
658
+ st.session_state["batch"] = []
659
+ if "reviewed_local_committed" not in st.session_state:
660
+ st.session_state["reviewed_local_committed"] = set()
661
+ if "reviewed_local_pending" not in st.session_state:
662
+ st.session_state["reviewed_local_pending"] = set()
663
+ if "reviewed_remote" not in st.session_state:
664
+ st.session_state["reviewed_remote"] = set()
665
+ if "reviewed_remote_dir" not in st.session_state:
666
+ st.session_state["reviewed_remote_dir"] = None
667
+ if "review_index_remote" not in st.session_state:
668
+ st.session_state["review_index_remote"] = {}
669
+ if "review_index_local" not in st.session_state:
670
+ st.session_state["review_index_local"] = {}
671
 
672
  ensure_state()
673
 
674
+ def row_keys_all(r: Dict[str, Any]) -> Set[str]:
675
+ keys = set()
676
+ sid = r.get("source_id")
677
+ if isinstance(sid, str) and sid.strip():
678
+ keys.add(f"SRC::{sid.strip()}")
679
+ wid = r.get("work_id")
680
+ if isinstance(wid, str) and wid.strip():
681
+ keys.add(f"W::{wid.strip()}")
682
+ return keys
 
 
 
 
683
 
684
+ canonical_dir = normalize_dir_id(selected_dir, pad2=False)
685
 
686
+ # Загрузка сохранённых оценок (тихо; remote-статусы не показываем)
687
+ need_reload = (st.session_state["reviewed_remote_dir"] != canonical_dir)
688
+ if need_reload:
689
+ # Без спиннера и без UI-вывода: просто подготавливаем пропуск/предзаполнение
690
+ st.session_state["reviewed_remote"] = load_reviewed_keys(REVIEWS_REPO, selected_dir, REVIEWS_LOG_PREFIX)
691
+ st.session_state["review_index_remote"] = load_review_index(REVIEWS_REPO, selected_dir, REVIEWS_LOG_PREFIX)
692
+ st.session_state["reviewed_remote_dir"] = canonical_dir
693
 
694
+ reviewed_committed = set(st.session_state["reviewed_remote"]) | set(st.session_state["reviewed_local_committed"])
695
+ reviewed_pending = set(st.session_state["reviewed_local_pending"])
696
+ reviewed_effective = reviewed_committed | reviewed_pending
697
 
698
+ def status_of_row(r: Dict[str, Any]) -> str:
699
  keys = row_keys_all(r)
700
  if keys and any(k in reviewed_committed for k in keys):
701
  return "✅"
 
703
  return "🕓"
704
  return "🆕"
705
 
706
+ def is_reviewed_effective(r: Dict[str, Any]) -> bool:
707
+ keys = row_keys_all(r)
708
+ return bool(keys and any(k in reviewed_effective for k in keys))
709
+
710
+ def last_review_for_row(r: Dict[str, Any]) -> Optional[Dict[str, Any]]:
711
+ """Берём последнюю известную оценку (pending/local -> remote)."""
712
+ keys = list(row_keys_all(r))
713
+ if not keys:
714
+ return None
715
+
716
+ best = None
717
+ best_ts = ""
718
+ # local index first
719
+ for k in keys:
720
+ obj = st.session_state["review_index_local"].get(k)
721
+ if isinstance(obj, dict):
722
+ t = str(obj.get("ts_utc") or obj.get("ts") or "")
723
+ if t > best_ts:
724
+ best_ts = t
725
+ best = obj
726
+ # remote index fallback
727
+ for k in keys:
728
+ obj = st.session_state["review_index_remote"].get(k)
729
+ if isinstance(obj, dict):
730
+ t = str(obj.get("ts_utc") or obj.get("ts") or "")
731
+ if t > best_ts:
732
+ best_ts = t
733
+ best = obj
734
+ return best
735
+
736
+ # =========================
737
+ # Sidebar: statuses (без remote)
738
+ # =========================
739
+ def split_key_set(keys: Set[str]) -> Tuple[Set[str], Set[str]]:
740
+ src_ids: Set[str] = set()
741
+ w_ids: Set[str] = set()
742
+ for k in keys:
743
+ if isinstance(k, str) and k.startswith("SRC::"):
744
+ src_ids.add(k[5:])
745
+ elif isinstance(k, str) and k.startswith("W::"):
746
+ w_ids.add(k[3:])
747
+ return src_ids, w_ids
748
+
749
+ comm_src_ids, comm_w_ids = split_key_set(reviewed_committed)
750
+ pend_src_ids, pend_w_ids = split_key_set(reviewed_pending)
751
 
 
752
  src_series = df["source_id"].fillna("") if "source_id" in df.columns else pd.Series([""] * len(df))
753
  wid_series = df["work_id"].fillna("") if "work_id" in df.columns else pd.Series([""] * len(df))
754
 
755
+ mask_committed = src_series.astype(str).isin(comm_src_ids) | wid_series.astype(str).isin(comm_w_ids)
756
+ mask_pending = src_series.astype(str).isin(pend_src_ids) | wid_series.astype(str).isin(pend_w_ids)
 
 
 
 
 
757
  mask_reviewed = mask_committed | mask_pending
758
 
759
  committed_pub_count = int(mask_committed.sum())
 
761
  unreviewed_pub_count = int((~mask_reviewed).sum())
762
 
763
  with st.sidebar:
764
+ with st.expander("Статусы", expanded=False):
765
+ st.write(f"✅ сохранено: {committed_pub_count} публикаций")
766
+ st.write(f"🕓 pending: {pending_pub_count} публикаций")
767
 
768
  if st.session_state.get("batch"):
769
  pending_jsonl = "".join(json.dumps(x, ensure_ascii=False) + "\n" for x in st.session_state["batch"])
 
775
  use_container_width=True,
776
  )
777
 
 
778
  # =========================
779
+ # Commit helper
780
  # =========================
 
781
  def commit_batch() -> Tuple[bool, str]:
782
  batch = st.session_state.get("batch") or []
783
  if not batch:
784
  return False, "batch_empty"
 
785
  try:
786
  push_batch_to_reviews_repo(REVIEWS_REPO, selected_dir, REVIEWS_LOG_PREFIX, batch)
787
  except Exception as e:
788
  return False, str(e)
789
 
790
+ # Update local sets: pending -> committed (keys)
791
  for obj in batch:
792
  sid = obj.get("source_id")
793
  wid = obj.get("work_id")
 
803
  st.session_state["batch"] = []
804
  return True, "ok"
805
 
806
+ # manual flush
807
+ if flush_now and st.session_state.get("batch"):
808
  with st.spinner("Синхронизирую pending батч…"):
809
  ok, msg = commit_batch()
810
  if ok:
811
+ st.success("Pending батч синхронизирован.")
812
  st.rerun()
813
  else:
814
  st.error(f"Не удалось синхронизировать: {msg}")
815
 
816
+ # recompute after possible commit
817
+ reviewed_committed = set(st.session_state["reviewed_remote"]) | set(st.session_state["reviewed_local_committed"])
818
+ reviewed_pending = set(st.session_state["reviewed_local_pending"])
819
+ reviewed_effective = reviewed_committed | reviewed_pending
820
 
821
  # =========================
822
+ # Navigation window + layout
823
  # =========================
 
824
  idx_key = f"idx_{canonical_dir}"
825
  if idx_key not in st.session_state:
826
  st.session_state[idx_key] = 0
827
 
828
+ current_idx = int(st.session_state[idx_key])
829
+ current_idx = max(0, min(total - 1, current_idx))
830
+ st.session_state[idx_key] = current_idx
831
+
832
+ window_size = int(PAGE_WINDOW_SIZE) if PAGE_WINDOW_SIZE else 50
833
+ window_start = int(current_idx // window_size) * int(window_size)
834
+ window_end = min(window_start + int(window_size), total)
835
+ page_indices = list(range(window_start, window_end))
836
 
837
+ def fmt_idx(i: int) -> str:
838
+ r = df.iloc[i].to_dict()
839
+ ico = status_of_row(r)
840
+ year_val = safe_int(r.get("year"))
841
+ year = str(year_val) if year_val is not None else "—"
842
+ title = (str(r.get("title")) if r.get("title") is not None else "—")
843
+ title_short = title if len(title) <= 80 else title[:77] + "…"
844
+ return f"{ico} {i+1:03d} | {year} | {title_short}"
845
+
846
+ def next_unreviewed(from_idx: int) -> int:
847
+ i = max(0, from_idx)
848
  while i < total:
 
 
849
  r = df.iloc[i].to_dict()
850
  if is_reviewed_effective(r):
851
  i += 1
852
  continue
853
  return i
854
+ return total - 1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
855
 
856
+ # main 2-column layout: left=list+scoring, right=publication
857
+ col_left, col_right = st.columns([1, 2], gap="large")
 
 
 
 
 
 
858
 
859
+ with col_left:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
860
  st.markdown("### Список публикаций")
861
+ st.caption(f"Окно: {window_start+1}–{window_end} из {total} Неоценено: {unreviewed_pub_count}")
862
 
863
  sel = st.selectbox(
864
+ "Публикации в текущем окне",
865
+ options=page_indices,
866
+ index=page_indices.index(current_idx) if current_idx in page_indices else 0,
867
  format_func=fmt_idx,
868
  label_visibility="collapsed",
869
  )
 
872
  st.session_state[idx_key] = int(sel)
873
  st.rerun()
874
 
875
+ # актуальная строка
876
+ current_idx = int(st.session_state[idx_key])
877
+ row = df.iloc[current_idx].to_dict()
878
+
879
+ # предзаполнение score/comment: если есть оценка -> она, иначе 0/пусто
880
+ existing = last_review_for_row(row) or {}
881
+ existing_score = safe_int(existing.get("score"))
882
+ if existing_score is None:
883
+ existing_score = 0
884
+ existing_comment = str(existing.get("comment") or "")
885
+
886
+ score_key = f"score_{canonical_dir}_{current_idx}"
887
+ comment_key = f"com_{canonical_dir}_{current_idx}"
888
+
889
+ if score_key not in st.session_state:
890
+ st.session_state[score_key] = int(existing_score)
891
+ # защита от старых/некорректных значений
892
+ if int(st.session_state[score_key]) not in (-2, -1, 0, 1, 2):
893
+ st.session_state[score_key] = 0
894
+
895
+ if comment_key not in st.session_state:
896
+ st.session_state[comment_key] = existing_comment
897
+
898
+ st.markdown("### Оценка")
899
+
900
+ label_map = {
901
+ -2: "точно нет",
902
+ -1: "скорее нет",
903
+ 0: "не знаю",
904
+ 1: "скорее да",
905
+ 2: "точно да",
906
+ }
907
+
908
+ score = st.radio(
909
+ "Оценка",
910
+ options=[-2, -1, 0, 1, 2],
911
+ format_func=lambda v: f"{v:+d} — {label_map.get(v, '')}".replace("+0", "0"),
912
+ index=[-2, -1, 0, 1, 2].index(int(st.session_state[score_key])),
913
+ key=score_key,
914
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
915
 
916
+ comment = st.text_area("Комментарий", height=140, key=comment_key)
917
 
918
+ b1, b2, b3 = st.columns([1, 2, 1])
 
919
 
920
+ with b1:
921
+ if st.button("⬅️ Назад", use_container_width=True):
922
+ st.session_state[idx_key] = max(0, current_idx - 1)
923
+ st.rerun()
924
 
925
+ with b2:
926
+ if st.button(" Сохрани��ь и далее", use_container_width=True):
927
+ # собираем мета для review
928
+ authors, abstract = get_authors_and_abstract(row)
929
 
930
+ y = safe_int(row.get("year"))
931
+ cites = safe_int(row.get("cited_by"))
932
+ dir_score = safe_float(row.get("dir_score"))
933
+ delta = safe_float(row.get("match_score"))
934
+ qw = safe_float(row.get("quality_weight"))
935
 
936
+ work_id = row.get("work_id") if isinstance(row.get("work_id"), str) else None
937
+ source_id = row.get("source_id") if isinstance(row.get("source_id"), str) else None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
938
 
 
939
  review = {
940
  "ts_utc": dt.datetime.utcnow().isoformat(timespec="seconds") + "Z",
941
  "dir_id": selected_dir,
 
953
  "delta": delta,
954
  "quality_weight": qw,
955
  "components": row.get("dir_native_components") or {},
956
+ "rank": int(current_idx),
957
  },
958
+ "score": int(score),
959
+ "comment": comment,
960
  "reviewer": reviewer,
961
  }
962
 
 
963
  st.session_state["batch"].append(review)
964
 
965
+ # pending keys + local index for prefill
966
+ for k in row_keys_all(row):
967
+ st.session_state["reviewed_local_pending"].add(k)
968
+ st.session_state["review_index_local"][k] = review
 
 
 
 
 
 
 
969
 
970
+ # auto flush by BATCH_SIZE
971
  flushed = False
972
  if len(st.session_state["batch"]) >= BATCH_SIZE:
973
  with st.spinner("Сохраняю в reviews repo…"):
974
  ok, msg = commit_batch()
975
  if ok:
976
  flushed = True
977
+ st.success("Сохранено.")
978
  else:
979
  st.error(f"Не удалось записать в reviews repo: {msg}")
980
+ st.warning("Оценка сохранена локально как pending. Скачайте pending reviews или повторите синхронизацию.")
981
 
982
+ # переход: следующая НЕоценённая (без тумблера)
983
+ nxt = next_unreviewed(current_idx + 1)
984
+ st.session_state[idx_key] = nxt
985
+ st.rerun()
986
 
987
+ with b3:
988
+ if st.button("➡️ Далее", use_container_width=True):
989
+ st.session_state[idx_key] = min(total - 1, current_idx + 1)
990
  st.rerun()
991
 
992
+ st.caption(f"Pending: {len(st.session_state.get('batch') or [])}")
993
+
994
+ with col_right:
995
+ # карточка публикации занимает максимум пространства справа
996
+ row = df.iloc[int(st.session_state[idx_key])].to_dict()
997
+
998
+ title = row.get("title") or "—"
999
+ authors, abstract = get_authors_and_abstract(row)
1000
+
1001
+ y = safe_int(row.get("year"))
1002
+ cites = safe_int(row.get("cited_by"))
1003
+ dir_score = safe_float(row.get("dir_score"))
1004
+ delta = safe_float(row.get("match_score"))
1005
+ qw = safe_float(row.get("quality_weight"))
1006
+
1007
+ work_id = row.get("work_id") if isinstance(row.get("work_id"), str) else None
1008
+ source_id = row.get("source_id") if isinstance(row.get("source_id"), str) else None
1009
+
1010
+ doi_link = doi_url(row.get("doi"))
1011
+ pdf_url = row.get("pdf_url")
1012
+ pl_url = row.get("primary_location_url")
1013
+
1014
+ oa = row.get("open_access") if isinstance(row.get("open_access"), dict) else {}
1015
+ oa_is = oa.get("is_oa") if isinstance(oa, dict) else None
1016
+ oa_status = oa.get("oa_status") if isinstance(oa, dict) else None
1017
+ oa_url = oa.get("oa_url") if isinstance(oa, dict) else None
1018
+
1019
+ keys = row_keys_all(row)
1020
+ committed = bool(keys and any(k in reviewed_committed for k in keys))
1021
+ pending = bool(keys and any(k in reviewed_pending for k in keys))
1022
+ status_str = "✅ оценено" if committed else ("🕓 pending" if pending else "🆕 не оценено")
1023
+
1024
+ # badges
1025
+ has_abs = (isinstance(abstract, str) and abstract.strip() and abstract.strip() != "—")
1026
+ has_pdf = (isinstance(pdf_url, str) and isinstance(pdf_url, str) and pdf_url.strip())
1027
+ has_pl = (isinstance(pl_url, str) and pl_url.strip())
1028
+ has_doi = bool(doi_link)
1029
+ badges = [
1030
+ f"Abstract {'✅' if has_abs else '❌'}",
1031
+ f"PDF {'✅' if has_pdf else '❌'}",
1032
+ f"URL {'✅' if has_pl else '❌'}",
1033
+ f"DOI {'✅' if has_doi else '❌'}",
1034
+ ]
1035
+ if oa_is is not None:
1036
+ badges.append(f"OA {'✅' if oa_is else '❌'}{(' (' + str(oa_status) + ')') if oa_status else ''}")
1037
+
1038
+ st.markdown(f"## {title}")
1039
+ st.caption(status_str + " • " + " | ".join(badges))
1040
+
1041
+ id_lines = []
1042
+ if source_id:
1043
+ id_lines.append(f"**Source ID:** `{source_id}`")
1044
+ if work_id:
1045
+ id_lines.append(f"**Work ID:** `{work_id}`")
1046
+ if id_lines:
1047
+ st.markdown(" • ".join(id_lines))
1048
+
1049
+ # Метрики
1050
+ y_s = str(y) if y is not None else "—"
1051
+ c_s = str(cites) if cites is not None else "—"
1052
+ ds_s = f"{dir_score:.3f}" if dir_score is not None else "—"
1053
+ dlt_s = f"{delta:.4f}" if delta is not None else "—"
1054
+ qw_s = f"{qw:.3f}" if qw is not None else "—"
1055
+ st.markdown(f"**Год:** {y_s} | **Цитаты:** {c_s} | **dir_score:** {ds_s} | **delta:** {dlt_s} | **quality:** {qw_s}")
1056
+
1057
+ # Авторы
1058
+ if authors and authors != "—":
1059
+ st.markdown("**Авторы:**")
1060
+ st.write(authors)
1061
+ else:
1062
+ st.caption("Авторы: — (в исходных DIR*.jsonl авторы не предоставляются)")
1063
+
1064
+ # Ссылки
1065
+ links = []
1066
+ if work_id:
1067
+ links.append(f"[OpenAlex]({work_id})")
1068
+ if doi_link:
1069
+ links.append(f"[DOI]({doi_link})")
1070
+ if isinstance(pdf_url, str) and pdf_url.strip():
1071
+ links.append(f"[PDF]({pdf_url.strip()})")
1072
+ if isinstance(pl_url, str) and pl_url.strip():
1073
+ links.append(f"[Primary URL]({pl_url.strip()})")
1074
+ if isinstance(oa_url, str) and oa_url.strip():
1075
+ links.append(f"[OA URL]({oa_url.strip()})")
1076
+ st.markdown("**Ссылки:** " + (" | ".join(links) if links else "—"))
1077
+
1078
+ # Аннотация
1079
+ st.markdown("### Аннотация")
1080
+ st.write(abstract if abstract else "—")
1081
+
1082
+ # Доп. детали автооценки (по желанию)
1083
+ with st.expander("Детали автооценки", expanded=True):
1084
+ comp = row.get("dir_native_components") or {}
1085
+ if isinstance(comp, dict) and comp:
1086
+ tr = comp.get("trace"); tp = comp.get("topic"); tx = comp.get("text"); nz = comp.get("noise")
1087
+ sc = comp.get("score"); dl = comp.get("delta")
1088
+ st.markdown(
1089
+ f"**Компоненты (native DIR):** "
1090
+ f"trace={tr if tr is not None else '—'} | "
1091
+ f"topic={tp if tp is not None else '—'} | "
1092
+ f"text={tx if tx is not None else '—'} | "
1093
+ f"noise={nz if nz is not None else '—'} | "
1094
+ f"score={sc if sc is not None else '—'} | "
1095
+ f"delta={dl if dl is not None else '—'}"
1096
+ )
1097
+ expl = row.get("dir_score_explanation")
1098
+ if expl:
1099
+ st.markdown("**dir_score_explanation:**")
1100
+ st.write(expl)
1101
+