Update app.py

app.py

@@ -2,9 +2,8 @@ import os
 import io
 import json
 import uuid
-import re
-from typing import Optional, Tuple, List, Dict, Any
 from datetime import datetime, timezone
+from typing import Optional, Tuple, List, Any
 
 import numpy as np
 import pandas as pd
@@ -30,7 +29,7 @@ HF_REQUESTS_REPO = os.getenv("HF_REQUESTS_REPO", "PlanetExpress2125/PostDocReque
 HF_REQUESTS_REPO_TYPE = os.getenv("HF_REQUESTS_REPO_TYPE", "dataset")  # dataset | model
 HF_WRITE_TOKEN = os.getenv("HF_WRITE_TOKEN")  # write token (рекомендуется)
 
-# Компактное OA-обогащение (вы уже собрали)
+# Компактное OA-обогащение (собрано вами)
 OA_ENRICH_REPO = os.getenv("OA_ENRICH_REPO", "yogl/postdoc_oa_enrichment")
 
 SLIDER_MIN_YEAR = 2005  # нижняя граница диапазона лет
@@ -84,6 +83,22 @@ except Exception:
     pass
 
 
+# ==========================
+# ВСПОМОГАТЕЛЬНЫЕ ФУНКЦИИ
+# ==========================
+
+def _norm_regnum(x: Any) -> str:
+    s = "" if x is None else str(x).strip()
+    # иногда номера могут попасть как float "123.0"
+    if s.endswith(".0") and s[:-2].isdigit():
+        s = s[:-2]
+    return s
+
+
+def _keyify(label: str) -> str:
+    return "k_" + "".join(ch if ch.isalnum() else "_" for ch in label).strip("_")
+
+
 # ==========================
 # СПРАВОЧНИКИ ФИЛЬТРОВ (UI)
 # ==========================
@@ -112,12 +127,7 @@ SCIENCE_LABELS = [
 ]
 SCIENCE_LABELS = sorted(list(dict.fromkeys(SCIENCE_LABELS)), key=lambda s: s.casefold())
 
-DEFAULT_SCIENCES = {
-    "Технические",
-    "Физико-математические",
-    "Химические",
-    "Биологические",
-}
+DEFAULT_SCIENCES = {"Технические", "Физико-математические", "Химические", "Биологические"}
 
 SCIENCE_PATTERNS = {
     "Архитектура": ["архитектур"],
@@ -143,10 +153,6 @@ SCIENCE_PATTERNS = {
 }
 
 
-def _keyify(label: str) -> str:
-    return "k_" + "".join(ch if ch.isalnum() else "_" for ch in label).strip("_")
-
-
 # ==========================
 # ЗАПИСЬ ЗАЯВОК В HF REPO
 # ==========================
@@ -186,10 +192,24 @@ def load_oa_enrichment() -> pd.DataFrame:
     """
     ds = load_dataset(OA_ENRICH_REPO, split="train")
     df = ds.to_pandas()
+
     if "registration_number" not in df.columns:
-        return pd.DataFrame().set_index(pd.Index([], name="registration_number"))
-    df["registration_number"] = df["registration_number"].astype(str)
-    return df.set_index("registration_number", drop=True)
+        return pd.DataFrame().set_index(pd.Index([], name="reg_norm"))
+
+    df["reg_norm"] = df["registration_number"].apply(_norm_regnum)
+
+    # совместимость, если в датасете использовались другие имена колонок
+    if "openalex_url" not in df.columns and "openalex" in df.columns:
+        df["openalex_url"] = df["openalex"]
+    if "orcid_url" not in df.columns and "orcid" in df.columns:
+        df["orcid_url"] = df["orcid"]
+
+    keep = ["reg_norm", "openalex_url", "orcid_url", "h_index", "i10_index", "works_count", "cited_by_count"]
+    keep = [c for c in keep if c in df.columns]
+    df = df[keep].copy()
+
+    df = df.drop_duplicates(subset=["reg_norm"]).set_index("reg_norm", drop=True)
+    return df
 
 
 oa_enrich = load_oa_enrichment()
@@ -203,12 +223,12 @@ oa_enrich = load_oa_enrichment()
 def load_data():
     ds_meta = load_dataset(HF_MERGED_REPO, split="train")
     df_meta = ds_meta.to_pandas()
-    df_meta["registration_number"] = df_meta["registration_number"].astype(str)
+    df_meta["registration_number"] = df_meta["registration_number"].astype(str).map(_norm_regnum)
     df_meta = df_meta.set_index("registration_number", drop=False)
 
     ds_emb = load_dataset(HF_EMB_REPO, split="train")
     df_emb = ds_emb.to_pandas()
-    df_emb["registration_number"] = df_emb["registration_number"].astype(str)
+    df_emb["registration_number"] = df_emb["registration_number"].astype(str).map(_norm_regnum)
 
     reg_nums = df_emb["registration_number"].tolist()
 
@@ -340,7 +360,7 @@ def extract_year_int(value) -> Optional[int]:
 def build_result_df(results):
     rows = []
     for r in results:
-        reg = r["registration_number"]
+        reg = _norm_regnum(r["registration_number"])
         score = r["score"]
 
         if reg in df_all.index:
@@ -384,27 +404,34 @@ def run_search(
     results = search_core(query, top_k, mask=mask)
     df_raw = build_result_df(results)
 
-    # OA enrichment join (по registration_number)
+    # OA enrichment join (выровнять по позициям, а не по индексу!)
     if not df_raw.empty and not oa_enrich.empty:
-        regs = df_raw["registration_number"].astype(str)
-        en = oa_enrich.reindex(regs)
-
-        def _col(name: str):
-            return en[name] if name in en.columns else pd.Series([np.nan] * len(en), index=en.index)
-
-        df_raw["openalex_url"] = _col("openalex_url").fillna("")
-        df_raw["orcid_url"] = _col("orcid_url").fillna("")
-        df_raw["h_index"] = _col("h_index")
-        df_raw["i10_index"] = _col("i10_index")
-        df_raw["works_count"] = _col("works_count")
-        df_raw["cited_by_count"] = _col("cited_by_count")
+        regs_norm = df_raw["registration_number"].apply(_norm_regnum)
+        en = oa_enrich.reindex(regs_norm).reset_index(drop=True)  # <-- КЛЮЧЕВО
+
+        def _get(col: str, default):
+            if col in en.columns:
+                return en[col]
+            return pd.Series([default] * len(df_raw))
+
+        df_raw["openalex_url"] = _get("openalex_url", "").fillna("")
+        df_raw["orcid_url"] = _get("orcid_url", "").fillna("")
+
+        for src, dst in [
+            ("h_index", "h_index"),
+            ("i10_index", "i10_index"),
+            ("works_count", "works_count"),
+            ("cited_by_count", "cited_by_count"),
+        ]:
+            s = _get(src, np.nan)
+            df_raw[dst] = pd.to_numeric(s, errors="coerce").astype("Int64")
     else:
         df_raw["openalex_url"] = ""
         df_raw["orcid_url"] = ""
-        df_raw["h_index"] = np.nan
-        df_raw["i10_index"] = np.nan
-        df_raw["works_count"] = np.nan
-        df_raw["cited_by_count"] = np.nan
+        df_raw["h_index"] = pd.Series([pd.NA] * len(df_raw), dtype="Int64")
+        df_raw["i10_index"] = pd.Series([pd.NA] * len(df_raw), dtype="Int64")
+        df_raw["works_count"] = pd.Series([pd.NA] * len(df_raw), dtype="Int64")
+        df_raw["cited_by_count"] = pd.Series([pd.NA] * len(df_raw), dtype="Int64")
 
     # UI: скрываем №, registration_number, dissertation_type
     df_ui = df_raw.copy().reset_index(drop=True)
@@ -457,8 +484,6 @@ st.markdown(
 )
 
 # session_state — чтобы результаты не исчезали на rerun
-if "last_df_ui" not in st.session_state:
-    st.session_state.last_df_ui = None
 if "last_df_raw" not in st.session_state:
     st.session_state.last_df_raw = None
 if "last_excel" not in st.session_state:
@@ -533,11 +558,9 @@ if do_search:
                 year_range=year_range,
             )
 
-        st.session_state.last_df_ui = df_ui
         st.session_state.last_df_raw = df_raw
         st.session_state.last_excel = excel_bytes
 
-        # подготовим таблицу с чекбоксом
         if not df_ui.empty:
             df_table = df_ui[
                 ["Сходство", "ORCID", "ФИО", "Название диссертации", "Организация", "Год",
@@ -548,7 +571,7 @@ if do_search:
         else:
             st.session_state.table_df = None
 
-        st.session_state.search_id += 1  # чтобы сбрасывать state редактора на новый поиск
+        st.session_state.search_id += 1  # сброс state редактора на новый поиск
 
 # Отображение результатов
 df_table_saved = st.session_state.table_df
@@ -558,7 +581,6 @@ excel_saved = st.session_state.last_excel
 if isinstance(df_table_saved, pd.DataFrame) and not df_table_saved.empty:
     st.success(f"Найдено записей: {len(df_table_saved)}")
 
-    # data_editor: сортировка по клику на заголовок + чекбокс в строке
     editor_key = f"results_editor_{st.session_state.search_id}"
 
     edited = st.data_editor(
@@ -574,6 +596,7 @@ if isinstance(df_table_saved, pd.DataFrame) and not df_table_saved.empty:
                 "ORCID",
                 display_text=r"https://orcid\.org/([0-9X\-]+)",
                 width="small",
+                help="Профиль автора в ORCID (если найден)",
             ),
             "ФИО": st.column_config.TextColumn("ФИО", width="medium"),
             "Название диссертации": st.column_config.TextColumn("Название диссертации", width="large"),
@@ -584,6 +607,7 @@ if isinstance(df_table_saved, pd.DataFrame) and not df_table_saved.empty:
                 "OpenAlex",
                 display_text=r"https://openalex\.org/(A\d+)",
                 width="small",
+                help="Профиль автора в OpenAlex (если найден)",
             ),
             "h-index": st.column_config.NumberColumn("h-index", width="small"),
             "i10-index": st.column_config.NumberColumn("i10-index", width="small"),
@@ -595,6 +619,15 @@ if isinstance(df_table_saved, pd.DataFrame) and not df_table_saved.empty:
     # сохраняем чекбоксы
     st.session_state.table_df = edited
 
+    # Диагностика соответствия reg -> OA (можно убрать позже)
+    with st.expander("Диагностика OpenAlex", expanded=False):
+        regs_norm = df_raw_saved["registration_number"].apply(_norm_regnum)
+        matched = regs_norm.isin(oa_enrich.index).sum() if not oa_enrich.empty else 0
+        st.write(f"Совпало registration_number → OA: {matched} из {len(df_raw_saved)}")
+        if matched == 0:
+            st.write("Пример registration_number из выдачи:", regs_norm.head(10).tolist())
+            st.write("Пример registration_number из enrichment:", list(oa_enrich.index[:10]) if not oa_enrich.empty else [])
+
     if excel_saved is not None:
         st.download_button(
             label="💾 Скачать результаты в Excel",