voila deployment

.github/workflows/update-hf.yml

@@ -0,0 +1,20 @@
+name: Update Hugging Face repository
+on:
+  push:
+    branches: [main]
+
+  workflow_dispatch:
+
+jobs:
+  push-to-hf:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Push to Hugging Face
+        env:
+          HF_USER: ${{ secrets.HF_USER }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push https://$HF_USER:$HF_TOKEN@huggingface.co/spaces/ncdrisc/AI-Literature-Screening-for-Population-Health main --force

environment.yml

@@ -0,0 +1,90 @@
+name: voila
+channels:
+  - conda-forge
+dependencies:
+  - python=3.10
+  - pip
+  - pip:
+    - anyio==4.9.0
+    - argon2-cffi==23.1.0
+    - argon2-cffi-bindings==21.2.0
+    - arrow==1.3.0
+    - attrs==25.3.0
+    - babel==2.17.0
+    - beautifulsoup4==4.13.3
+    - biopython==1.85
+    - bleach==6.2.0
+    - certifi==2025.1.31
+    - cffi==1.17.1
+    - charset-normalizer==3.4.1
+    - comm==0.2.2
+    - debugpy==1.8.13
+    - decorator==5.2.1
+    - defusedxml==0.7.1
+    - fastjsonschema==2.21.1
+    - filelock==3.18.0
+    - fsspec==2025.3.2
+    - huggingface-hub==0.30.1
+    - idna==3.10
+    - ipykernel==6.29.5
+    - ipython==8.35.0
+    - ipywidgets==8.1.5
+    - jedi==0.19.2
+    - Jinja2==3.1.6
+    - jsonschema==4.23.0
+    - jupyter-events==0.12.0
+    - jupyter_client==8.6.3
+    - jupyter_core==5.7.2
+    - jupyter_server==2.15.0
+    - jupyter_server_terminals==0.5.3
+    - jupyterlab_pygments==0.3.0
+    - jupyterlab_server==2.27.3
+    - jupyterlab_widgets==3.0.13
+    - MarkupSafe==3.0.2
+    - mistune==3.1.3
+    - nbclient==0.10.2
+    - nbconvert==7.16.6
+    - nbformat==5.10.4
+    - nest-asyncio==1.6.0
+    - networkx==3.4.2
+    - numpy==1.26.4
+    - packaging==24.2
+    - pandas==2.2.3
+    - platformdirs==4.3.7
+    - prometheus_client==0.21.1
+    - prompt_toolkit==3.0.50
+    - psutil==7.0.0
+    - pure_eval==0.2.3
+    - pycparser==2.22
+    - Pygments==2.19.1
+    - python-dateutil==2.9.0.post0
+    - python-json-logger==3.3.0
+    - pytz==2025.2
+    - PyYAML==6.0.2
+    - pyzmq==26.4.0
+    - referencing==0.36.2
+    - regex==2024.11.6
+    - requests==2.32.3
+    - rpds-py==0.24.0
+    - safetensors==0.5.3
+    - six==1.17.0
+    - sniffio==1.3.1
+    - soupsieve==2.6
+    - sympy==1.13.3
+    - terminado==0.18.1
+    - tinycss2==1.4.0
+    - tokenizers==0.15.2
+    - torch==2.2.2
+    - tornado==6.4.2
+    - tqdm==4.67.1
+    - traitlets==5.14.3
+    - transformers==4.38.2
+    - typing_extensions==4.13.1
+    - tzdata==2025.2
+    - urllib3==2.3.0
+    - voila==0.5.8
+    - wcwidth==0.2.13
+    - webencodings==0.5.1
+    - websocket-client==1.8.0
+    - websockets==15.0.1
+    - widgetsnbextension==4.0.13

notebooks/notebook.ipynb

@@ -0,0 +1,47 @@
+{
+ "cells": [
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-04-10T15:52:17.264471Z",
+     "start_time": "2025-04-10T15:52:17.156072Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "import subprocess\nfrom datetime import date\nfrom dateutil.relativedelta import relativedelta\nimport os\nimport ipywidgets as widgets\nfrom IPython.display import HTML, clear_output\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# --- UI Elements ---\n\n# Dropdown for risk factor selection\nrisk_factor = widgets.Dropdown(\n    options=['Anthro', 'BP', 'Lipids', 'Diabetes'],\n    value='Anthro',\n    description='',\n    disabled=False\n)\n\nquery_method = widgets.RadioButtons(\n    options=[\"default\", \"custom\"],\n    value=\"default\",\n    layout=widgets.Layout(width=\"180px\")\n)\n\n# Custom query input field\ncustom_query_file = widgets.Text(\n    placeholder=\"Enter query filename\",\n    description=\"\",\n    disabled=True,\n    layout=widgets.Layout(width=\"300px\")\n)\n\n# Enable text box only when 'custom' is selected\ndef toggle_custom_query(change):\n    custom_query_file.disabled = (change.new != \"custom\")\n\nquery_method.observe(toggle_custom_query, names=\"value\")\n\nstart_date_value = date.today() - relativedelta(months=1)\nend_date_value = date.today()\n\nstart_date = widgets.DatePicker(\n    description=\"Start date:   \",\n    value=start_date_value\n)\n\nend_date = widgets.DatePicker(\n    description=\"End Date:   \",\n    value=end_date_value\n)\n\nrecall_target = widgets.Dropdown(\n    options=[('95%', 95), ('90%', 90), ('80%', 80), ('70%', 70), ('60%', 60)],\n    value=95,\n    description='',\n    layout=widgets.Layout(width=\"120px\")\n)\n\n# Buttons\n\n# Button style\ndisplay(HTML(\"\"\"\n<style>\n.widget-button {\n    justify-content: flex-start !important;\n    text-align: left !important;\n    font-weight: bold !important;\n    font-size: 15px !important;\n    padding-left: 12px !important;\n}\n</style>\n\"\"\"))\n\ndownload_btn = widgets.Button(\n    description=\"Download abstracts from PubMed\",\n    layout=widgets.Layout(width=\"300px\"),\n    style=widgets.ButtonStyle(font_weight=\"bold\")\n)\n\nload_model_btn = widgets.Button(\n    description=\"Load the model\",\n    layout=widgets.Layout(width=\"300px\"),\n    style=widgets.ButtonStyle(font_weight=\"bold\")\n)\n\npredict_btn = widgets.Button(\n    description=\"Run the model and screen articles\",\n    layout=widgets.Layout(width=\"300px\"),\n    style=widgets.ButtonStyle(font_weight=\"bold\")\n)\n\n# Output areas\ndownload_output = widgets.Output()\nload_model_output = widgets.Output()\npredict_output = widgets.Output()\n\n# --- Helper function for UI feedback ---\ndef mark_done(output_placeholder, header_widget):\n    output_placeholder.clear_output()\n    with output_placeholder:\n        display(header_widget)\n        display(widgets.HTML(\n            \"<span style='color: green; font-weight: bold; font-size:15px; margin-left:12px;'>&#10004; Done</span>\"\n        ))\n\n# --- Event Handlers ---\n\ndef toggle_query_visibility(change):\n    \"\"\"Enable/Disable custom query input based on checkbox state.\"\"\"\n    custom_query_file.disabled = not change.new\n\nquery_method.observe(toggle_custom_query, names=\"value\")\n\ndef run_download_citations(b):\n    b.close()\n    with download_output:\n        clear_output(wait=True)\n        header = widgets.HTML(\n            \"<div style='font-size:15px; font-weight:bold; margin-left:12px;'>\"\n            \"Download abstracts from PubMed\"\n            \"</div>\"\n        )\n        display(header)\n        try:\n            print(f\" Downloading articles published between {start_date.value} and {end_date.value}...\")\n            with open(os.path.abspath(os.path.join(os.getcwd(), \"src/download_citations.py\"))) as f:\n                exec(f.read(), globals())\n            mark_done(download_output, header)\n            path_to_articles = os.path.join(globals().get('directory'), 'downloaded_articles.csv')\n            display(widgets.HTML(\n                f\"<p style='font-size:12px; color:gray; margin-top:6px; margin-left:12px; margin-bottom:0px;'>\"\n                f\"Article abstracts downloaded to: {path_to_articles}</p>\"\n            ))\n        except Exception:\n            import traceback\n            traceback.print_exc()\n\ndef run_load_model(b):\n    b.close()\n    with load_model_output:\n        clear_output(wait=True)\n        header = widgets.HTML(\n            \"<div style='font-size:15px; font-weight:bold; margin-left:12px;'>\"\n            \"Load the model\"\n            \"</div>\"\n        )\n        display(header)\n        try:\n            print(f\" Loading the model...\")\n            with open(os.path.abspath(os.path.join(os.getcwd(), \"src/load_model.py\"))) as f:\n                exec(f.read(), globals())\n            mark_done(load_model_output, header)\n        except Exception:\n            import traceback\n            traceback.print_exc()\n\ndef open_ris_file(b):\n    path = os.path.join(globals().get('directory'), 'articles_to_review.ris')\n    \"\"\"Opens the .ris file using the system's default application correctly.\"\"\"\n    try:\n        if os.name == \"posix\":  # macOS / Linux\n            subprocess.Popen([\"open\", path])\n        elif os.name == \"nt\":  # Windows\n            subprocess.Popen([\"start\", \"\", path], shell=True)\n    except Exception:\n        import traceback\n        traceback.print_exc()\n\ndef run_prediction(b):\n    b.close()\n    with predict_output:\n        clear_output(wait=True)\n        header = widgets.HTML(\n            \"<div style='font-size:15px; font-weight:bold; margin-left:12px;'>\"\n            \"Run the model and screen articles\"\n            \"</div>\"\n        )\n        display(header)\n        try:\n            print(\" Running the model...\")\n            with open(os.path.abspath(os.path.join(os.getcwd(), \"src/predict.py\"))) as f:\n                exec(f.read(), globals())\n            mark_done(predict_output, header)\n        except Exception:\n            import traceback\n            traceback.print_exc()\n\n    path_to_ris = os.path.join(globals().get('directory'), 'articles_to_review.ris')\n\n    open_file_btn = widgets.Button(description=f\"📄 Open in EndNote\", layout=widgets.Layout(width=\"350px\"))\n    open_file_btn.on_click(open_ris_file)\n\n    path_display = widgets.HTML(\n        f\"<p style='font-size:12px; color:gray; margin-top:6px; margin-left:12px;'>\"\n        f\"Path to the EndNote file: {path_to_ris}</p>\"\n    )\n    final_message_1 = widgets.HTML(\n        \"<p style='font-size:14px; color:black; margin-top:10px; margin-left:12px;'>\"\n        \"Open the .ris file in EndNote by clicking on the button above or navigating to the file</p>\"\n    )\n    final_message_2 = widgets.HTML(\n        \"<p style='font-size:14px; color:black; margin-top:10px; margin-left:12px;'>\"\n        \"Select RefMan - RIS as the input file format</p>\"\n    )\n\n    with predict_output:\n        display(widgets.VBox([\n            open_file_btn,\n            path_display,\n            final_message_1,\n            final_message_2\n        ]))\n\n\n# Attach event handlers to buttons\ndownload_btn.on_click(run_download_citations)\nload_model_btn.on_click(run_load_model)\npredict_btn.on_click(run_prediction)\n\n# --- Layout ---\n\ntitle_style = \"font-size: 22px; font-weight: bold; margin-bottom: 40px;\"\nsection_style = \"font-size: 18px; font-weight: bold; margin-bottom: 15px;\"\ntext_style = \"font-size: 16px;\"\nspacing = widgets.Layout(margin=\"20px 0px\")\n\n# Titles\nheader = widgets.HTML(\n    f\"<h2 style='margin-left:12px; {title_style}'>Automated screening of the literature</h2>\"\n)\n\nsection1 = widgets.VBox([\n\n    widgets.HTML(\"<b style='font-size:16px;'>Choose a risk factor</b>\"),\n    risk_factor,\n    widgets.HTML(\"<div style='height:25px;'></div>\"),\n\n    widgets.HTML(\"<b style='font-size:16px;'>Define the PubMed search</b>\"),\n    widgets.HBox([\n        widgets.VBox([\n            start_date,\n            end_date\n        ], layout=widgets.Layout(margin=\"0 20px 0 0\")),\n\n        widgets.VBox([\n            widgets.HBox([\n                widgets.Label(\"Query:   \", layout=widgets.Layout(width=\"60px\")),\n                query_method\n            ]),\n            custom_query_file\n        ], layout=widgets.Layout(margin=\"0 0 0 20px\"))\n    ], layout=widgets.Layout(justify_content=\"flex-start\", gap=\"20px\", margin=\"10px 0\"))\n], layout=widgets.Layout(margin=\"0 0 0 12px\"))\n\nsection2 = widgets.VBox([\n    # Line 3 — vertically stacked buttons with equal width\n    widgets.VBox([\n        download_btn,\n        download_output,\n        widgets.HTML(\"<div style='height:25px;'></div>\"),\n\n        load_model_btn,\n        load_model_output,\n        widgets.HTML(\"<div style='height:25px;'></div>\"),\n\n        widgets.HTML(\"<b style='font-size:16px;'>Define how inclusive the model should be</b>\"),\n        widgets.HTML(\"<span style='font-size:13px; color:gray; margin-top:2px; display:block;'>Based on the recall achieved in previous testing; the higher the recall, the more inclusive the model</span>\"),\n        widgets.HTML(\"<div style='height:8px;'></div>\"),\n        recall_target,\n        widgets.HTML(\"<div style='height:15px;'></div>\"),\n\n        predict_btn,\n        predict_output\n    ], layout=widgets.Layout(margin=\"30px 0\"))\n])\n\n\ndisplay(header, section1, section2)",
+   "id": "35bcc7331ef5d1dc",
+   "outputs": [],
+   "execution_count": null
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "eeebfa287ae99109"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.9"
+  },
+  "trusted": true
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/src/download_citations.py

@@ -0,0 +1,79 @@
+import os
+from datetime import date
+from Bio import Entrez
+import pandas as pd
+
+risk_factor = globals().get('risk_factor').value
+
+risk_factor_directory = os.path.abspath(os.path.join(os.getcwd(), risk_factor))
+
+queries_directory = os.path.abspath(os.path.join(os.getcwd(), 'queries', risk_factor))
+use_default_query = globals().get('query_method').value == "default"
+if use_default_query:
+    with open(os.path.join(queries_directory, 'default.txt'), 'r') as file:
+        base_query = file.read()
+    query_suffix = ""
+else:
+    custom_query_file = globals().get('custom_query_file').value
+    with open(os.path.join(queries_directory, f'{custom_query_file}.txt'), 'r') as file:
+        base_query = file.read()
+    query_suffix = f"_{custom_query_file}"
+
+base_folder_name = date.today().strftime("%Y-%m-%d") + query_suffix
+if not os.path.isdir(os.path.join(risk_factor_directory, base_folder_name)):
+    directory = os.path.join(risk_factor_directory, base_folder_name)
+    os.makedirs(directory, exist_ok=True)
+else:
+    version = 2
+    folder_name = f"{base_folder_name}-v{version}"
+    directory = os.path.join(risk_factor_directory, folder_name)
+    while os.path.isdir(directory):
+        version += 1
+        folder_name = f"{base_folder_name}-v{version}"
+        directory = os.path.join(risk_factor_directory, folder_name)
+    os.makedirs(directory, exist_ok=True)
+
+start_date = globals().get('start_date').value
+end_date = globals().get('end_date').value
+query = base_query + f'AND (("{start_date}"[Date - Publication] : "{end_date}"[Date - Publication]))'
+
+Entrez.email = os.getenv('email')
+search_handle = Entrez.esearch(db="pubmed", term=query, retmax=10000)
+search_results = Entrez.read(search_handle)
+search_handle.close()
+id_list = search_results['IdList']
+fetch_handle = Entrez.efetch(db="pubmed", id=id_list, rettype="xml")
+fetch_results = Entrez.read(fetch_handle)
+fetch_handle.close()
+
+papers = []
+for article in fetch_results['PubmedArticle']:
+    medline = article['MedlineCitation']
+    article_data = medline['Article']
+
+    title = str(article_data.get('ArticleTitle', ''))
+    abstract_text = ' '.join(article_data.get('Abstract', {}).get('AbstractText', ['']))
+    abstract_text = (title + ' ' + abstract_text).strip()
+    authors = ', '.join(['{} {}'.format(a.get('ForeName', ''), a.get('LastName', ''))
+                         for a in article_data.get('AuthorList', []) if 'LastName' in a])
+    journal = article_data.get('Journal', {}).get('Title', '')
+    year = article_data.get('Journal', {}).get('JournalIssue', {}).get('PubDate', {}).get('Year', '')
+    pmid = medline.get('PMID', '')
+
+    doi = ''
+    for eid in article_data.get('ELocationID', []):
+        if eid.attributes.get('EIdType') == 'doi':
+            doi = str(eid)
+
+    papers.append({
+        'PMID': pmid,
+        'Title': title,
+        'Abstract': abstract_text,
+        'Authors': authors,
+        'Journal': journal,
+        'Year': year,
+        'DOI': doi
+    })
+
+df = pd.DataFrame(papers)
+df.to_csv(os.path.join(directory, 'downloaded_articles.csv'), index=False)

notebooks/src/handlers/hyperparameters_handler.py

@@ -0,0 +1,29 @@
+class Hyperparameters:
+
+    def __init__(self,
+                 ds,
+                 input_col,
+                 output_col,
+                 test_size,
+                 seed,
+                 pre_trained_model,
+                 max_length=512,
+                 freezed_layers=0,
+                 batch_size=8,
+                 learning_rate=0.00003,
+                 max_epochs=None,
+                 stop_loss_epochs=5
+                 ):
+        self.ds = ds
+        self.input_col = input_col
+        self.output_col = output_col
+        self.test_size = test_size
+        self.seed = seed
+        self.pre_trained_model = pre_trained_model
+        self.max_length = max_length
+        self.freezed_layers = freezed_layers
+        self.batch_size = batch_size
+        self.learning_rate = learning_rate
+        self.max_epochs = max_epochs
+        if self.max_epochs is None:
+            self.stop_loss_epochs = stop_loss_epochs

notebooks/src/handlers/model_IO_handler.py

@@ -0,0 +1,32 @@
+import os
+import numpy as np
+import pandas as pd
+from src.utils import file_handling
+
+
+class IOHandler:
+
+    def __init__(self, directory, model_dir_path):
+        self.directory = directory
+        self.model_path = os.path.join(model_dir_path, 'model.pth')
+        prc_path = os.path.join(model_dir_path, 'precision_recall_curve.csv')
+        self._prc = pd.read_csv(prc_path)
+
+    def get_threshold(self, recall_target):
+        """Return the highest threshold that still achieves at least recall_target recall."""
+        above = self._prc[self._prc['Recall'] >= recall_target]
+        if above.empty:
+            return float(self._prc['Threshold'].iloc[-1])
+        return float(above['Threshold'].max())
+
+    def write_predictions(self, data, y_prob, threshold):
+        data = data.copy()
+        data['y_prob'] = y_prob
+        data['y_pred'] = (np.array(y_prob) >= threshold).astype(int)
+        file_handling.write_file(data, self.directory, 'articles_with_predictions.csv')
+
+    def write_review_file(self, data, y_prob, threshold):
+        data = data.loc[np.array(y_prob) >= threshold].copy()
+        data = data.drop(columns=['y_prob', 'y_pred'], errors='ignore')
+        file_handling.write_file(data, self.directory, 'articles_to_review.csv')
+        file_handling.write_file(data, self.directory, 'articles_to_review.ris')

notebooks/src/handlers/model_code_translator.py

@@ -0,0 +1,113 @@
+from enum import Enum
+from itertools import product
+
+
+MODELS = {
+    'BiomedBERT_abstract': '1A2B06A00'
+}
+
+
+class Dataset(Enum):
+    _1 = 'ds1'
+    _2 = 'ds2'
+    _3 = 'ds3'
+
+
+class PretrainedModel(Enum):
+    O = 'BERT'
+    A = 'bioBERT'
+    B = 'BiomedBERT'
+    L = 'Longformer'
+    M = 'BigBird'
+
+
+class InputColumn(Enum):
+    A = 'Abstract'
+    T = 'Text'
+    M = 'Methods'
+    N = 'A+Methods'
+
+
+class OutputColumn(Enum):
+    _1 = 'CLASS--stage_1'
+    _2 = 'CLASS'
+
+
+class LearningRate(Enum):
+    A = 0.00002
+    B = 0.00003
+    C = 0.00004
+
+
+TEST_SIZE = 0.1
+SEED = 100
+
+
+class ModelBatchSize(Enum):
+    O = 8
+    A = 8
+    B = 8
+    L = 2
+    M = 2
+
+
+class ModelMaxLength(Enum):
+    O = 512
+    A = 512
+    B = 512
+    L = 4096
+    M = 4096
+
+
+POSSIBLE_CODE_ELEMENTS = {
+    0: [e.name[-1] for e in Dataset],
+    1: [e.name for e in InputColumn],
+    2: [e.name[-1] for e in OutputColumn],
+    3: [e.name for e in PretrainedModel],
+    4: [i for i in range(10)],
+    5: [i for i in range(10)],
+    6: [e.name for e in LearningRate],
+    7: [i for i in range(10)],
+    8: [i for i in range(10)],
+}
+
+
+def get_model_specs(code):
+    """
+    Generate model specifications from the provided code.
+    The code must not contain 'x'.
+    """
+
+    model_specs = {
+        'ds': Dataset[f"_{code[0]}"].value,
+        'test_size': TEST_SIZE,
+        'seed': SEED,
+        'input_col': InputColumn[code[1]].value,
+        'output_col': OutputColumn[f"_{code[2]}"].value,
+        'pre_trained_model': PretrainedModel[code[3]].value,
+        'max_length': ModelMaxLength[code[3]].value,
+        'freezed_layers': int(code[4:6]),
+        'learning_rate': LearningRate[code[6]].value,
+        'batch_size': ModelBatchSize[code[3]].value,
+    }
+    if code[7:] != "00":
+        model_specs["max_epochs"] = int(code[7:])
+
+    return model_specs
+
+
+class ModelCodeTranslator:
+    def __init__(self, code):
+        if len(code) != 9:
+            raise Exception("Code must be of length 9")
+        self.code = code
+
+        if not 'x' in self.code:
+            self.model_specs = get_model_specs(self.code)
+            self.model_specs_list = None
+        else:
+            iterables = [POSSIBLE_CODE_ELEMENTS[i] if char == 'x' else [char] for i, char in enumerate(self.code)]
+            codes = [''.join(combination) for combination in product(*iterables)]
+            self.model_specs = None
+            self.model_specs_list = [get_model_specs(code) for code in codes]
+

notebooks/src/handlers/model_loader.py

@@ -0,0 +1,35 @@
+from enum import Enum
+
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from src.handlers.hyperparameters_handler import Hyperparameters
+from src.nn import pytorch_models
+
+
+class Library(Enum):
+    PYTORCH = 'pytorch'
+
+
+class ModelUrl(Enum):
+    BERT = 'bert-base-uncased'
+    bioBERT = 'dmis-lab/biobert-v1.1'
+    BiomedBERT = 'microsoft/BiomedNLP-BiomedBERT-base-uncased-abstract-fulltext'
+    Longformer = 'allenai/longformer-base-4096'
+
+
+class ModelLoader:
+    def __init__(self, logger, model_path, specs: dict, library: Library, ):
+        self.logger = logger
+
+        self.target_library = library
+        self.hyperparameters = Hyperparameters(**specs)
+
+        self.model_url = ModelUrl[self.hyperparameters.pre_trained_model].value
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_url, use_fast=False)
+        if self.target_library == Library.PYTORCH:
+            self.model = AutoModelForSequenceClassification.from_pretrained(self.model_url, num_labels=1,
+                                                                   problem_type='multi_label_classification')
+            self.model_wrapper = pytorch_models.NLPClassifier(
+                self.logger, model_path, self.tokenizer, self.model, self.hyperparameters
+            )
+        else:
+            raise NotImplementedError

notebooks/src/load_model.py

@@ -0,0 +1,25 @@
+import os
+
+from src.utils import logging
+from src.handlers.model_code_translator import ModelCodeTranslator, MODELS
+from src.handlers.model_loader import ModelLoader, Library
+from src.handlers.model_IO_handler import IOHandler
+
+risk_factor = globals().get("risk_factor")
+directory = globals().get("directory")
+model_name = 'BiomedBERT_abstract'
+
+log_dir = os.path.join(directory, 'logs')
+logger = logging.Logger(log_dir=log_dir)
+
+logger.info(f"Creating the model")
+model_dir = os.path.abspath(os.path.join(os.getcwd(), 'models', f'PopulationHealthScreener ({risk_factor})'))
+model_io_handler = IOHandler(directory, model_dir)
+model_specs = ModelCodeTranslator(MODELS[model_name]).model_specs
+library = Library.PYTORCH
+model_path = os.path.join(model_dir, 'model.pth')
+model = ModelLoader(logger, model_path, model_specs, library).model_wrapper
+input_col = model_specs["input_col"]
+
+logger.info(f"Loading model weights from {model_path}")
+model.load_fine_tuned_weights()

notebooks/src/nn/pytorch_models.py

@@ -0,0 +1,108 @@
+import sys
+
+import numpy as np
+import torch
+from torch.utils.data import DataLoader, TensorDataset, WeightedRandomSampler
+
+
+class NLPClassifier:
+    def __init__(self, logger, model_path, tokenizer, model, hyperparameters):
+        self.logger = logger
+        self.model_path = model_path
+        self.hyperparameters = hyperparameters
+
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+        self.tokenizer = tokenizer
+        self.model = model
+
+        self.model_arch = self.get_model_arch()
+        self.freeze_layers()
+        self.model.to(self.device)
+
+    def get_model_arch(self):
+        if 'BERT' in self.hyperparameters.pre_trained_model:
+            return self.model.bert
+        elif self.hyperparameters.pre_trained_model == "Longformer":
+            return self.model.longformer
+        else:
+            raise ValueError("Invalid model type")
+
+    def load_training_data(self, data, labels):
+        labels = torch.tensor(np.array(labels).reshape(-1, 1))
+        classes, class_counts = torch.unique(labels, sorted=True, return_counts=True)
+        class_weights = 1.0 / torch.tensor(class_counts, dtype=torch.float)
+        weights_dict = {cls.item(): weight for cls, weight in zip(classes, class_weights)}
+        sample_weights = torch.tensor([weights_dict[t.item()] for t in labels])
+        sampler = WeightedRandomSampler(sample_weights, len(sample_weights))
+
+        encodings = self.tokenizer(data, truncation=True, padding='max_length',
+                                   max_length=self.hyperparameters.max_length, return_tensors='pt')
+        dataset = TensorDataset(
+            encodings['input_ids'], encodings['attention_mask'], labels.to(torch.float32)
+        )
+        return DataLoader(dataset, batch_size=self.hyperparameters.batch_size, sampler=sampler)
+
+    def freeze_layers(self):
+        for param in self.model_arch.embeddings.parameters():
+            param.requires_grad = False
+
+        for layer in self.model_arch.encoder.layer[:self.hyperparameters.freezed_layers]:
+            for param in layer.parameters():
+                param.requires_grad = False
+
+    def fit(self, x_train, y_train, trained_model_path):
+        train_data = self.load_training_data(x_train, y_train)
+        optimizer = torch.optim.AdamW(self.model.parameters(), lr=self.hyperparameters.learning_rate)
+
+        for epoch in range(3):
+            self.logger.info(f'Epoch {epoch + 1}/3')
+            self.model.train()
+            for i, batch in enumerate(train_data):
+                progress = f"Batch {i+1}/{len(train_data)}"
+                sys.stdout.write('\r' + progress)
+                sys.stdout.flush()
+                optimizer.zero_grad()
+                input_ids = batch[0].to(self.device)
+                attention_mask = batch[1].to(self.device)
+                labels = batch[2].to(self.device)
+                outputs = self.model(input_ids, attention_mask=attention_mask, labels=labels)
+                loss = outputs.loss
+                loss.backward()
+                optimizer.step()
+
+        self.logger.info("Saving the model at {}".format(trained_model_path))
+        torch.save(self.model.state_dict(), trained_model_path)
+
+    def load_fine_tuned_weights(self):
+        state_dict = torch.load(self.model_path, map_location=self.device)
+        self.model.load_state_dict(state_dict, strict=False)
+
+    def load_test_data(self, data):
+        encodings = self.tokenizer(data, truncation=True, padding='max_length',
+                                   max_length=self.hyperparameters.max_length, return_tensors='pt')
+        dataset = TensorDataset(
+            encodings['input_ids'], encodings['attention_mask']
+        )
+        return DataLoader(dataset, batch_size=self.hyperparameters.batch_size, shuffle=False)
+
+    def predict(self, data):
+        test_data = self.load_test_data(data)
+
+        predictions = []
+        self.model.eval()
+
+        for i, batch in enumerate(test_data):
+            msg = f" Processing article {8 * (i+1)}/{8 *len(test_data)} ⏳"
+            sys.stdout.write('\r' + msg)
+            sys.stdout.flush()
+
+            input_ids = batch[0].to(self.device)
+            attention_mask = batch[1].to(self.device)
+
+            with torch.no_grad():
+                outputs = self.model(input_ids, attention_mask=attention_mask,
+                                     output_hidden_states=True)
+                predictions.extend(torch.sigmoid(outputs.logits).flatten().tolist())
+
+        return predictions

notebooks/src/predict.py

@@ -0,0 +1,33 @@
+import os
+from src.utils import file_handling
+
+risk_factor = globals().get('risk_factor')
+directory = globals().get('directory')
+logger = globals().get("logger")
+model_io_handler = globals().get("model_io_handler")
+model = globals().get('model')
+input_col = globals().get('input_col')
+
+logger.info(f"Reading new titles and abstracts to screen")
+data = file_handling.read_file(directory, file="downloaded_articles.csv").dropna(subset=[input_col])
+x_test = data[input_col].astype(str).to_list()
+
+logger.info("Truncating abstracts to max token length before prediction")
+x_test = [
+    model.tokenizer.decode(
+        model.tokenizer.encode(text, max_length=model.hyperparameters.max_length, truncation=True),
+        skip_special_tokens=True
+    )
+    for text in x_test
+]
+
+recall_target = globals().get('recall_target').value / 100.0
+threshold = model_io_handler.get_threshold(recall_target)
+logger.info(f"Using threshold {threshold:.4f} for {int(recall_target * 100)}% recall target")
+
+logger.info("Running the model and getting predictions")
+y_prob = model.predict(x_test)
+model_io_handler.write_predictions(data, y_prob, threshold)
+logger.info("Writing predictions")
+model_io_handler.write_review_file(data, y_prob, threshold)
+logger.info(f"Writing the review file -- find it at {os.path.join(directory, 'articles_to_review.ris')}")

notebooks/src/utils/file_handling.py

@@ -0,0 +1,63 @@
+import os
+import pandas as pd
+import xml.etree.ElementTree as ET
+
+
+def read_file(dir: str, file: str):
+    os.makedirs(dir, exist_ok=True)
+    extension = file.split('.')[1]
+    func_name = f'read_{extension}'
+    func = globals()[func_name]
+    return func(dir=dir, file=file)
+
+
+def write_file(data_element, dir: str, file_name: str):
+    os.makedirs(dir, exist_ok=True)
+    extension = file_name.split('.')[1]
+    func_name = f'write_{extension}'
+    func = globals()[func_name]
+    return func(data_element, dir=dir, file=file_name)
+
+
+def read_csv(dir: str, file: str) -> pd.DataFrame:
+    return pd.read_csv(os.path.join(dir, file), escapechar='\\', index_col=False)
+
+
+def read_xml(dir: str, file: str) -> pd.DataFrame:
+    tree = ET.parse(os.path.join(dir, file))
+    root = tree.getroot()
+    records = []
+    for rec in root.findall('.//record'):
+        title = rec.findtext('.//titles/title') or ""
+        abstract = rec.findtext('.//abstract') or ""
+        pmid = rec.findtext('.//electronic-resource-num')
+        label = rec.findtext('.//label/style')
+        records.append({
+            'Title': title,
+            'Abstract': abstract,
+            'PMID': pmid,
+            'Label': label
+        })
+    return pd.DataFrame(records)
+
+
+def write_csv(df: pd.DataFrame, dir: str, file: str):
+    df.to_csv(os.path.join(dir, file), escapechar='\\', index=False)
+
+
+def write_xlsx(df: pd.DataFrame, dir: str, file: str):
+    df.to_excel(os.path.join(dir, file), engine="openpyxl", index=False)
+
+
+def write_ris(df: pd.DataFrame, dir: str, file: str):
+    with open(os.path.join(dir, file), 'w', encoding='utf-8') as f:
+        for _, row in df.iterrows():
+            f.write("TI  - " + str(row.get('Title') or '') + "\n")
+            f.write("AB  - " + str(row.get('Abstract') or '') + "\n")
+            for author in str(row.get('Authors') or '').split(','):
+                f.write("AU  - " + author.strip() + "\n")
+            f.write("DO  - " + str(row.get('DOI') or '') + "\n")
+            f.write("PM  - " + str(row.get('PMID') or '') + "\n")
+            f.write("PY  - " + str(row.get('Year') or '') + "\n")
+            f.write("JO  - " + str(row.get('Journal') or '') + "\n")
+            f.write("ER  - \n\n")

notebooks/src/utils/logging.py

@@ -0,0 +1,48 @@
+import datetime
+import logging
+import os
+
+
+class LogLevel:
+    INFO = logging.INFO
+    DEBUG = logging.DEBUG
+    WARN = logging.WARN
+    ERROR = logging.ERROR
+
+
+class Logger:
+    def __init__(self, log_dir, log_name='log', log_level=LogLevel.INFO):
+        self.log_dir = log_dir
+        self.log_filename = self.__generate_log_filename(log_name)
+
+        if not os.path.exists(self.log_dir):
+            os.makedirs(self.log_dir, exist_ok=True)
+
+        try:
+            self.__configure_logging(_level=log_level)
+        except FileNotFoundError:
+            os.makedirs(self.log_dir, exist_ok=True)
+            self.__configure_logging(_level=log_level)
+
+        logging.getLogger("transformers.modeling_utils").setLevel(logging.ERROR)
+
+    @staticmethod
+    def __generate_log_filename(log_name):
+        current_datetime = datetime.datetime.now()
+        formatted_date_time = current_datetime.strftime('%d_%m_%Y_%H_%M_%S')
+        return f'{log_name}_{formatted_date_time}.log'
+
+    def __configure_logging(self, _level=LogLevel.INFO):
+        logging.basicConfig(level=_level, format="%(asctime)s [%(levelname)s] %(message)s",
+                            handlers=[logging.FileHandler(os.path.join(self.log_dir, self.log_filename)),
+                                      logging.StreamHandler()])
+
+    def info(self, msg):
+        # logging.info(msg)
+        pass  # TODO change back
+
+    def warn(self, msg):
+        logging.warning(msg)
+
+    def error(self, msg):
+        logging.error(msg)