Upload folder using huggingface_hub

.gitattributes

@@ -5333,3 +5333,4 @@ images/plates/Exp23DM09_inoc2_T6_P059.JPG filter=lfs diff=lfs merge=lfs -text
 images/plates/Exp23DM09_inoc2_T6_P060.JPG filter=lfs diff=lfs merge=lfs -text
 images/plates/Exp23DM09_inoc2_T6_P061.JPG filter=lfs diff=lfs merge=lfs -text
 images/plates/Exp23DM09_inoc2_T6_P062.JPG filter=lfs diff=lfs merge=lfs -text
+src/leaf_patch_extractor.ipynb filter=lfs diff=lfs merge=lfs -text

src/.gitattributes

@@ -0,0 +1,5 @@
+leaf_patch_annotation.ipynb filter=lfs diff=lfs merge=lfs -text
+leaf_patch_extractor.ipynb filter=lfs diff=lfs merge=lfs -text
+leaf_patch_gen_diff.ipynb filter=lfs diff=lfs merge=lfs -text
+leaf_patch_oiv_predictor.ipynb filter=lfs diff=lfs merge=lfs -text
+repo_manager.ipynb filter=lfs diff=lfs merge=lfs -text

src/com_const.py

@@ -4,7 +4,7 @@ path_to_here = Path(__file__).resolve().parent
 path_to_root = path_to_here.parent
 
 path_to_data = path_to_root.joinpath("data")
-
+path_to_resources = path_to_root.joinpath("resources")
 path_to_images = path_to_root.joinpath("images")
 path_to_plates = path_to_images.joinpath("plates")
 path_to_leaf_discs = path_to_images.joinpath("leaf_discs")

src/leaf_patch_annotation.ipynb

@@ -4,7 +4,40 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## 202311 Dataset Annotation"
+    "# 202311 Dataset Annotation"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Please launch with command \n",
+    "\n",
+    "    panel serve leaf_patch_annotation.ipynb --show --dev\n",
+    "    \n",
+    "from the \"src\" folder"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Source Selection\n",
+    "Three options for SOURCE_FILE:\n",
+    "- oiv_annotation.csv for the already annotated CSV file\n",
+    "- oiv_annotation_empty.csv for an empty file ready to be annotated\n",
+    "- Your own semicolon separated CSV file containing at least a column named \"file_name\" with the name of patches located in the \"images/leaf_patches\" folder"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SOURCE_FILE = \"oiv_annotation.csv\"\n",
+    "# SOURCE_FILE = \"oiv_annotation_empty.csv\"\n",
+    "# SOURCE_FILE = \"my_csv.csv\""
    ]
   },
   {
@@ -158,11 +191,18 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df = cf.read_dataframe(path=cc.path_to_data.joinpath(\"oiv_annotation.csv\")).sort_values(\n",
-    "    [\"experiment\", \"inoc\", \"dpi\", \"plaque\", \"row\", \"col\"]\n",
-    ")\n",
+    "df = cf.read_dataframe(path=cc.path_to_data.joinpath(SOURCE_FILE))\n",
     "if \"seen_at\" not in df:\n",
     "    df = df >> mutate(seen_at=np.nan)\n",
+    "if \"oiv_annotated_at\" not in df:\n",
+    "    df = df >> mutate(oiv_annotated_at=np.nan)\n",
+    "if \"source_annotated_at\" not in df:\n",
+    "    df = df >> mutate(source_annotated_at=np.nan)\n",
+    "if \"source\" not in df:\n",
+    "    df = df >> mutate(source=np.nan)\n",
+    "if \"oiv\" not in df:\n",
+    "    df = df >> mutate(oiv=np.nan)\n",
+    "\n",
     "df.seen_at = pd.to_datetime(df.seen_at)\n",
     "df = df.set_index(\"file_name\")\n",
     "df"
@@ -183,7 +223,9 @@
    "source": [
     "def update_image(image_name:str, color, brightness, contrast, sharpness):\n",
     "    image_path = cc.path_to_leaf_patches.joinpath(image_name)\n",
-    "    if image_path.is_file() is False:\n",
+    "    if not image_name:\n",
+    "        fig = px.imshow(Image.open(cc.path_to_resources.joinpath(\"well_done.png\")))\n",
+    "    elif image_path.is_file() is False:\n",
     "        fig = px.imshow(\n",
     "            np.array(\n",
     "                [\n",
@@ -342,12 +384,6 @@
     "    sizing_mode=\"scale_width\",\n",
     ")\n",
     "\n",
-    "sw_ui_state = pn.widgets.Switch(name=\"active\", value=False)\n",
-    "alt_ui_state = pn.pane.Alert(\"Annotations will be stored\", alert_type=\"primary\")\n",
-    "\n",
-    "pn_ui_state = pn.Row(sw_ui_state, alt_ui_state)\n",
-    "\n",
-    "\n",
     "bt_next = pn.widgets.Button(name=\"Next\", button_type=\"primary\")\n",
     "bt_previous = pn.widgets.Button(name=\"Previous\", button_type=\"primary\")\n",
     "\n",
@@ -389,20 +425,6 @@
     "    )\n",
     "\n",
     "\n",
-    "def update_ui_state(ui_state: bool):\n",
-    "    if ui_state is True:\n",
-    "        alt_ui_state.object = \"Annotations will be stored\"\n",
-    "        alt_ui_state.alert_type = \"primary\"\n",
-    "    else:\n",
-    "        alt_ui_state.object = \"Annotations will be discarded\"\n",
-    "        alt_ui_state.alert_type = \"danger\"\n",
-    "\n",
-    "\n",
-    "@pn.depends(sw_ui_state, watch=True)\n",
-    "def on_ui_State_changed(new_state: bool):\n",
-    "    update_ui_state(new_state)\n",
-    "\n",
-    "\n",
     "def select_next(event):\n",
     "    global current_row\n",
     "    global df\n",
@@ -419,9 +441,7 @@
     "            ]\n",
     "        cf.write_dataframe(\n",
     "            df=df.reset_index(),\n",
-    "            path=cc.path_to_data.joinpath(\n",
-    "                \"oiv_annotation.csv\" if sw_ui_state.value is True else \"oiv_annotation_test.csv\"\n",
-    "            ),\n",
+    "            path=cc.path_to_data.joinpath(SOURCE_FILE),\n",
     "        )\n",
     "        df.at[current_row.file_name, \"seen_at\"] = now\n",
     "\n",
@@ -477,15 +497,9 @@
     "    rgb_source.disabled = target == \"OIV\"\n",
     "\n",
     "\n",
-    "# @pn.depends(rgb_oiv, watch=True)\n",
-    "# def on_oiv_changed(_):\n",
-    "#     select_next(None)\n",
-    "\n",
-    "\n",
     "bt_next.on_click(select_next)\n",
     "bt_previous.on_click(select_next)\n",
     "\n",
-    "update_ui_state(sw_ui_state.value)\n",
     "select_next(None)"
    ]
   },
@@ -502,14 +516,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "template.sidebar.append(pn_ui_state)\n",
     "template.sidebar.append(c_image_processing)\n",
     "template.sidebar.append(c_anno_options)\n",
     "\n",
     "template.main.append(\n",
     "    pn.Row(\n",
     "        pn.Column(\n",
-    "            # mkd_current,\n",
     "            img_current,\n",
     "            ui_annotation,\n",
     "        ),\n",
@@ -519,13 +531,6 @@
     "\n",
     "template.servable()"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Please launch with command \"panel serve leaf_patch_annotation.ipynb --show --dev\" from the \"src\" folder"
-   ]
   }
  ],
  "metadata": {
@@ -544,7 +549,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.2"
+   "version": "3.12.4"
   }
  },
  "nbformat": 4,

src/leaf_patch_extractor.ipynb

@@ -1,470 +1,3 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Extract Leaf Patches From Plates"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Imports"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%load_ext autoreload\n",
-    "%autoreload 2"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from datetime import datetime as dt\n",
-    "import warnings\n",
-    "import random\n",
-    "\n",
-    "from tqdm import tqdm\n",
-    "\n",
-    "import cv2\n",
-    "\n",
-    "import pandas as pd\n",
-    "\n",
-    "from siuba import _ as s\n",
-    "from siuba import filter as sfilter\n",
-    "from siuba import mutate, select, if_else\n",
-    "\n",
-    "import panel as pn\n",
-    "\n",
-    "import torch\n",
-    "\n",
-    "from pytorch_lightning.callbacks import (\n",
-    "    RichProgressBar,\n",
-    "    ModelCheckpoint,\n",
-    "    LearningRateMonitor,\n",
-    ")\n",
-    "from pytorch_lightning import Trainer\n",
-    "from pytorch_lightning.callbacks.early_stopping import EarlyStopping\n",
-    "from pytorch_lightning.loggers import TensorBoardLogger\n",
-    "\n",
-    "\n",
-    "import com_const as cc\n",
-    "import com_image as ci\n",
-    "import com_func as cf\n",
-    "import leaf_patch_extractor_model as lpem"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Setup"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "warnings.simplefilter(action=\"ignore\", category=UserWarning)\n",
-    "warnings.simplefilter(action=\"ignore\", category=FutureWarning)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pd.set_option(\"display.max_colwidth\", 500)\n",
-    "pd.set_option(\"display.max_columns\", 500)\n",
-    "pd.set_option(\"display.width\", 1000)\n",
-    "pd.set_option(\"display.max_rows\", 16)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pn.extension(notifications=True, console_output=\"disable\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Train Disc Detector"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Load Datasets"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "train, val, test = [\n",
-    "    cf.read_dataframe(cc.path_to_data.joinpath(f\"ldd_{d}.csv\"))\n",
-    "    for d in [\"train\", \"val\", \"test\"]\n",
-    "]\n",
-    "\n",
-    "print(len(train), len(test), len(val))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Test Augmentations"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# aug_ = lpem.get_augmentations(image_size=10, kinds=[\"resize\", \"train\"])\n",
-    "\n",
-    "# test_aug_dataset = lpem.LeafDiskDetectorDataset(csv=train, transform=aug_)\n",
-    "\n",
-    "# file_name = train.sample(n=1).plate_name.to_list()[0]\n",
-    "\n",
-    "# print(aug_[0].width, aug_[0].height)\n",
-    "\n",
-    "# lpem.make_patches_grid(\n",
-    "#     images=[\n",
-    "#         test_aug_dataset.draw_image_with_boxes(plate_name=file_name) for _ in range(12)\n",
-    "#     ],\n",
-    "#     row_count=3,\n",
-    "#     col_count=4,\n",
-    "#     figsize=(12, 6),\n",
-    "# )"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Train"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# model = lpem.LeafDiskDetector(\n",
-    "#     batch_size=15,\n",
-    "#     learning_rate=7.0e-05,\n",
-    "#     image_factor=10,\n",
-    "#     max_epochs=1000,\n",
-    "#     train_data=train,\n",
-    "#     val_data=val,\n",
-    "#     test_data=test,\n",
-    "#     augmentations_kinds=[\"resize\", \"train\", \"to_tensor\"],\n",
-    "#     augmentations_params={\"gamma\": (60, 180)},\n",
-    "#     num_workers=2,\n",
-    "#     accumulate_grad_batches=5,\n",
-    "#     scheduler=\"steplr\",\n",
-    "#     scheduler_params={\"step_size\": 10, \"gamma\": 0.80},\n",
-    "# )\n",
-    "\n",
-    "# model.eval()\n",
-    "# len(model(torch.rand(2, 3, 128, 128)))\n",
-    "\n",
-    "# model.hr_desc()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# trainer = Trainer(\n",
-    "#     default_root_dir=str(cc.path_to_chk_detector),\n",
-    "#     logger=TensorBoardLogger(\n",
-    "#         save_dir=str(cc.path_to_chk_detector),\n",
-    "#         version=model.model_name + \"_\" + dt.now().strftime(\"%Y%m%d_%H%M%S\"),\n",
-    "#         name=\"lightning_logs\",\n",
-    "#     ),\n",
-    "#     accelerator=\"gpu\",\n",
-    "#     max_epochs=model.max_epochs,\n",
-    "#     log_every_n_steps=5,\n",
-    "#     callbacks=[\n",
-    "#         RichProgressBar(),\n",
-    "#         EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=10, min_delta=0.0005),\n",
-    "#         ModelCheckpoint(\n",
-    "#             save_top_k=1,\n",
-    "#             monitor=\"val_loss\",\n",
-    "#             auto_insert_metric_name=True,\n",
-    "#             filename=model.model_name\n",
-    "#             + \"-{val_loss:.3f}-{epoch}-{train_loss:.3f}-{step}\",\n",
-    "#         ),\n",
-    "#         LearningRateMonitor(logging_interval=\"epoch\"),\n",
-    "#     ],\n",
-    "#     accumulate_grad_batches=model.accumulate_grad_batches,\n",
-    "# )\n",
-    "\n",
-    "# trainer.fit(model)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Extract Patches"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Load Model"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ld_model: lpem.LeafDiskDetector = lpem.LeafDiskDetector.load_from_checkpoint(\n",
-    "    cc.path_to_chk_detector.joinpath(\"leaf_disc_detector.ckpt\")\n",
-    ")\n",
-    "ld_model.hr_desc()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Predict All Bounding Boxes"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "bb_predictions_path = cc.path_to_data.joinpath(\"train_ld_bounding_boxes.csv\")\n",
-    "\n",
-    "bb_predictions = (\n",
-    "    cf.read_dataframe(bb_predictions_path)\n",
-    "    if bb_predictions_path.is_file() is True\n",
-    "    else pd.DataFrame()\n",
-    ")\n",
-    "\n",
-    "bb_predictions"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "plates = list(cc.path_to_plates.rglob(\"*.JPG\"))\n",
-    "len(plates)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "errors = []\n",
-    "handled_plates = bb_predictions.file_name.unique()\n",
-    "\n",
-    "for plate in tqdm(plates):\n",
-    "    if \"file_name\" in bb_predictions and plate.name in handled_plates:\n",
-    "        continue\n",
-    "    try:\n",
-    "        current_data = ld_model.index_plate(plate) >> mutate(\n",
-    "            disc_name=s.file_name.str.replace(\" \", \"\").replace(\".JPG\", \"\")\n",
-    "            + \"_\"\n",
-    "            + s.row.astype(str)\n",
-    "            + \"_\"\n",
-    "            + s.col.astype(str)\n",
-    "            + \".png\"\n",
-    "        )\n",
-    "        bb_predictions = pd.concat([bb_predictions, current_data])\n",
-    "    except:\n",
-    "        errors.append(plate)\n",
-    "\n",
-    "print(errors)\n",
-    "cf.write_dataframe(\n",
-    "    bb_predictions.sort_values([\"file_name\", \"col\", \"row\"]).reset_index(drop=True)\n",
-    "    >> mutate(disc_name=s.disc_name.str.replace(\".JPG\", \"\")),\n",
-    "    bb_predictions_path,\n",
-    ")\n",
-    "\n",
-    "bb_predictions = cf.read_dataframe(bb_predictions_path)\n",
-    "bb_predictions"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "selected_image = random.choice(plates)\n",
-    "bboxes = bb_predictions >> sfilter(s.file_name == selected_image.name)\n",
-    "pn.Column(\n",
-    "    pn.pane.Markdown(f\"### {selected_image.name}\"),\n",
-    "    pn.pane.DataFrame(bboxes),\n",
-    "    pn.pane.Image(\n",
-    "        ci.to_pil(\n",
-    "            lpem.print_boxes(\n",
-    "                image_name=selected_image,\n",
-    "                boxes=bboxes,\n",
-    "                draw_first_line=True,\n",
-    "                return_plot=False,\n",
-    "            )  #\n",
-    "        ),\n",
-    "        sizing_mode=\"scale_width\",\n",
-    "    ),\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Extract Needed Patches"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "#### Model Training"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_model_training = pd.concat(\n",
-    "    [\n",
-    "        cf.read_dataframe(cc.path_to_data.joinpath(f\"oiv_{d}.csv\"))\n",
-    "        for d in [\"train\", \"val\", \"test\"]\n",
-    "    ]\n",
-    ").sort_values([\"file_name\"]).reset_index(drop=True)\n",
-    "df_model_training"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "err = {}\n",
-    "\n",
-    "for file_name in tqdm(df_model_training.file_name):\n",
-    "    row = (bb_predictions >> sfilter(s.disc_name == file_name)).reset_index(drop=True)\n",
-    "    lpem.handle_bbox(\n",
-    "        row.iloc[0],\n",
-    "        add_process_image=True,\n",
-    "        paths=dict(\n",
-    "            segmented_leaf_disc=cc.path_to_leaf_discs,\n",
-    "            leaf_disc_patch=cc.path_to_leaf_patches,\n",
-    "            plates=cc.path_to_plates,\n",
-    "        ),\n",
-    "        errors=err,\n",
-    "    )\n",
-    "err"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "#### Genotype differenciation"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_gd = cf.read_dataframe(\n",
-    "    cc.path_to_data.joinpath(\"genotype_differenciation_dataset.csv\")\n",
-    ")\n",
-    "df_gd"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "err = {}\n",
-    "\n",
-    "for file_name in tqdm(df_gd.file_name):\n",
-    "    row = (bb_predictions >> sfilter(s.disc_name == file_name)).reset_index(drop=True)\n",
-    "    lpem.handle_bbox(\n",
-    "        row.iloc[0],\n",
-    "        add_process_image=True,\n",
-    "        paths=dict(\n",
-    "            segmented_leaf_disc=cc.path_to_leaf_discs,\n",
-    "            leaf_disc_patch=cc.path_to_leaf_patches,\n",
-    "            plates=cc.path_to_plates,\n",
-    "        ),\n",
-    "        errors=err,\n",
-    "    )\n",
-    "err"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "env",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.2"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0baba4ac3a68926405c55c55f18e86a03e58db10eb26d91c6b2e496d6c11108
+size 12983025

src/leaf_patch_gen_diff.py

@@ -0,0 +1,227 @@
+import numpy as np
+import pandas as pd
+
+import scipy.stats as stats
+import statsmodels.api as sm
+from statsmodels.formula.api import ols
+from statsmodels.regression.linear_model import RegressionResultsWrapper
+from statsmodels.stats.multicomp import pairwise_tukeyhsd
+
+from matplotlib.figure import Figure
+import seaborn as sns
+import panel as pn
+
+import com_const as cc
+import com_func as cf
+import com_image as ci
+
+stars = [-np.log(0.05), -np.log(0.01), -np.log(0.001), -np.log(0.0001)]
+
+
+def plot_single_progression(
+    ax,
+    df,
+    target,
+    title: str,
+    hue="gen",
+    style="gen",
+    show_legend: bool = False,
+):
+    lp = sns.lineplot(
+        df.sort_values(hue),
+        x="dpi",
+        y=target,
+        hue=hue,
+        markers=True,
+        style=style,
+        dashes=False,
+        palette="tab10",
+        markersize=12,
+        ax=ax,
+    )
+    lp.set_yticklabels(["", "3", "", "5", "", "7", "", "9"])
+    ax.set_title(title)
+    if show_legend is True:
+        sns.move_legend(ax, "upper left", bbox_to_anchor=(1, 1))
+    else:
+        ax.get_legend().set_visible(False)
+
+
+def get_model(
+    df: pd.DataFrame, target: str, formula: str, dpi: int = None
+) -> RegressionResultsWrapper:
+    df_ = df[df.dpi == dpi] if dpi is not None else df
+    return ols(f"{target} {formula}", data=df_).fit()
+
+
+def anova_table(aov, add_columns: bool = True):
+    """
+    The function below was created specifically for the one-way ANOVA table
+    results returned for Type II sum of squares
+    """
+    if add_columns is True:
+        aov["mean_sq"] = aov[:]["sum_sq"] / aov[:]["df"]
+
+        aov["eta_sq"] = aov[:-1]["sum_sq"] / sum(aov["sum_sq"])
+
+        aov["omega_sq"] = (
+            aov[:-1]["sum_sq"] - (aov[:-1]["df"] * aov["mean_sq"][-1])
+        ) / (sum(aov["sum_sq"]) + aov["mean_sq"][-1])
+
+        cols = ["sum_sq", "df", "mean_sq", "F", "PR(>F)", "eta_sq", "omega_sq"]
+        aov = aov[cols]
+    return aov
+
+
+def plot_assumptions(models: list, titles: list, figsize=(12, 4)):
+    fig = Figure(figsize=figsize)
+    fig.suptitle("Probability plot of model residual's", fontsize="x-large")
+    axii = fig.subplots(1, len(models))
+    for ax, model, title in zip(axii, models, titles):
+        _ = stats.probplot(model.resid, plot=ax, rvalue=True)
+        ax.set_title(title)
+
+    return fig
+
+
+def hghlight_rejection(s):
+    df = pd.DataFrame(columns=s.columns, index=s.index)
+    df.loc[s["reject_pred"].ne(s["reject_obs"]), ["group1", "group2"]] = (
+        "background: red"
+    )
+    df.loc[s["reject_pred"].eq(s["reject_obs"]), ["group1", "group2"]] = (
+        "background: green"
+    )
+    df.loc[s.reject_pred, ["reject_pred"]] = "background: green"
+    df.loc[~s.reject_pred, ["reject_pred"]] = "background: red"
+    df.loc[s.reject_obs, ["reject_obs"]] = "background: green"
+    df.loc[~s.reject_obs, ["reject_obs"]] = "background: red"
+    return df
+
+
+def get_tuckey_df(endog, groups, df_genotypes) -> pd.DataFrame:
+    tukey = pairwise_tukeyhsd(endog=endog, groups=groups)
+    df_tuc = pd.DataFrame(tukey._results_table)
+    df_tuc.columns = [str(c) for c in df_tuc.iloc[0]]
+    ret = (
+        df_tuc.drop(df_tuc.index[0])
+        .assign(group1=lambda s: s.group1.astype(str))
+        .assign(group2=lambda s: s.group2.astype(str))
+        .assign(reject=lambda s: s.reject.astype(str) == "True")
+    )
+    ret["p-adj"] = tukey.pvalues
+    if df_genotypes is None:
+        return ret
+    else:
+        return (
+            ret.merge(right=df_genotypes, how="left", left_on="group1", right_on="gen")
+            .drop(["gen"], axis=1)
+            .rename(columns={"rpvloci": "group1_rpvloci"})
+            .merge(right=df_genotypes, how="left", left_on="group2", right_on="gen")
+            .drop(["gen"], axis=1)
+            .rename(columns={"rpvloci": "group2_rpvloci"})
+        )
+
+
+def get_tuckey_compare(df, df_genotypes=None, groups: str = "gen"):
+    merge_on = (
+        ["group1", "group2"]
+        if df_genotypes is None
+        else ["group1", "group2", "group1_rpvloci", "group2_rpvloci"]
+    )
+    df_poiv = get_tuckey_df(df.p_oiv, df[groups], df_genotypes=df_genotypes)
+    df_oiv = get_tuckey_df(df.oiv, df[groups], df_genotypes=df_genotypes)
+    df = pd.merge(left=df_poiv, right=df_oiv, on=merge_on, suffixes=["_pred", "_obs"])
+    return df
+
+
+def df_tukey_cmp_plot(df, groups):
+    df_tukey = (
+        get_tuckey_compare(df=df, groups=groups, df_genotypes=None)
+        .assign(pair_groups=lambda s: s.group1 + "\n" + s.group2)
+        .sort_values("p-adj_obs")
+    )
+
+    df_tukey_reject = df_tukey[df_tukey.reject_obs & df_tukey.reject_pred]
+    df_tukey_accept = df_tukey[~df_tukey.reject_obs & ~df_tukey.reject_pred]
+    df_tukey_diverge = df_tukey[df_tukey.reject_obs != df_tukey.reject_pred]
+
+    fig = Figure(figsize=(20, 6))
+    ax_reject, ax_diverge, ax_accept = fig.subplots(
+        1,
+        3,
+        gridspec_kw={
+            "width_ratios": [
+                len(df_tukey_reject),
+                len(df_tukey_diverge),
+                len(df_tukey_accept),
+            ]
+        },
+        sharey=True,
+    )
+
+    for ax in [ax_reject, ax_accept, ax_diverge]:
+        ax.set_yticks(ticks=stars, labels=["*", "**", "***", "****"])
+        ax.grid(False)
+
+    ax_reject.set_title("Rejected")
+    ax_diverge.set_title("Conflict")
+    ax_accept.set_title("Accepted")
+
+    for ax, df in zip(
+        [ax_reject, ax_accept, ax_diverge],
+        [df_tukey_reject, df_tukey_accept, df_tukey_diverge],
+    ):
+        for star in stars:
+            ax.axhline(y=star, linestyle="-", color="black", alpha=0.5)
+        ax.bar(
+            x=df["pair_groups"],
+            height=-np.log(df["p-adj_pred"]),
+            width=-0.4,
+            align="edge",
+            color="green",
+            label="predictions",
+        )
+        ax.bar(
+            x=df["pair_groups"],
+            height=-np.log(df["p-adj_obs"]),
+            width=0.4,
+            align="edge",
+            color="blue",
+            label="scorings",
+        )
+        ax.margins(0.01)
+
+    ax_accept.legend(loc="upper left", bbox_to_anchor=[0, 1], ncols=1, fancybox=True)
+    ax_reject.set_ylabel("-log(p value)")
+    ax_reject.tick_params(axis="y", which="major", labelsize=16)
+
+    fig.subplots_adjust(wspace=0.05, hspace=0.05)
+
+    return fig
+
+
+def plot_patches(df, diff_only: bool = True):
+    if diff_only is True:
+        df = df[(df.oiv != df.p_oiv)]
+    df = df.assign(diff=lambda s: s.oiv != s.p_oiv).sort_values(
+        ["diff", "oiv", "p_oiv"]
+    )
+    return pn.GridBox(
+        *[
+            pn.Column(
+                pn.pane.Markdown(f"### {row.file_name}|{row.oiv}->p{row.p_oiv}"),
+                pn.pane.Image(
+                    object=ci.enhance_pil_image(
+                        image=ci.load_image(
+                            file_name=row.file_name,
+                            path_to_images=cc.path_to_leaf_patches,
+                        ),
+                        brightness=1.5,
+                    )
+                ),
+            )
+            for _, row in df.iterrows()
+        ],
+        ncols=len(df),
+    )

src/repo_manager.ipynb

@@ -2,10 +2,13 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
+    "from pathlib import Path\n",
+    "import time\n",
+    "\n",
     "from huggingface_hub import create_repo, HfApi\n",
     "\n",
     "import com_const as cc"
@@ -13,7 +16,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -22,27 +25,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "RepoUrl('https://huggingface.co/treizh/oiv_ld_phenotyping', endpoint='https://huggingface.co', repo_type='model', repo_id='treizh/oiv_ld_phenotyping')"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "create_repo(repo_id, exist_ok=True)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -51,22 +43,71 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def upload_file(file_path: Path, dst_path=None):\n",
+    "    api.upload_file(\n",
+    "        path_or_fileobj=file_path,\n",
+    "        path_in_repo=(\n",
+    "            dst_path\n",
+    "            if dst_path is not None\n",
+    "            else str(file_path.relative_to(cc.path_to_root))\n",
+    "        ),\n",
+    "        repo_id=repo_id,\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def upload_folder(fld, multi_commits=True, multi_commits_verbose=True):\n",
+    "    api.upload_folder(\n",
+    "        folder_path=fld,\n",
+    "        repo_id=repo_id,\n",
+    "        path_in_repo=str(fld.relative_to(cc.path_to_root)),\n",
+    "        multi_commits=multi_commits,\n",
+    "        multi_commits_verbose=multi_commits_verbose,\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def upload_big_folder(\n",
+    "    fld, multi_commits=True, multi_commits_verbose=True, max_attempts: int = -1\n",
+    "):\n",
+    "    i = 1\n",
+    "    while i < max_attempts or max_attempts < 0:\n",
+    "        try:\n",
+    "            upload_folder(\n",
+    "                fld=fld,\n",
+    "                multi_commits=multi_commits,\n",
+    "                multi_commits_verbose=multi_commits_verbose,\n",
+    "            )\n",
+    "        except:\n",
+    "            print(f\"Trying once more: {i}\")\n",
+    "            i += 1\n",
+    "            time.sleep(100)\n",
+    "        else:\n",
+    "            break"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "CommitInfo(commit_url='https://huggingface.co/treizh/oiv_ld_phenotyping/commit/ac0c4c71a2b6842d45cd5fa99ca15429f647027c', commit_message='Upload .gitignore with huggingface_hub', commit_description='', oid='ac0c4c71a2b6842d45cd5fa99ca15429f647027c', pr_url=None, pr_revision=None, pr_num=None)"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "api.upload_file(path_or_fileobj=cc.path_to_root.joinpath(\".gitignore\"), path_in_repo=\".gitignore\", repo_id=repo_id)"
+    "# for file in [\".gitignore\", \"LICENSE\", \"README.md\", \"requirements.txt\"]:\n",
+    "#     upload_file(Path(\"..\").joinpath(file), dst_path=file)"
    ]
   },
   {
@@ -75,28 +116,35 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def upload_folder(fld):\n",
-    "    api.upload_folder(folder_path=fld, repo_id=repo_id, path_in_repo=fld.name)"
+    "# upload_folder(cc.path_to_src)\n",
+    "# upload_folder(cc.path_to_data)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "CommitInfo(commit_url='https://huggingface.co/treizh/oiv_ld_phenotyping/commit/fc262e75f2db77ba4440372fe5e564be596968bb', commit_message='Upload folder using huggingface_hub', commit_description='', oid='fc262e75f2db77ba4440372fe5e564be596968bb', pr_url=None, pr_revision=None, pr_num=None)"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
+   "source": [
+    "# upload_file(file_path=cc.path_to_chk_detector.joinpath(\"leaf_disc_detector.ckpt\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# upload_file(file_path=cc.path_to_chk_oiv.joinpath(\"oiv_scorer.ckpt\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
    "source": [
-    "api.upload_folder(folder_path=cc.path_to_src, repo_id=repo_id, path_in_repo=cc.path_to_src.name)"
+    "# upload_big_folder(fld=cc.path_to_leaf_patches, multi_commits_verbose=False)"
    ]
   },
   {
@@ -105,9 +153,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# for file in cc.path_to_src.rglob(\"*\"):\n",
-    "#     if api.file_exists(repo_id=repo_name, filename=file.name) is True:\n",
-    "#         api.delete_file(path_in_repo=file.name, repo_id=repo_name)"
+    "# upload_big_folder(fld=cc.path_to_plates, multi_commits_verbose=False)"
    ]
   },
   {