Upload folder using huggingface_hub

lidar_postprocessing/create_range_imgs.ipynb

@@ -37,7 +37,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "a1d44156",
    "metadata": {},
    "outputs": [],
@@ -48,8 +48,18 @@
     "condition = 'flooded'\n",
     "camera_pos = 'front'\n",
     "# root_directory = f\"../../Datasets/FRED/{condition}/KITTI-style\"\n",
-    "root_directory = f\"C:/Users/conno/Documents/data/FRED/{condition}/KITTI-style\"\n",
+    "# root_directory = f\"C:/Users/conno/Documents/data/FRED/{condition}/KITTI-style\"\n",
     "# 01000000\n",
+    "root_directory = f\"/data/FRED/{condition}/KITTI-style\"\n",
+    "\n",
+    "if not os.path.exists(root_directory):\n",
+    "    snapshot_download(\n",
+    "        repo_id=\"CMalone-Jupiter/FRED\",\n",
+    "        repo_type=\"dataset\",\n",
+    "        local_dir=\"/data/FRED\",\n",
+    "        allow_patterns=f\"{condition}/KITTI-style/{location}_{sequence}/**\",\n",
+    "        token=os.environ.get(\"HF_TOKEN\")\n",
+    "    )\n",
     "\n",
     "############ Define filenames and directories ####################################\n",
     "\n",

localisation/VPR_eval.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -29,6 +29,7 @@
     "from natsort import natsorted, index_natsorted\n",
     "import torch\n",
     "from tqdm.notebook import tqdm\n",
+    "from huggingface_hub import snapshot_download\n",
     "\n",
     "################## set device based on cuda availability #################\n",
     "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
@@ -82,7 +83,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -119,8 +120,18 @@
     "qry_sequence = '20210909_124816_v2'\n",
     "qry_condition = ''\n",
     "qry_camera_pos = 'front'\n",
-    "qry_root_directory = f\"../../Datasets/dalby/KITTI-style/{location}\"\n",
+    "# qry_root_directory = f\"../../Datasets/dalby/KITTI-style/{location}\"\n",
     "qry_vpr_root = f\"../../Datasets/dalby/KITTI-style/{location}/vpr_ftrs/\"\n",
+    "qry_root_directory = f\"/data/FRED/{qry_condition}/KITTI-style\"\n",
+    "\n",
+    "if not os.path.exists(qry_root_directory):\n",
+    "    snapshot_download(\n",
+    "        repo_id=\"CMalone-Jupiter/FRED\",\n",
+    "        repo_type=\"dataset\",\n",
+    "        local_dir=\"/data/FRED\",\n",
+    "        allow_patterns=f\"{qry_condition}/KITTI-style/{location}_{qry_sequence}/**\",\n",
+    "        token=os.environ.get(\"HF_TOKEN\")\n",
+    "    )\n",
     "\n",
     "qry_image_dir = f\"{qry_root_directory}/{qry_sequence}/{qry_camera_pos}-imgs/\"\n",
     "qry_utm_dir = f\"{qry_root_directory}/{qry_sequence}/utm/\"\n",
@@ -147,8 +158,18 @@
     "ref_sequence = '20230509_115540_v2'\n",
     "ref_condition = ''\n",
     "ref_camera_pos = 'front'\n",
-    "ref_root_directory = f\"../../Datasets/dalby/KITTI-style/{location}\"\n",
+    "# ref_root_directory = f\"../../Datasets/dalby/KITTI-style/{location}\"\n",
     "ref_vpr_root = f\"../../Datasets/dalby/KITTI-style/{location}/vpr_ftrs/\"\n",
+    "ref_root_directory = f\"/data/FRED/{ref_condition}/KITTI-style\"\n",
+    "\n",
+    "if not os.path.exists(ref_root_directory):\n",
+    "    snapshot_download(\n",
+    "        repo_id=\"CMalone-Jupiter/FRED\",\n",
+    "        repo_type=\"dataset\",\n",
+    "        local_dir=\"/data/FRED\",\n",
+    "        allow_patterns=f\"{ref_condition}/KITTI-style/{location}_{ref_sequence}/**\",\n",
+    "        token=os.environ.get(\"HF_TOKEN\")\n",
+    "    )\n",
     "\n",
     "ref_image_dir = f\"{ref_root_directory}/{ref_sequence}/{ref_camera_pos}-imgs/\"\n",
     "ref_utm_dir = f\"{ref_root_directory}/{ref_sequence}/utm/\"\n",

localisation/groundtruth_utm-single.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -18,12 +18,13 @@
     "from utils.lidar import PointCloud\n",
     "from utils.camera import ImageData\n",
     "import utils.utils as utils\n",
-    "from natsort import natsorted"
+    "from natsort import natsorted\n",
+    "from huggingface_hub import snapshot_download"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -40,7 +41,17 @@
     "qry_condition = 'flooded'\n",
     "qry_camera_pos = 'front'\n",
     "# qry_root_directory = f\"../../Datasets/FRED/{qry_condition}/KITTI-style\"\n",
-    "qry_root_directory = f\"U:/Research/Projects/KVFPRA9190/FRED/{qry_condition}/KITTI-style\"\n",
+    "# qry_root_directory = f\"U:/Research/Projects/KVFPRA9190/FRED/{qry_condition}/KITTI-style\"\n",
+    "qry_root_directory = f\"/data/FRED/{qry_condition}/KITTI-style\"\n",
+    "\n",
+    "if not os.path.exists(qry_root_directory):\n",
+    "    snapshot_download(\n",
+    "        repo_id=\"CMalone-Jupiter/FRED\",\n",
+    "        repo_type=\"dataset\",\n",
+    "        local_dir=\"/data/FRED\",\n",
+    "        allow_patterns=f\"{qry_condition}/KITTI-style/{location}_{qry_sequence}/**\",\n",
+    "        token=os.environ.get(\"HF_TOKEN\")\n",
+    "    )\n",
     "\n",
     "qry_image_dir = f\"{qry_root_directory}/{location}_{qry_sequence}/{qry_camera_pos}-imgs/\"\n",
     "qry_utm_dir = f\"{qry_root_directory}/{location}_{qry_sequence}/utm/\"\n",
@@ -54,7 +65,17 @@
     "ref_condition = 'dry'\n",
     "ref_camera_pos = 'front'\n",
     "# ref_root_directory = f\"../../Datasets/FRED/{ref_condition}/KITTI-style\"\n",
-    "ref_root_directory = f\"U:/Research/Projects/KVFPRA9190/FRED/{ref_condition}/KITTI-style\"\n",
+    "# ref_root_directory = f\"U:/Research/Projects/KVFPRA9190/FRED/{ref_condition}/KITTI-style\"\n",
+    "ref_root_directory = f\"/data/FRED/{ref_condition}/KITTI-style\"\n",
+    "\n",
+    "if not os.path.exists(ref_root_directory):\n",
+    "    snapshot_download(\n",
+    "        repo_id=\"CMalone-Jupiter/FRED\",\n",
+    "        repo_type=\"dataset\",\n",
+    "        local_dir=\"/data/FRED\",\n",
+    "        allow_patterns=f\"{ref_condition}/KITTI-style/{location}_{ref_sequence}/**\",\n",
+    "        token=os.environ.get(\"HF_TOKEN\")\n",
+    "    )\n",
     "\n",
     "ref_image_dir = f\"{ref_root_directory}/{location}_{ref_sequence}/{ref_camera_pos}-imgs/\"\n",
     "ref_utm_dir = f\"{ref_root_directory}/{location}_{ref_sequence}/utm/\"\n",

localisation/plot_utm_traj.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -29,13 +29,14 @@
     "from utils.camera import ImageData\n",
     "import utils.utils as utils\n",
     "from natsort import natsorted\n",
+    "from huggingface_hub import snapshot_download\n",
     "\n",
     "cmap = plt.get_cmap(\"jet\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -59,7 +60,17 @@
     "qry_condition = 'flooded'\n",
     "qry_camera_pos = 'front'\n",
     "# qry_root_directory = f\"../../Datasets/FRED/{qry_condition}/KITTI-style\"\n",
-    "qry_root_directory = f\"U:/Research/Projects/KVFPRA9190/FRED/{qry_condition}/KITTI-style\"\n",
+    "# qry_root_directory = f\"U:/Research/Projects/KVFPRA9190/FRED/{qry_condition}/KITTI-style\"\n",
+    "qry_root_directory = f\"/data/FRED/{qry_condition}/KITTI-style\"\n",
+    "\n",
+    "if not os.path.exists(qry_root_directory):\n",
+    "    snapshot_download(\n",
+    "        repo_id=\"CMalone-Jupiter/FRED\",\n",
+    "        repo_type=\"dataset\",\n",
+    "        local_dir=\"/data/FRED\",\n",
+    "        allow_patterns=f\"{qry_condition}/KITTI-style/{location}_{qry_sequence}/**\",\n",
+    "        token=os.environ.get(\"HF_TOKEN\")\n",
+    "    )\n",
     "\n",
     "qry_image_dir = f\"{qry_root_directory}/{location}_{qry_sequence}/{qry_camera_pos}-imgs/\"\n",
     "qry_utm_dir = f\"{qry_root_directory}/{location}_{qry_sequence}/utm/\"\n",
@@ -72,7 +83,17 @@
     "ref_condition = 'dry'\n",
     "ref_camera_pos = 'front'\n",
     "# ref_root_directory = f\"../../Datasets/FRED/{ref_condition}/KITTI-style\"\n",
-    "ref_root_directory = f\"U:/Research/Projects/KVFPRA9190/FRED/{ref_condition}/KITTI-style\"\n",
+    "# ref_root_directory = f\"U:/Research/Projects/KVFPRA9190/FRED/{ref_condition}/KITTI-style\"\n",
+    "ref_root_directory = f\"/data/FRED/{ref_condition}/KITTI-style\"\n",
+    "\n",
+    "if not os.path.exists(ref_root_directory):\n",
+    "    snapshot_download(\n",
+    "        repo_id=\"CMalone-Jupiter/FRED\",\n",
+    "        repo_type=\"dataset\",\n",
+    "        local_dir=\"/data/FRED\",\n",
+    "        allow_patterns=f\"{ref_condition}/KITTI-style/{location}_{ref_sequence}/**\",\n",
+    "        token=os.environ.get(\"HF_TOKEN\")\n",
+    "    )\n",
     "\n",
     "ref_image_dir = f\"{ref_root_directory}/{location}_{ref_sequence}/{ref_camera_pos}-imgs/\"\n",
     "ref_utm_dir = f\"{ref_root_directory}/{location}_{ref_sequence}/utm/\"\n",

segmentation/show_labels-single.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19,12 +19,13 @@
     "from utils.lidar import PointCloud\n",
     "from utils.camera import ImageData\n",
     "import utils.utils as utils\n",
-    "from natsort import natsorted"
+    "from natsort import natsorted\n",
+    "from huggingface_hub import snapshot_download"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -37,7 +38,17 @@
     "# sequence = '20250811_113017'\n",
     "condition = 'flooded'\n",
     "camera_pos = 'front'\n",
-    "root_directory = f\"../../Datasets/FRED/{condition}/KITTI-style\"\n",
+    "# root_directory = f\"../../Datasets/FRED/{condition}/KITTI-style\"\n",
+    "root_directory = f\"/data/FRED/{condition}/KITTI-style\"\n",
+    "\n",
+    "if not os.path.exists(root_directory):\n",
+    "    snapshot_download(\n",
+    "        repo_id=\"CMalone-Jupiter/FRED\",\n",
+    "        repo_type=\"dataset\",\n",
+    "        local_dir=\"/data/FRED\",\n",
+    "        allow_patterns=f\"{condition}/KITTI-style/{location}_{sequence}/**\",\n",
+    "        token=os.environ.get(\"HF_TOKEN\")\n",
+    "    )\n",
     "\n",
     "image_dir = f\"{root_directory}/{location}_{sequence}/{camera_pos}-imgs/\"\n",
     "label_dir = f\"{root_directory}/{location}_{sequence}/{camera_pos}-labels/\"\n",

visualisation/points2image-single.ipynb

@@ -21,7 +21,7 @@
     "from utils.camera import ImageData\n",
     "import utils.utils as utils\n",
     "from natsort import natsorted\n",
-    "import huggingface_hub\n",
+    "from huggingface_hub import snapshot_download\n",
     "\n",
     "cmap = plt.get_cmap(\"jet\")"
    ]
@@ -48,18 +48,16 @@
     "# root_directory = f\"../../Datasets/FRED/{condition}/KITTI-style\"\n",
     "# root_directory = f\"C:/Users/conno/Documents/data/FRED/{condition}/KITTI-style\"\n",
     "# root_directory = f\"U:/Research/Projects/KVFPRA9190/FRED/{condition}/KITTI-style\"\n",
-    "root_directory = \"data/FRED/{condition}/KITTI-style\"\n",
-    "# 01000000\n",
-    "\n",
-    "from huggingface_hub import snapshot_download\n",
-    "\n",
-    "snapshot_download(\n",
-    "    repo_id=\"CMalone-Jupiter/FRED\",\n",
-    "    repo_type=\"dataset\",\n",
-    "    local_dir=\"/data/FRED\",\n",
-    "    allow_patterns=f\"{condition}/KITTI-style/{location}_{sequence}/**\",\n",
-    "    token=os.environ.get(\"HF_TOKEN\")\n",
-    ")\n",
+    "root_directory = f\"/data/FRED/{condition}/KITTI-style\"\n",
+    "\n",
+    "if not os.path.exists(root_directory):\n",
+    "    snapshot_download(\n",
+    "        repo_id=\"CMalone-Jupiter/FRED\",\n",
+    "        repo_type=\"dataset\",\n",
+    "        local_dir=\"/data/FRED\",\n",
+    "        allow_patterns=f\"{condition}/KITTI-style/{location}_{sequence}/**\",\n",
+    "        token=os.environ.get(\"HF_TOKEN\")\n",
+    "    )\n",
     "\n",
     "############ Define filenames and directories ####################################\n",
     "\n",