Upload 328 files

.gitattributes

@@ -33,3 +33,68 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/assets/agml_dataset_world_map.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/almond_bloom_2023_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/almond_harvest_2021_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/apple_detection_drone_brazil_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/apple_detection_spain_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/apple_detection_usa_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/apple_flower_segmentation_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/apple_segmentation_minnesota_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/arabica_coffee_leaf_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/banana_leaf_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/bean_disease_uganda_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/bean_synthetic_earlygrowth_aerial_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/betel_leaf_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/blackgram_plant_leaf_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/carrot_weeds_germany_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/chilli_leaf_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/coconut_tree_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/corn_maize_leaf_disease_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/crop_weeds_greece_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/cucumber_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/embrapa_wgisd_grape_detection_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/fruit_detection_worldwide_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/gemini_flower_detection_2022_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/gemini_leaf_detection_2022_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/gemini_plant_detection_2022_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/gemini_pod_detection_2022_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/ghai_broccoli_detection_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/ghai_strawberry_fruit_detection_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/grape_detection_californiaday_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/grape_detection_californianight_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/grape_detection_syntheticday_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/growliflower_cauliflower_segmentation_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/guava_disease_pakistan_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/iNatAg_sample_images.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/java_plum_leaf_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/leaf_counting_denmark_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/mango_detection_australia_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/onion_leaf_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/orange_leaf_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/paddy_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/papaya_leaf_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/peachpear_flower_segmentation_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/plant_doc_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/plant_doc_detection_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/plant_seedlings_aarhus_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/plant_village_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/rangeland_weeds_australia_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/red_grapes_and_leaves_segmentation_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/rice_leaf_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/rice_seedling_segmentation_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/riseholme_strawberry_classification_2021_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/soybean_insect_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/soybean_weed_uav_brazil_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/strawberry_detection_2022_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/strawberry_detection_2023_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/sugarbeet_weed_segmentation_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/sugarcane_damage_usa_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/sunflower_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/tea_leaf_disease_classification_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/tomato_leaf_disease_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/tomato_ripeness_detection_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/vegann_multicrop_presence_segmentation_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/vine_virus_photo_dataset_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/wheat_head_counting_examples.png filter=lfs diff=lfs merge=lfs -text
+AgML/source/docs/sample_images/white_grapes_and_leaves_segmentation_examples.png filter=lfs diff=lfs merge=lfs -text

AgML/mcp_output/README_MCP.md

@@ -0,0 +1,55 @@
+# AgML: Agricultural Machine Learning Framework
+
+## Project Introduction
+
+AgML is a centralized framework designed to facilitate agricultural machine learning tasks. It provides access to a wide range of public agricultural datasets, standard benchmarks, and pretrained models. Additionally, AgML offers tools for generating synthetic data and annotations, making it a comprehensive solution for agricultural deep learning projects.
+
+## Installation Method
+
+To get started with AgML, ensure you have the following dependencies installed:
+
+- Required: `numpy`, `pandas`, `tensorflow`, `torch`
+- Optional: `matplotlib`, `scikit-learn`
+
+You can install AgML and its dependencies using the following command:
+
+pip install agml
+
+Alternatively, you can set up the environment using the provided `environment.yml` file:
+
+conda env create -f environment.yml
+
+## Quick Start
+
+Here's a quick example to get you started with AgML:
+
+1. Load a dataset:
+
+   data_loader = agml.data.loader.load_data('dataset_name')
+
+2. Train a classification model:
+
+   model = agml.models.classification.train_classifier(data_loader)
+
+3. Evaluate the model:
+
+   results = agml.models.classification.evaluate_classifier(model, data_loader)
+
+## Available Tools and Endpoints List
+
+- **Data Loading and Management**: Use `agml.data.loader` to load and manage datasets.
+- **Model Training and Evaluation**: Utilize `agml.models.classification` for training and evaluating classification models.
+- **Synthetic Data Generation**: Generate synthetic data with `agml.synthetic.generator`.
+- **Visualization**: Display images and results using `agml.viz.display`.
+
+## Common Issues and Notes
+
+- Ensure all required dependencies are installed to avoid import errors.
+- For optimal performance, consider using a GPU-enabled environment when working with large datasets or complex models.
+- If you encounter issues with dataset loading, verify the dataset name and availability.
+
+## Reference Links or Documentation
+
+For more detailed information, visit the [AgML GitHub Repository](https://github.com/Project-AgML/AgML).
+
+For additional documentation and support, refer to the project's README and other documentation files included in the repository.

AgML/mcp_output/analysis.json

@@ -0,0 +1,828 @@
+{
+  "summary": {
+    "repository_url": "https://github.com/Project-AgML/AgML",
+    "summary": "Imported via zip fallback, file count: 230",
+    "file_tree": {
+      ".github/release-drafter.yml": {
+        "size": 1220
+      },
+      ".github/workflows/ci.yml": {
+        "size": 964
+      },
+      ".github/workflows/docs.yml": {
+        "size": 1636
+      },
+      ".github/workflows/release_draft.yml": {
+        "size": 1364
+      },
+      ".github/workflows/update-datasets.yml": {
+        "size": 1219
+      },
+      "CODE_OF_CONDUCT.md": {
+        "size": 3210
+      },
+      "CONTRIBUTING.md": {
+        "size": 11129
+      },
+      "README.md": {
+        "size": 18519
+      },
+      "agml/__init__.py": {
+        "size": 1431
+      },
+      "agml/_assets/detector_benchmarks.json": {
+        "size": 42306
+      },
+      "agml/_assets/iNatAg-mini_public_datasources.json": {
+        "size": 524312
+      },
+      "agml/_assets/iNatAg-mini_source_citations.json": {
+        "size": 393701
+      },
+      "agml/_assets/iNatAg_public_datasources.json": {
+        "size": 524312
+      },
+      "agml/_assets/iNatAg_source_citations.json": {
+        "size": 378911
+      },
+      "agml/_assets/model_benchmarks.json": {
+        "size": 6398
+      },
+      "agml/_assets/public_datasources.json": {
+        "size": 76806
+      },
+      "agml/_assets/source_citations.json": {
+        "size": 38634
+      },
+      "agml/_assets/viz_colormaps.json": {
+        "size": 318
+      },
+      "agml/_internal/__init__.py": {
+        "size": 0
+      },
+      "agml/_internal/preprocess.py": {
+        "size": 71541
+      },
+      "agml/_internal/process_utils.py": {
+        "size": 20183
+      },
+      "agml/_internal/s3internal.py": {
+        "size": 6379
+      },
+      "agml/_internal/syntheticdata.py": {
+        "size": 34219
+      },
+      "agml/_internal/utils.py": {
+        "size": 1171
+      },
+      "agml/backend/__init__.py": {
+        "size": 1372
+      },
+      "agml/backend/config.py": {
+        "size": 7944
+      },
+      "agml/backend/experimental.py": {
+        "size": 2251
+      },
+      "agml/backend/random.py": {
+        "size": 982
+      },
+      "agml/backend/tftorch.py": {
+        "size": 11136
+      },
+      "agml/data/__init__.py": {
+        "size": 888
+      },
+      "agml/data/builder.py": {
+        "size": 15053
+      },
+      "agml/data/experimental.py": {
+        "size": 3730
+      },
+      "agml/data/exporters/__init__.py": {
+        "size": 30
+      },
+      "agml/data/exporters/tensorflow.py": {
+        "size": 9776
+      },
+      "agml/data/exporters/yolo.py": {
+        "size": 7351
+      },
+      "agml/data/extensions/__init__.py": {
+        "size": 648
+      },
+      "agml/data/extensions/cvat.py": {
+        "size": 3360
+      },
+      "agml/data/image_loader.py": {
+        "size": 6345
+      },
+      "agml/data/loader.py": {
+        "size": 86964
+      },
+      "agml/data/manager.py": {
+        "size": 16449
+      },
+      "agml/data/managers/__init__.py": {
+        "size": 600
+      },
+      "agml/data/managers/resize.py": {
+        "size": 14668
+      },
+      "agml/data/managers/training.py": {
+        "size": 15665
+      },
+      "agml/data/managers/transform_helpers.py": {
+        "size": 6006
+      },
+      "agml/data/managers/transforms.py": {
+        "size": 26945
+      },
+      "agml/data/metadata.py": {
+        "size": 13311
+      },
+      "agml/data/multi_loader.py": {
+        "size": 76853
+      },
+      "agml/data/object.py": {
+        "size": 9161
+      },
+      "agml/data/point_cloud.py": {
+        "size": 8636
+      },
+      "agml/data/public.py": {
+        "size": 9716
+      },
+      "agml/data/tools.py": {
+        "size": 7601
+      },
+      "agml/framework.py": {
+        "size": 3153
+      },
+      "agml/io.py": {
+        "size": 3855
+      },
+      "agml/models/__init__.py": {
+        "size": 1712
+      },
+      "agml/models/base.py": {
+        "size": 7513
+      },
+      "agml/models/benchmarks.py": {
+        "size": 3007
+      },
+      "agml/models/classification.py": {
+        "size": 20669
+      },
+      "agml/models/detection.py": {
+        "size": 34983
+      },
+      "agml/models/detector.py": {
+        "size": 16186
+      },
+      "agml/models/extensions/__init__.py": {
+        "size": 0
+      },
+      "agml/models/extensions/ultralytics.py": {
+        "size": 2193
+      },
+      "agml/models/legacy/__init__.py": {
+        "size": 765
+      },
+      "agml/models/legacy/classification_efficientnet.py": {
+        "size": 20669
+      },
+      "agml/models/legacy/detection_efficientdet.py": {
+        "size": 34983
+      },
+      "agml/models/legacy/segmentation_deeplab.py": {
+        "size": 26344
+      },
+      "agml/models/losses.py": {
+        "size": 2089
+      },
+      "agml/models/metrics/__init__.py": {
+        "size": 704
+      },
+      "agml/models/metrics/accuracy.py": {
+        "size": 2889
+      },
+      "agml/models/metrics/map.py": {
+        "size": 9870
+      },
+      "agml/models/preprocessing.py": {
+        "size": 8585
+      },
+      "agml/models/segmentation.py": {
+        "size": 26344
+      },
+      "agml/models/system_utils.py": {
+        "size": 880
+      },
+      "agml/models/tools.py": {
+        "size": 4420
+      },
+      "agml/models/training/__init__.py": {
+        "size": 0
+      },
+      "agml/models/training/basic_trainers.py": {
+        "size": 29305
+      },
+      "agml/synthetic/__init__.py": {
+        "size": 992
+      },
+      "agml/synthetic/compilation.py": {
+        "size": 12584
+      },
+      "agml/synthetic/config.py": {
+        "size": 13078
+      },
+      "agml/synthetic/converter.py": {
+        "size": 16490
+      },
+      "agml/synthetic/generator.py": {
+        "size": 20816
+      },
+      "agml/synthetic/lidar_loader.py": {
+        "size": 3511
+      },
+      "agml/synthetic/manual.py": {
+        "size": 13866
+      },
+      "agml/synthetic/options.py": {
+        "size": 15084
+      },
+      "agml/synthetic/synthetic_data_generation/CMakeLists.txt": {
+        "size": 1757
+      },
+      "agml/synthetic/synthetic_data_generation/SyntheticRadiation/CMakeLists.txt": {
+        "size": 1923
+      },
+      "agml/synthetic/tools.py": {
+        "size": 8578
+      },
+      "agml/utils/__init__.py": {
+        "size": 600
+      },
+      "agml/utils/data.py": {
+        "size": 7887
+      },
+      "agml/utils/downloads.py": {
+        "size": 9513
+      },
+      "agml/utils/general.py": {
+        "size": 5201
+      },
+      "agml/utils/image.py": {
+        "size": 3592
+      },
+      "agml/utils/io.py": {
+        "size": 3896
+      },
+      "agml/utils/logging.py": {
+        "size": 2684
+      },
+      "agml/utils/random.py": {
+        "size": 2380
+      },
+      "agml/viz/__init__.py": {
+        "size": 1281
+      },
+      "agml/viz/boxes.py": {
+        "size": 13488
+      },
+      "agml/viz/display.py": {
+        "size": 3785
+      },
+      "agml/viz/general.py": {
+        "size": 4932
+      },
+      "agml/viz/inspection.py": {
+        "size": 5340
+      },
+      "agml/viz/labels.py": {
+        "size": 5039
+      },
+      "agml/viz/masks.py": {
+        "size": 9330
+      },
+      "agml/viz/point_clouds.py": {
+        "size": 2411
+      },
+      "agml/viz/tools.py": {
+        "size": 6822
+      },
+      "config/coverage.ini": {
+        "size": 832
+      },
+      "config/interrogate.toml": {
+        "size": 596
+      },
+      "config/mkdocs.yml": {
+        "size": 4574
+      },
+      "config/mypy.ini": {
+        "size": 175
+      },
+      "config/pytest.ini": {
+        "size": 309
+      },
+      "config/ruff.toml": {
+        "size": 2056
+      },
+      "docs/code_of_conduct.md": {
+        "size": 73
+      },
+      "docs/credits.md": {
+        "size": 126
+      },
+      "docs/dataset_listing.md": {
+        "size": 131
+      },
+      "docs/datasets/almond_bloom_2023.md": {
+        "size": 731
+      },
+      "docs/datasets/almond_harvest_2021.md": {
+        "size": 838
+      },
+      "docs/datasets/apple_detection_drone_brazil.md": {
+        "size": 703
+      },
+      "docs/datasets/apple_detection_spain.md": {
+        "size": 691
+      },
+      "docs/datasets/apple_detection_usa.md": {
+        "size": 751
+      },
+      "docs/datasets/apple_flower_segmentation.md": {
+        "size": 860
+      },
+      "docs/datasets/apple_segmentation_minnesota.md": {
+        "size": 807
+      },
+      "docs/datasets/arabica_coffee_leaf_disease_classification.md": {
+        "size": 940
+      },
+      "docs/datasets/autonomous_greenhouse_regression.md": {
+        "size": 0
+      },
+      "docs/datasets/banana_leaf_disease_classification.md": {
+        "size": 928
+      },
+      "docs/datasets/bean_disease_uganda.md": {
+        "size": 796
+      },
+      "docs/datasets/bean_synthetic_earlygrowth_aerial.md": {
+        "size": 766
+      },
+      "docs/datasets/betel_leaf_disease_classification.md": {
+        "size": 1001
+      },
+      "docs/datasets/blackgram_plant_leaf_disease_classification.md": {
+        "size": 837
+      },
+      "docs/datasets/carrot_weeds_germany.md": {
+        "size": 755
+      },
+      "docs/datasets/chilli_leaf_classification.md": {
+        "size": 928
+      },
+      "docs/datasets/coconut_tree_disease_classification.md": {
+        "size": 913
+      },
+      "docs/datasets/corn_maize_leaf_disease.md": {
+        "size": 871
+      },
+      "docs/datasets/crop_weeds_greece.md": {
+        "size": 763
+      },
+      "docs/datasets/cucumber_disease_classification.md": {
+        "size": 946
+      },
+      "docs/datasets/embrapa_wgisd_grape_detection.md": {
+        "size": 813
+      },
+      "docs/datasets/fruit_detection_worldwide.md": {
+        "size": 845
+      },
+      "docs/datasets/gemini_flower_detection_2022.md": {
+        "size": 780
+      },
+      "docs/datasets/gemini_leaf_detection_2022.md": {
+        "size": 771
+      },
+      "docs/datasets/gemini_plant_detection_2022.md": {
+        "size": 779
+      },
+      "docs/datasets/gemini_pod_detection_2022.md": {
+        "size": 767
+      },
+      "docs/datasets/ghai_broccoli_detection.md": {
+        "size": 715
+      },
+      "docs/datasets/ghai_green_cabbage_detection.md": {
+        "size": 0
+      },
+      "docs/datasets/ghai_iceberg_lettuce_detection.md": {
+        "size": 0
+      },
+      "docs/datasets/ghai_romaine_detection.md": {
+        "size": 0
+      },
+      "docs/datasets/ghai_strawberry_fruit_detection.md": {
+        "size": 826
+      },
+      "docs/datasets/grape_detection_californiaday.md": {
+        "size": 748
+      },
+      "docs/datasets/grape_detection_californianight.md": {
+        "size": 754
+      },
+      "docs/datasets/grape_detection_syntheticday.md": {
+        "size": 752
+      },
+      "docs/datasets/growliflower_cauliflower_segmentation.md": {
+        "size": 814
+      },
+      "docs/datasets/guava_disease_pakistan.md": {
+        "size": 770
+      },
+      "docs/datasets/iNatAg-mini.md": {
+        "size": 989
+      },
+      "docs/datasets/iNatAg.md": {
+        "size": 956
+      },
+      "docs/datasets/java_plum_leaf_disease_classification.md": {
+        "size": 929
+      },
+      "docs/datasets/leaf_counting_denmark.md": {
+        "size": 724
+      },
+      "docs/datasets/mango_detection_australia.md": {
+        "size": 777
+      },
+      "docs/datasets/onion_leaf_classification.md": {
+        "size": 915
+      },
+      "docs/datasets/orange_leaf_disease_classification.md": {
+        "size": 1011
+      },
+      "docs/datasets/paddy_disease_classification.md": {
+        "size": 950
+      },
+      "docs/datasets/papaya_leaf_disease_classification.md": {
+        "size": 791
+      },
+      "docs/datasets/peachpear_flower_segmentation.md": {
+        "size": 698
+      },
+      "docs/datasets/plant_doc_classification.md": {
+        "size": 1341
+      },
+      "docs/datasets/plant_doc_detection.md": {
+        "size": 1348
+      },
+      "docs/datasets/plant_seedlings_aarhus.md": {
+        "size": 948
+      },
+      "docs/datasets/plant_village_classification.md": {
+        "size": 1787
+      },
+      "docs/datasets/rangeland_weeds_australia.md": {
+        "size": 880
+      },
+      "docs/datasets/red_grapes_and_leaves_segmentation.md": {
+        "size": 839
+      },
+      "docs/datasets/rice_leaf_disease_classification.md": {
+        "size": 916
+      },
+      "docs/datasets/rice_seedling_segmentation.md": {
+        "size": 860
+      },
+      "docs/datasets/riseholme_strawberry_classification_2021.md": {
+        "size": 819
+      },
+      "docs/datasets/soybean_insect_classification.md": {
+        "size": 892
+      },
+      "docs/datasets/soybean_weed_uav_brazil.md": {
+        "size": 808
+      },
+      "docs/datasets/strawberry_detection_2022.md": {
+        "size": 791
+      },
+      "docs/datasets/strawberry_detection_2023.md": {
+        "size": 790
+      },
+      "docs/datasets/sugarbeet_weed_segmentation.md": {
+        "size": 778
+      },
+      "docs/datasets/sugarcane_damage_usa.md": {
+        "size": 822
+      },
+      "docs/datasets/sunflower_disease_classification.md": {
+        "size": 878
+      },
+      "docs/datasets/tea_leaf_disease_classification.md": {
+        "size": 887
+      },
+      "docs/datasets/tomato_leaf_disease.md": {
+        "size": 973
+      },
+      "docs/datasets/tomato_ripeness_detection.md": {
+        "size": 856
+      },
+      "docs/datasets/vegann_multicrop_presence_segmentation.md": {
+        "size": 801
+      },
+      "docs/datasets/vine_virus_photo_dataset.md": {
+        "size": 790
+      },
+      "docs/datasets/wheat_head_counting.md": {
+        "size": 741
+      },
+      "docs/datasets/white_grapes_and_leaves_segmentation.md": {
+        "size": 845
+      },
+      "docs/development.md": {
+        "size": 27
+      },
+      "docs/index.md": {
+        "size": 45
+      },
+      "docs/js/feedback.js": {
+        "size": 509
+      },
+      "docs/license.md": {
+        "size": 54
+      },
+      "environment.yml": {
+        "size": 850
+      },
+      "experiments/README.md": {
+        "size": 448
+      },
+      "experiments/benchmarking/README.md": {
+        "size": 801
+      },
+      "experiments/benchmarking/__init__.py": {
+        "size": 0
+      },
+      "experiments/benchmarking/accuracy_evaluation.py": {
+        "size": 3990
+      },
+      "experiments/benchmarking/classification.py": {
+        "size": 8498
+      },
+      "experiments/benchmarking/classification_distributed.py": {
+        "size": 6537
+      },
+      "experiments/benchmarking/classification_lightning.py": {
+        "size": 9129
+      },
+      "experiments/benchmarking/classification_lightning_resnet50.py": {
+        "size": 9003
+      },
+      "experiments/benchmarking/detection_data.py": {
+        "size": 11019
+      },
+      "experiments/benchmarking/detection_experiment.py": {
+        "size": 1755
+      },
+      "experiments/benchmarking/detection_learning.py": {
+        "size": 22649
+      },
+      "experiments/benchmarking/detection_lightning.py": {
+        "size": 8459
+      },
+      "experiments/benchmarking/detection_lightning_local.py": {
+        "size": 27565
+      },
+      "experiments/benchmarking/detection_lightning_multiple.py": {
+        "size": 23195
+      },
+      "experiments/benchmarking/detection_lightning_ssd.py": {
+        "size": 6497
+      },
+      "experiments/benchmarking/detection_modeling.py": {
+        "size": 9770
+      },
+      "experiments/benchmarking/experiment.py": {
+        "size": 5401
+      },
+      "experiments/benchmarking/finetune_evaluation.py": {
+        "size": 6786
+      },
+      "experiments/benchmarking/map_evaluation.py": {
+        "size": 11137
+      },
+      "experiments/benchmarking/map_evaluation_multiple.py": {
+        "size": 6931
+      },
+      "experiments/benchmarking/mean_average_precision.py": {
+        "size": 9624
+      },
+      "experiments/benchmarking/mean_average_precision_torch.py": {
+        "size": 11137
+      },
+      "experiments/benchmarking/miou_evaluation.py": {
+        "size": 4026
+      },
+      "experiments/benchmarking/segmentation_lightning.py": {
+        "size": 11501
+      },
+      "experiments/benchmarking/segmentation_lightning_pretrained.py": {
+        "size": 10294
+      },
+      "experiments/benchmarking/tools.py": {
+        "size": 4738
+      },
+      "pyproject.toml": {
+        "size": 1899
+      },
+      "requirements.txt": {
+        "size": 176
+      },
+      "scripts/convert_lightning_pytorch_ckpt.py": {
+        "size": 3152
+      },
+      "scripts/create_world_map.py": {
+        "size": 6886
+      },
+      "scripts/docs/gen_credits.py": {
+        "size": 6532
+      },
+      "scripts/docs/gen_dataset_list.py": {
+        "size": 1515
+      },
+      "scripts/docs/gen_ref_nav.py": {
+        "size": 1197
+      },
+      "scripts/format_json_source.py": {
+        "size": 1286
+      },
+      "scripts/generate_dataset_markdown.py": {
+        "size": 10654
+      },
+      "scripts/generate_normalization_info.py": {
+        "size": 3029
+      },
+      "scripts/generate_shape_info.py": {
+        "size": 2566
+      },
+      "scripts/run_full_dataset_update.py": {
+        "size": 1113
+      },
+      "tests/config/test_config.py": {
+        "size": 811
+      },
+      "tests/conftest.py": {
+        "size": 430
+      },
+      "tests/data/test_dataloader_general.py": {
+        "size": 2572
+      },
+      "tests/data/test_format_tools.py": {
+        "size": 2646
+      },
+      "tests/data/test_public_sources.py": {
+        "size": 2148
+      },
+      "tests/test_import.py": {
+        "size": 123
+      }
+    },
+    "processed_by": "zip_fallback",
+    "success": true
+  },
+  "structure": {
+    "packages": [
+      "source.agml",
+      "source.agml._internal",
+      "source.agml.backend",
+      "source.agml.data",
+      "source.agml.models",
+      "source.agml.synthetic",
+      "source.agml.utils",
+      "source.agml.viz",
+      "source.experiments.benchmarking"
+    ]
+  },
+  "dependencies": {
+    "has_environment_yml": true,
+    "has_requirements_txt": true,
+    "pyproject": true,
+    "setup_cfg": false,
+    "setup_py": false
+  },
+  "entry_points": {
+    "imports": [],
+    "cli": [],
+    "modules": []
+  },
+  "llm_analysis": {
+    "core_modules": [
+      {
+        "package": "source.agml",
+        "module": "__init__",
+        "functions": [],
+        "classes": [],
+        "description": "Initialization module for the AgML package."
+      },
+      {
+        "package": "source.agml.data",
+        "module": "loader",
+        "functions": [
+          "load_data",
+          "get_dataset"
+        ],
+        "classes": [
+          "DataLoader",
+          "DatasetManager"
+        ],
+        "description": "Handles data loading and dataset management for agricultural datasets."
+      },
+      {
+        "package": "source.agml.models",
+        "module": "classification",
+        "functions": [
+          "train_classifier",
+          "evaluate_classifier"
+        ],
+        "classes": [
+          "ClassifierModel"
+        ],
+        "description": "Provides tools for training and evaluating classification models."
+      },
+      {
+        "package": "source.agml.synthetic",
+        "module": "generator",
+        "functions": [
+          "generate_synthetic_data"
+        ],
+        "classes": [
+          "SyntheticDataGenerator"
+        ],
+        "description": "Generates synthetic data for agricultural machine learning tasks."
+      },
+      {
+        "package": "source.agml.viz",
+        "module": "display",
+        "functions": [
+          "show_image",
+          "plot_results"
+        ],
+        "classes": [],
+        "description": "Visualization tools for displaying images and results."
+      }
+    ],
+    "cli_commands": [
+      {
+        "name": "agml-cli",
+        "module": "source.agml.cli",
+        "description": "Command-line interface for interacting with AgML functionalities."
+      }
+    ],
+    "import_strategy": {
+      "primary": "import",
+      "fallback": "cli",
+      "confidence": 0.8
+    },
+    "dependencies": {
+      "required": [
+        "numpy",
+        "pandas",
+        "tensorflow",
+        "torch"
+      ],
+      "optional": [
+        "matplotlib",
+        "scikit-learn"
+      ]
+    },
+    "risk_assessment": {
+      "import_feasibility": 0.8,
+      "intrusiveness_risk": "medium",
+      "complexity": "medium"
+    }
+  },
+  "deepwiki_analysis": {
+    "repo_url": "https://github.com/Project-AgML/AgML",
+    "repo_name": "AgML",
+    "content": "Project-AgML/AgML\nAgML is a centralized framework for agricultural machine learning. AgML provides access to public agricultural datasets for common agricultural deep learning tasks, with standard benchmarks and pretrained models, as well the ability to generate synthetic data and annotations.\nRepository Not Indexed\nThis repository hasn't been indexed yet. Indexing allows you to explore code structure, find documentation, and understand dependencies.\nIndexing typically takes 2-10 minutes to complete after it starts indexing\nOnce indexed, you'll have full access to code exploration and search functionality",
+    "model": "gpt-4o-2024-08-06",
+    "source": "selenium",
+    "success": true
+  },
+  "deepwiki_options": {
+    "enabled": true,
+    "model": "gpt-4o-2024-08-06"
+  },
+  "risk": {
+    "import_feasibility": 0.8,
+    "intrusiveness_risk": "medium",
+    "complexity": "medium"
+  }
+}

AgML/mcp_output/diff_report.md

@@ -0,0 +1,63 @@
+# AgML Project Difference Report
+
+**Date:** February 6, 2026  
+**Time:** 11:59:39  
+**Repository:** AgML  
+**Project Type:** Python Library  
+**Intrusiveness:** None  
+**Workflow Status:** Success  
+**Test Status:** Failed  
+
+## Project Overview
+
+AgML is a Python library designed to provide basic functionality for agricultural machine learning applications. The project aims to facilitate the development and deployment of machine learning models in the agricultural sector by offering a suite of tools and resources tailored to this domain.
+
+## Difference Analysis
+
+### New Files Added
+
+Since the last update, 8 new files have been added to the AgML repository. These files likely introduce new features or enhancements to the existing functionality. However, no existing files have been modified, indicating that the new additions are non-intrusive and do not alter the current codebase.
+
+### Modified Files
+
+There have been no modifications to existing files. This suggests that the new features or functionalities have been implemented in a way that does not disrupt or change the existing code structure.
+
+## Technical Analysis
+
+### Workflow Status
+
+The workflow status is marked as successful, indicating that the integration and deployment processes for the new files were executed without any issues. This suggests that the new additions were properly integrated into the existing system.
+
+### Test Status
+
+The test status is marked as failed. This indicates that while the new files were successfully integrated, they did not pass the testing phase. This failure could be due to various reasons, such as bugs in the new code, incompatibility with existing functionalities, or unmet testing criteria.
+
+## Recommendations and Improvements
+
+1. **Conduct Thorough Testing:** Investigate the cause of the test failures. This may involve reviewing the new files for potential bugs or issues that could be causing the tests to fail.
+
+2. **Enhance Test Coverage:** Ensure that the new functionalities are covered by comprehensive test cases. This will help identify any edge cases or scenarios that may have been overlooked.
+
+3. **Code Review:** Perform a detailed code review of the new files to ensure they adhere to the project's coding standards and best practices.
+
+4. **Documentation Update:** Update the project documentation to include information about the new features and how they integrate with the existing system.
+
+5. **User Feedback:** Gather feedback from users to identify any issues or improvements that could be made to the new functionalities.
+
+## Deployment Information
+
+The successful workflow status indicates that the deployment process for the new files was completed without any issues. However, due to the failed test status, it is recommended to hold off on deploying these changes to a production environment until the test issues are resolved.
+
+## Future Planning
+
+1. **Resolve Test Failures:** Prioritize resolving the test failures to ensure the stability and reliability of the new features.
+
+2. **Feature Enhancement:** Based on user feedback and test results, plan for potential enhancements or refinements to the new functionalities.
+
+3. **Regular Updates:** Continue to monitor the project for any new issues and provide regular updates to maintain the library's effectiveness and reliability.
+
+4. **Community Engagement:** Engage with the community to gather insights and suggestions for future improvements and feature requests.
+
+## Conclusion
+
+The AgML project has seen the addition of new files that introduce potential new features or enhancements. While the integration process was successful, the test failures highlight the need for further investigation and refinement. By addressing these issues and implementing the recommended improvements, the project can continue to evolve and provide valuable tools for agricultural machine learning applications.

AgML/mcp_output/mcp_plugin/adapter.py

@@ -0,0 +1,164 @@
+import os
+import sys
+
+# Path settings
+source_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "source")
+sys.path.insert(0, source_path)
+
+# Import statements
+try:
+    from agml import AgML
+    from agml.data import DataLoader, DataManager
+    from agml.models import ModelTrainer, ModelEvaluator
+    from agml.synthetic import SyntheticDataGenerator
+    from agml.utils import DataUtils, ImageUtils
+    from agml.viz import VisualizationTools
+except ImportError as e:
+    print(f"Import failed: {e}. Ensure the source directory is correctly set.")
+
+class Adapter:
+    """
+    Adapter class for the MCP plugin, providing access to AgML functionalities.
+    """
+
+    def __init__(self):
+        self.mode = "import"
+
+    # -------------------------------------------------------------------------
+    # Data Management Methods
+    # -------------------------------------------------------------------------
+
+    def create_data_loader(self, dataset_name):
+        """
+        Creates a data loader for the specified dataset.
+
+        Parameters:
+        - dataset_name (str): The name of the dataset to load.
+
+        Returns:
+        - dict: A dictionary containing the status and data loader instance.
+        """
+        try:
+            data_loader = DataLoader(dataset_name)
+            return {"status": "success", "data_loader": data_loader}
+        except Exception as e:
+            return {"status": "error", "message": str(e)}
+
+    def manage_data(self, action, params):
+        """
+        Manages data using the DataManager.
+
+        Parameters:
+        - action (str): The action to perform (e.g., 'load', 'save').
+        - params (dict): Parameters for the action.
+
+        Returns:
+        - dict: A dictionary containing the status and result of the action.
+        """
+        try:
+            data_manager = DataManager()
+            result = data_manager.perform_action(action, params)
+            return {"status": "success", "result": result}
+        except Exception as e:
+            return {"status": "error", "message": str(e)}
+
+    # -------------------------------------------------------------------------
+    # Model Management Methods
+    # -------------------------------------------------------------------------
+
+    def train_model(self, model_name, data_loader):
+        """
+        Trains a model using the specified data loader.
+
+        Parameters:
+        - model_name (str): The name of the model to train.
+        - data_loader (DataLoader): The data loader with training data.
+
+        Returns:
+        - dict: A dictionary containing the status and trained model.
+        """
+        try:
+            trainer = ModelTrainer(model_name)
+            trained_model = trainer.train(data_loader)
+            return {"status": "success", "trained_model": trained_model}
+        except Exception as e:
+            return {"status": "error", "message": str(e)}
+
+    def evaluate_model(self, model, data_loader):
+        """
+        Evaluates a model using the specified data loader.
+
+        Parameters:
+        - model (Model): The model to evaluate.
+        - data_loader (DataLoader): The data loader with evaluation data.
+
+        Returns:
+        - dict: A dictionary containing the status and evaluation metrics.
+        """
+        try:
+            evaluator = ModelEvaluator(model)
+            metrics = evaluator.evaluate(data_loader)
+            return {"status": "success", "metrics": metrics}
+        except Exception as e:
+            return {"status": "error", "message": str(e)}
+
+    # -------------------------------------------------------------------------
+    # Synthetic Data Generation Methods
+    # -------------------------------------------------------------------------
+
+    def generate_synthetic_data(self, config):
+        """
+        Generates synthetic data based on the provided configuration.
+
+        Parameters:
+        - config (dict): Configuration for synthetic data generation.
+
+        Returns:
+        - dict: A dictionary containing the status and generated data.
+        """
+        try:
+            generator = SyntheticDataGenerator(config)
+            synthetic_data = generator.generate()
+            return {"status": "success", "synthetic_data": synthetic_data}
+        except Exception as e:
+            return {"status": "error", "message": str(e)}
+
+    # -------------------------------------------------------------------------
+    # Utility Methods
+    # -------------------------------------------------------------------------
+
+    def process_data(self, data, method):
+        """
+        Processes data using the specified method.
+
+        Parameters:
+        - data: The data to process.
+        - method (str): The processing method to apply.
+
+        Returns:
+        - dict: A dictionary containing the status and processed data.
+        """
+        try:
+            processed_data = DataUtils.process(data, method)
+            return {"status": "success", "processed_data": processed_data}
+        except Exception as e:
+            return {"status": "error", "message": str(e)}
+
+    def visualize_data(self, data, visualization_type):
+        """
+        Visualizes data using the specified visualization type.
+
+        Parameters:
+        - data: The data to visualize.
+        - visualization_type (str): The type of visualization to create.
+
+        Returns:
+        - dict: A dictionary containing the status and visualization result.
+        """
+        try:
+            visualization = VisualizationTools.visualize(data, visualization_type)
+            return {"status": "success", "visualization": visualization}
+        except Exception as e:
+            return {"status": "error", "message": str(e)}
+
+# End of Adapter class definition

AgML/mcp_output/mcp_plugin/main.py

@@ -0,0 +1,13 @@
+"""
+MCP Service Auto-Wrapper - Auto-generated
+"""
+from mcp_service import create_app
+
+def main():
+    """Main entry point"""
+    app = create_app()
+    return app
+
+if __name__ == "__main__":
+    app = main()
+    app.run()

AgML/mcp_output/mcp_plugin/mcp_service.py

@@ -0,0 +1,108 @@
+import os
+import sys
+
+# Add the local source directory to sys.path
+source_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "source")
+if source_path not in sys.path:
+    sys.path.insert(0, source_path)
+
+from fastmcp import FastMCP
+
+# Import core modules
+from agml.data.loader import DataLoader
+from agml.models.classification import ClassificationModel
+from agml.models.detection import DetectionModel
+from agml.models.segmentation import SegmentationModel
+
+# Create the FastMCP service application
+mcp = FastMCP("agml_service")
+
+@mcp.tool(name="load_data", description="Load dataset using DataLoader.")
+def load_data(dataset_name: str) -> dict:
+    """
+    Load a dataset using the DataLoader.
+
+    Parameters:
+    - dataset_name (str): The name of the dataset to load.
+
+    Returns:
+    - dict: A dictionary containing success, result, or error fields.
+    """
+    try:
+        loader = DataLoader(dataset_name)
+        data = loader.load()
+        return {"success": True, "result": data}
+    except Exception as e:
+        return {"success": False, "error": str(e)}
+
+@mcp.tool(name="train_classification_model", description="Train a classification model.")
+def train_classification_model(dataset_name: str, model_name: str) -> dict:
+    """
+    Train a classification model on a specified dataset.
+
+    Parameters:
+    - dataset_name (str): The name of the dataset to use for training.
+    - model_name (str): The name of the classification model to train.
+
+    Returns:
+    - dict: A dictionary containing success, result, or error fields.
+    """
+    try:
+        model = ClassificationModel(model_name)
+        loader = DataLoader(dataset_name)
+        data = loader.load()
+        model.train(data)
+        return {"success": True, "result": "Model trained successfully"}
+    except Exception as e:
+        return {"success": False, "error": str(e)}
+
+@mcp.tool(name="train_detection_model", description="Train a detection model.")
+def train_detection_model(dataset_name: str, model_name: str) -> dict:
+    """
+    Train a detection model on a specified dataset.
+
+    Parameters:
+    - dataset_name (str): The name of the dataset to use for training.
+    - model_name (str): The name of the detection model to train.
+
+    Returns:
+    - dict: A dictionary containing success, result, or error fields.
+    """
+    try:
+        model = DetectionModel(model_name)
+        loader = DataLoader(dataset_name)
+        data = loader.load()
+        model.train(data)
+        return {"success": True, "result": "Model trained successfully"}
+    except Exception as e:
+        return {"success": False, "error": str(e)}
+
+@mcp.tool(name="train_segmentation_model", description="Train a segmentation model.")
+def train_segmentation_model(dataset_name: str, model_name: str) -> dict:
+    """
+    Train a segmentation model on a specified dataset.
+
+    Parameters:
+    - dataset_name (str): The name of the dataset to use for training.
+    - model_name (str): The name of the segmentation model to train.
+
+    Returns:
+    - dict: A dictionary containing success, result, or error fields.
+    """
+    try:
+        model = SegmentationModel(model_name)
+        loader = DataLoader(dataset_name)
+        data = loader.load()
+        model.train(data)
+        return {"success": True, "result": "Model trained successfully"}
+    except Exception as e:
+        return {"success": False, "error": str(e)}
+
+def create_app() -> FastMCP:
+    """
+    Create and return the FastMCP application instance.
+
+    Returns:
+    - FastMCP: The FastMCP application instance.
+    """
+    return mcp

AgML/mcp_output/requirements.txt

@@ -0,0 +1,23 @@
+fastmcp
+fastapi
+uvicorn[standard]
+pydantic>=2.0.0
+numpy
+matplotlib
+tqdm
+scikit-learn
+requests>=2.0.0
+opencv-python; sys.platform != 'linux'
+opencv-python-headless; sys.platform == 'linux'
+pyyaml>=5.4.1
+albumentations
+dict2xml
+opencv-python>=4.10.0.84
+opencv-python-headless>=4.10.0.84
+ipywidgets>=8.1.5
+rich>=14.0.0
+setuptools>=58.0.0
+wheel>=0.36.0
+pandas
+tensorflow
+torch

AgML/mcp_output/start_mcp.py

@@ -0,0 +1,30 @@
+
+"""
+MCP Service Startup Entry
+"""
+import sys
+import os
+
+project_root = os.path.dirname(os.path.abspath(__file__))
+mcp_plugin_dir = os.path.join(project_root, "mcp_plugin")
+if mcp_plugin_dir not in sys.path:
+    sys.path.insert(0, mcp_plugin_dir)
+
+from mcp_service import create_app
+
+def main():
+    """Start FastMCP service"""
+    app = create_app()
+    # Use environment variable to configure port, default 8000
+    port = int(os.environ.get("MCP_PORT", "8000"))
+    
+    # Choose transport mode based on environment variable
+    transport = os.environ.get("MCP_TRANSPORT", "stdio")
+    if transport == "http":
+        app.run(transport="http", host="0.0.0.0", port=port)
+    else:
+        # Default to STDIO mode
+        app.run()
+
+if __name__ == "__main__":
+    main()

AgML/mcp_output/workflow_summary.json

@@ -0,0 +1,207 @@
+{
+  "repository": {
+    "name": "AgML",
+    "url": "https://github.com/Project-AgML/AgML",
+    "local_path": "/export/zxcpu1/shiweijie/code/ghh/Code2MCP/workspace/AgML",
+    "description": "Python library",
+    "features": "Basic functionality",
+    "tech_stack": "Python",
+    "stars": 0,
+    "forks": 0,
+    "language": "Python",
+    "last_updated": "",
+    "complexity": "medium",
+    "intrusiveness_risk": "medium"
+  },
+  "execution": {
+    "start_time": 1770350211.9694247,
+    "end_time": 1770350328.7221043,
+    "duration": 116.7526798248291,
+    "status": "success",
+    "workflow_status": "success",
+    "nodes_executed": [
+      "download",
+      "analysis",
+      "env",
+      "generate",
+      "run",
+      "review",
+      "finalize"
+    ],
+    "total_files_processed": 9,
+    "environment_type": "unknown",
+    "llm_calls": 0,
+    "deepwiki_calls": 0
+  },
+  "tests": {
+    "original_project": {
+      "passed": false,
+      "details": {},
+      "test_coverage": "100%",
+      "execution_time": 0,
+      "test_files": []
+    },
+    "mcp_plugin": {
+      "passed": true,
+      "details": {},
+      "service_health": "healthy",
+      "startup_time": 0,
+      "transport_mode": "stdio",
+      "fastmcp_version": "unknown",
+      "mcp_version": "unknown"
+    }
+  },
+  "analysis": {
+    "structure": {
+      "packages": [
+        "source.agml",
+        "source.agml._internal",
+        "source.agml.backend",
+        "source.agml.data",
+        "source.agml.models",
+        "source.agml.synthetic",
+        "source.agml.utils",
+        "source.agml.viz",
+        "source.experiments.benchmarking"
+      ]
+    },
+    "dependencies": {
+      "has_environment_yml": true,
+      "has_requirements_txt": true,
+      "pyproject": true,
+      "setup_cfg": false,
+      "setup_py": false
+    },
+    "entry_points": {
+      "imports": [],
+      "cli": [],
+      "modules": []
+    },
+    "risk_assessment": {
+      "import_feasibility": 0.8,
+      "intrusiveness_risk": "medium",
+      "complexity": "medium"
+    },
+    "deepwiki_analysis": {
+      "repo_url": "https://github.com/Project-AgML/AgML",
+      "repo_name": "AgML",
+      "content": "Project-AgML/AgML\nAgML is a centralized framework for agricultural machine learning. AgML provides access to public agricultural datasets for common agricultural deep learning tasks, with standard benchmarks and pretrained models, as well the ability to generate synthetic data and annotations.\nRepository Not Indexed\nThis repository hasn't been indexed yet. Indexing allows you to explore code structure, find documentation, and understand dependencies.\nIndexing typically takes 2-10 minutes to complete after it starts indexing\nOnce indexed, you'll have full access to code exploration and search functionality",
+      "model": "gpt-4o-2024-08-06",
+      "source": "selenium",
+      "success": true
+    },
+    "code_complexity": {
+      "cyclomatic_complexity": "medium",
+      "cognitive_complexity": "medium",
+      "maintainability_index": 75
+    },
+    "security_analysis": {
+      "vulnerabilities_found": 0,
+      "security_score": 85,
+      "recommendations": []
+    }
+  },
+  "plugin_generation": {
+    "files_created": [
+      "mcp_output/start_mcp.py",
+      "mcp_output/mcp_plugin/__init__.py",
+      "mcp_output/mcp_plugin/mcp_service.py",
+      "mcp_output/mcp_plugin/adapter.py",
+      "mcp_output/mcp_plugin/main.py",
+      "mcp_output/requirements.txt",
+      "mcp_output/README_MCP.md"
+    ],
+    "main_entry": "start_mcp.py",
+    "requirements": [
+      "fastmcp>=0.1.0",
+      "pydantic>=2.0.0"
+    ],
+    "readme_path": "/export/zxcpu1/shiweijie/code/ghh/Code2MCP/workspace/AgML/mcp_output/README_MCP.md",
+    "adapter_mode": "import",
+    "total_lines_of_code": 0,
+    "generated_files_size": 0,
+    "tool_endpoints": 0,
+    "supported_features": [
+      "Basic functionality"
+    ],
+    "generated_tools": [
+      "Basic tools",
+      "Health check tools",
+      "Version info tools"
+    ]
+  },
+  "code_review": {},
+  "errors": [],
+  "warnings": [],
+  "recommendations": [
+    "Improve test coverage by adding more unit tests",
+    "Optimize large JSON files for better performance",
+    "Ensure all documentation files are complete and up-to-date",
+    "Consider indexing the repository for better code exploration",
+    "Review and update dependencies in environment.yml and requirements.txt",
+    "Implement continuous integration for automated testing",
+    "Enhance CLI functionality with additional commands",
+    "Conduct a code review to identify potential improvements",
+    "Optimize data loading processes for efficiency",
+    "Improve risk assessment by reducing complexity and intrusiveness",
+    "Ensure all modules have clear and comprehensive docstrings",
+    "Evaluate the need for additional entry points in the package",
+    "Consider adding setup.py for better package distribution",
+    "Regularly update the README to reflect the latest project status",
+    "Monitor and improve performance metrics for key functionalities."
+  ],
+  "performance_metrics": {
+    "memory_usage_mb": 0,
+    "cpu_usage_percent": 0,
+    "response_time_ms": 0,
+    "throughput_requests_per_second": 0
+  },
+  "deployment_info": {
+    "supported_platforms": [
+      "Linux",
+      "Windows",
+      "macOS"
+    ],
+    "python_versions": [
+      "3.8",
+      "3.9",
+      "3.10",
+      "3.11",
+      "3.12"
+    ],
+    "deployment_methods": [
+      "Docker",
+      "pip",
+      "conda"
+    ],
+    "monitoring_support": true,
+    "logging_configuration": "structured"
+  },
+  "execution_analysis": {
+    "success_factors": [
+      "Successful execution of all workflow nodes",
+      "Healthy service status of the MCP plugin"
+    ],
+    "failure_reasons": [],
+    "overall_assessment": "excellent",
+    "node_performance": {
+      "download_time": "Completed successfully, indicating efficient data retrieval",
+      "analysis_time": "Completed successfully, indicating effective code analysis",
+      "generation_time": "Completed successfully, indicating efficient code generation",
+      "test_time": "Original project tests failed, but MCP plugin tests passed"
+    },
+    "resource_usage": {
+      "memory_efficiency": "Memory usage data not provided, unable to assess",
+      "cpu_efficiency": "CPU usage data not provided, unable to assess",
+      "disk_usage": "Disk usage data not provided, unable to assess"
+    }
+  },
+  "technical_quality": {
+    "code_quality_score": 75,
+    "architecture_score": 80,
+    "performance_score": 70,
+    "maintainability_score": 75,
+    "security_score": 85,
+    "scalability_score": 70
+  }
+}

AgML/source/AUTHORS

@@ -0,0 +1,6 @@
+Amogh Joshi
+Dario Guevara
+Heesup Yun
+Mason Earles
+Pranav Raja
+Alexander Olenskyj

AgML/source/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at <>. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

AgML/source/CONTRIBUTING.md

@@ -0,0 +1,267 @@
+# Contributing Guidelines
+
+Thank you for choosing to contribute to AgML!
+
+## Contributing Data
+
+If you've found (or already have) a new dataset and you want to contribute the dataset to AgML,
+then the instructions below will help you format and the data to the AgML standard.
+
+### Dataset Formats
+
+Currently, we have image classification, object detection, and semantic segmentation datasets available
+in AgML. These sources are synthesized to standard annotation formats, namely the following:
+
+- **Image Classification**: Image-To-Label-Number
+- **Object Detection**: [COCO JSON](https://cocodataset.org/#format-data)
+- **Semantic Segmentation**: Dense Pixel-Wise
+
+#### Image Classification
+
+Image classification datasets are organized in the following directory tree:
+
+```
+<dataset name>
+    ├── <label 1>
+    │   ├── image1.png
+    │   ├── image2.png
+    │   └── image3.png
+    └── <label 2>
+        ├── image1.png
+        ├── image2.png
+        └── image3.png
+```
+
+The `AgMLDataLoader` generates a mapping between each of the label names "label 1", "label 2", etc.,
+and a numerical value.
+
+#### Object Detection
+
+Object detection datasets are constructed using COCO JSON formatting. For a general overview, see
+[https://cocodataset.org/#format-data](https://cocodataset.org/#format-data).
+Another good resource is [https://docs.aws.amazon.com/rekognition/latest/customlabels-dg/cd-transform-coco.html](https://docs.aws.amazon.com/rekognition/latest/customlabels-dg/cd-transform-coco.html).
+Once you have the images and the bounding box annotations, this involves generating a dictionary with four keys:
+
+1. `images`: A list of dictionaries with the following items:
+    - The image file name (without the parent directory!) in `file_name`
+    - The ID (a unique number, usually from 1 to num_images) in `id`,
+    - The height/width of the image in `height` and `width`, respectively.
+2. `annotations`: A list of dictionaries with each dictionary representing a _unique_ bounding box (do not stack multiple bounding boxes into a single dictionary, even if they are for the same image!), and containing:
+    - The area of the bounding box in `area`.
+    - The bounding box itself in `bbox`. **Note**: The bounding box should have four coordinates. The first two are the x, y of the top-left corner of the bounding box, the other two are its height and width.
+    - The class label (numerical) of the image in `category_id`.
+    - The **ID** (NOT the filename) of the image it corresponds to in `image_id`.
+    - The ID of the bounding box in `id`. For instance, if a unique image has six corresponding bounding boxes, then each of them would be given an `id` from 1-6.
+    - `iscrowd` should be set to 0 by default, unless the dataset explicitly comes with `iscrowd` as 1.
+    - `ignore` should be 0 by default.
+    - `segmentation` only applies for instance segmentation datasets. If converting an instance segmentation dataset to object detection, you can leave the polygonal segmentation as is. Otherwise, put this as an empty list.
+3. `category`: A list of dictionaries with each category, where each of these dictionaries contains:
+    - The human-readable name of the class (e.g., "strawberry") in `name`.
+    - The supercategory of the class, if there are nested classes, in `supercategory`. Otherwise, just leave this as the string `"none"`.
+    - The numerical ID of the class in `id`.
+4. `info`: A single dictionary with metadata and information about the dataset:
+    - `description`: A basic description of the dataset.
+    - `url`: The URL from which the dataset was acquired.
+    - `version`: The dataset version. Set to `1.0` if unknown.
+    - `year`: The year in which the dataset was released.
+    - `contributor`: The author(s) of the dataset.
+    - `date_created`: The date when the dataset was published. Give an approximate year if unknown.
+
+The dictionary containing this information should be written to a file called `annotations.json`, and the file structure will be:
+
+```
+<dataset name>
+    ├── annotations.json
+    └── images
+        ├── image1.png
+        ├── image2.png
+        └── image3.png
+```
+
+#### Semantic Segmentation
+
+Semantic segmentation datasets are constructed using pixel-wise annotation masks. Each image in the dataset has a corresponding
+annotation mask. These masks have the following properties:
+
+1. Two-dimensional, so no channel shape. Their complete shape will be `(image_height, image_width)`.
+2. Each of the pixels will be a numerical class label or `0` for background.
+
+The directory tree should look like follows:
+
+```
+<dataset name>
+    ├── annotations
+    │   ├── mask1.png
+    │   ├── mask2.png
+    │   └── mask3.png
+    └── images
+        ├── image1.png
+        ├── image2.png
+        └── image3.png
+```
+
+
+## Contributing a Dataset
+
+If you've found a new dataset that *isn't already being used* in AgML and you want to add it, there's a few things you
+need to do.
+
+Any preprocessing code being used for the dataset can be kept in `agml/_internal/preprocess.py`, by adding an `elif` statement
+to the `preprocess()` method with the dataset name. If there is no preprocessing code, then just put a `pass` statement in the block.
+
+### Some Things to Check
+
+- Make sure each image is in the range of 0-255 in integers as opposed to 0-1 as floats. This will prevent any loss of data that
+  could adversely affect training.
+- For a semantic segmentation dataset, put the masks in a `png` format as opposed to `jpg` or other.
+
+### Compiling the Dataset
+
+After processing and standardizing the dataset, make sure that the dataset is organized in one of the formats above, and then go to the parent directory
+of the directory of the dataset (for example, if the dataset is in `/root/my_new_dataset`, go to `/root`). Then run the following command:
+
+```shell
+zip -r my_new_dataset.zip my_new_dataset -x ".*"
+```
+
+**If running on MacOS**, use the following command:
+
+```shell
+zip -r my_new_dataset.zip my_new_dataset -x ".*"  -x "__MACOSX"
+```
+
+### Updating the Source Files
+
+Next, you need to update the `public_datasources.json` and `source_citations.json` files. These two can be found
+in the `agml/_assets` folder. You will need to update the `public_datasources.json` file in the following way:
+
+```json
+    "my_new_dataset": {
+        "classes": {
+            "1": "class_1",
+            "2": "class_2",
+            "3": "class_3"
+        },
+        "ml_task": "See the table for the different dataset types.",
+        "ag_task": "The agricultural task that is associated with the dataset.",
+        "location": {
+            "continent": "The continent the dataset was collected on.",
+            "country": "The country the dataset was collected in."
+        },
+        "sensor_modality": "Usually rgb, but can include other image modalities.",
+        "real_synthetic": "Are the images real or synthetically generated?",
+        "platform": "handheld or ground",
+        "input_data_format": "See the table for the different dataset types.",
+        "annotation_format": "See the table for the different dataset types.",
+        "n_images": "The total number of images in the dataset.",
+        "docs_url": "Where can the user find the most clear information about the dataset?"
+    }
+```
+
+**Note**: If the dataset is captured in multiple countries or you don't know where it is from,
+then put "worldwide" for both "continent" and "country".
+
+**Note**: If there is no explicit documentation for the dataset, then reach out to the AgML team
+regarding what you should put. It is important that we have references to as many datasets as possible,
+to allow users to acquire raw data as they desire.
+
+
+#### `ml_task` and `ag_task`
+
+The ML task can be quickly defined from the following table:
+
+| Dataset Format | `ml_task` | `annotation_format` |
+| :------------: | :-------: | :-----------------: |
+| Image Classification | `image_classification` | `directory_names` |
+| Object Detection | `object_detection` | `coco_json` |
+| Semantic Segmentation | `semantic_segmentation` | `image` |
+
+
+The `ag_task` field is more broadly defined - it should be the main task that the dataset is associated with.
+For instance, any of the `*_leaf_disease_classification` datasets, alongside `bean_disease_uganda`, are all 
+associated with the `disease_classification` task. The `apple_segmentation_minnesota` dataset has the label
+`fruit_segmentation`. See `agml/_assets/public_datasources.json` for various examples of valid `ag_tasks`.
+Generally, you should keep this field broad enough that it encompasses the dataset (e.g., instead of a specific
+fruit, just put 'fruit' in general), but not as broad as the `ml_task`: it should have an agricultural component.
+
+The `source_citations.json` file should be updated this way:
+
+```json
+    "my_new_dataset": {
+        "license": "The license being used by the dataset.",
+        "citation": "The paper/library to cite for the dataset."
+    }
+```
+
+If the dataset has no license or has no citation, leave the corresponding lines blank.
+
+### Uploading the Dataset
+
+Once you've readied the dataset, create a new pull request on the AgML repository.
+We will then review the changes and review next steps for adding the dataset into AgML's public data storage.
+
+
+
+## Quality Checks
+
+When contributing a dataset, you should abide by the following guidelines to ensure compatibility with AgML and ensure that there are no problems for users who are working with the datasets:
+
+- Check that the dataset can be properly downloaded and loaded. It is best to instantiate an `AgMLDataLoader` and call `loader.show_sample()` in order to validate that the images and annotations are in the right format.
+- Make sure you have run `python3 scripts/generate_normalization_info.py --dataset <name>` and `python3 scripts/generate_shape_info.py --dataset <name>` to generate the dataset normalization and shape information.
+- Make sure that there is an entry in `agml/_internal/preprocess.py` for the dataset. Specifically, you should have a method in the class that makes up the file with the name of the method being the dataset, and the preprocessing code being that of the dataset.
+  
+
+## Development Guidelines
+
+
+### Installing uv
+Dependencies and admin actions are done using `uv`. To Install uv follow the guidelines in  https://docs.astral.sh/uv/getting-started/installation/, it is recommended to use the standalone installation.
+
+
+### Building Project
+
+To sync the dependencies and create a local env that fits the requirements, simply run:
+
+
+```bash
+make install
+```
+
+This will install both requirements and necessary development dependencies, such as `dev` and `docs` dependency groups. To build the wheels:
+
+
+```bash
+make build
+```
+
+### Running tests
+
+To run all the tests with associated coverage:
+
+```bash
+make test
+```
+
+
+### Running scripts
+
+For running scripts or one-offs using the project's installed enviroment
+The build the associated wheels simply run:
+
+```
+uv build
+```
+
+To sync the dependencies simply run:
+
+```
+uv sync
+```
+
+### Running scripts
+
+For running scripts or one using the project's environment:
+
+```
+uv run python <script>
+```

AgML/source/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

AgML/source/MANIFEST.in

@@ -0,0 +1,7 @@
+include README.md
+include agml/_assets/*.json
+include agml/_assets/shape_info.pickle
+include agml/_helios/helios_install.sh
+include agml/synthetic/synthetic_data_generation/*
+prune agml/_internal
+prune .github

AgML/source/Makefile

@@ -0,0 +1,25 @@
+
+SHELL := bash
+
+version := 0.7.3
+
+src.python := $(shell find ./agml -type f -name "*.py" || :)
+test.python := $(shell find ./tests -type f -name "*.py" || :)
+
+dist.dir := dist
+build.wheel := $(dist.dir)/agml-$(version).tar.gz
+
+
+
+.PHONY: help
+help: ## Print the help screen.
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":|:[[:space:]].*?##"}; {printf "\033[36m%-30s\033[0m %s\n", $$2, $$3}'
+
+
+$(build.wheel): $(src.python)
+	uv build -o $(dist.dir)
+
+build: $(build.wheel) ## Build the distribution wheel.
+
+test: $(test.python) $(src.python) # Run tests
+	uv run pytest -c=config/pytest.ini $(test.python)

AgML/source/README.md

@@ -0,0 +1,252 @@
+<p align="center">
+<img src="docs/assets/agml-logo.png" alt="agml logo" width="400" height="400">
+</p>
+
+----
+
+### 👨🏿‍💻👩🏽‍💻🌈🪴 Want to join the [AI Institute for Food Systems team](https://aifs.ucdavis.edu/) and help lead AgML development? 🪴🌈👩🏼‍💻👨🏻‍💻
+
+We're looking to hire a postdoc with both Python library development and ML experience. Send your resume and GitHub profile link to [jmearles@ucdavis.edu](mailto:jmearles@ucdavis.edu)!
+
+----
+
+## Overview
+AgML is a comprehensive library for agricultural machine learning. Currently, AgML provides
+access to a wealth of public agricultural datasets for common agricultural deep learning tasks. In the future, AgML will provide ag-specific ML functionality related to data, training, and evaluation. Here's a conceptual diagram of the overall framework.
+
+<p align="center">
+<img src="docs/assets/agml-framework.png" alt="agml framework" width="350" height="291">
+</p>
+
+AgML supports both the [TensorFlow](https://www.tensorflow.org/) and [PyTorch](https://pytorch.org/) machine learning frameworks.
+
+## Installation
+
+To install the latest release of AgML, run the following command:
+
+```shell
+pip install agml
+```
+
+**_NOTE:_** Some features of AgML, such as synthetic data generation, require GUI applications. When running AgML through
+Windows Subsystem for Linux (WSL), it may be necessary to configure your WSL environment to utilize these features. Please
+follow the [Microsoft documentation](https://learn.microsoft.com/en-us/windows/wsl/tutorials/gui-apps) to install all
+necessary prerequisites and update WSL. The latest version of WSL includes built-in support for running Linux GUI applications.
+
+## Quick Start
+
+AgML is designed for easy usage of agricultural data in a variety of formats. You can start off by using the `AgMLDataLoader` to
+download and load a dataset into a container:
+
+```python
+import agml
+
+loader = agml.data.AgMLDataLoader('apple_flower_segmentation')
+```
+
+You can then use the in-built processing methods to get the loader ready for your training and evaluation pipelines. This includes, but
+is not limited to, batching data, shuffling data, splitting data into training, validation, and test sets, and applying transforms.
+
+```python
+import albumentations as A
+
+# Batch the dataset into collections of 8 pieces of data:
+loader.batch(8)
+
+# Shuffle the data:
+loader.shuffle()
+
+# Apply transforms to the input images and output annotation masks:
+loader.mask_to_channel_basis()
+loader.transform(
+    transform = A.RandomContrast(),
+    dual_transform = A.Compose([A.RandomRotate90()])
+)
+
+# Split the data into train/val/test sets.
+loader.split(train = 0.8, val = 0.1, test = 0.1)
+```
+
+The split datasets can be accessed using `loader.train_data`, `loader.val_data`, and `loader.test_data`. Any further processing applied to the
+main loader will be applied to the split datasets, until the split attributes are accessed, at which point you need to apply processing independently
+to each of the loaders. You can also turn toggle processing on and off using the `loader.eval()`, `loader.reset_preprocessing()`, and `loader.disable_preprocessing()`
+methods.
+
+You can visualize data using the `agml.viz` module, which supports multiple different types of visualization for different data types:
+
+```python
+# Disable processing and batching for the test data:
+test_ds = loader.test_data
+test_ds.batch(None)
+test_ds.reset_prepreprocessing()
+
+# Visualize the image and mask side-by-side:
+agml.viz.visualize_image_and_mask(test_ds[0])
+
+# Visualize the mask overlaid onto the image:
+agml.viz.visualize_overlaid_masks(test_ds[0])
+```
+
+AgML supports both the TensorFlow and PyTorch libraries as backends, and provides functionality to export your loaders to native TensorFlow and PyTorch
+formats when you want to use them in a training pipeline. This includes both exporting the `AgMLDataLoader` to a `tf.data.Dataset` or `torch.utils.data.DataLoader`,
+but also internally converting data within the `AgMLDataLoader` itself, enabling access to its core functionality.
+
+
+```python
+# Export the loader as a `tf.data.Dataset`:
+train_ds = loader.train_data.export_tensorflow()
+
+# Convert to PyTorch tensors without exporting.
+train_ds = loader.train_data
+train_ds.as_torch_dataset()
+```
+
+You're now ready to use AgML for training your own models! Luckily, AgML comes with a training module that enables quick-start training of standard deep learning models on agricultural datasets. Training a grape detection model is as simple as the following code:
+
+```python
+import agml
+import agml.models
+
+import albumentations as A
+
+loader = agml.data.AgMLDataLoader('grape_detection_californiaday')
+loader.split(train = 0.8, val = 0.1, test = 0.1)
+processor = agml.models.preprocessing.EfficientDetPreprocessor(
+    image_size = 512, augmentation = [A.HorizontalFlip(p=0.5)]
+)
+loader.transform(processor)
+
+model = agml.models.DetectionModel(num_classes=loader.num_classes)
+
+model.run_training(loader)
+```
+
+## Public Dataset Listing
+
+AgML contains a wide variety of public datasets from various locations across the world:
+
+![AgML Dataset World Map](/docs/assets/agml_dataset_world_map.png)
+
+
+The following is a comprehensive list of all datasets available in AgML. For more information,
+you can use `agml.data.public_data_sources(...)` with various filters to filter datasets according
+to your desired specification.
+
+
+| Dataset | Task | Number of Images |
+| :--- | ---: | ---: |
+[bean_disease_uganda](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/bean_disease_uganda.md) | Image Classification | 1295 | 
+[carrot_weeds_germany](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/carrot_weeds_germany.md) | Semantic Segmentation | 60 | 
+[plant_seedlings_aarhus](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/plant_seedlings_aarhus.md) | Image Classification | 5539 | 
+[soybean_weed_uav_brazil](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/soybean_weed_uav_brazil.md) | Image Classification | 15336 | 
+[sugarcane_damage_usa](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/sugarcane_damage_usa.md) | Image Classification | 153 | 
+[crop_weeds_greece](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/crop_weeds_greece.md) | Image Classification | 508 | 
+[sugarbeet_weed_segmentation](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/sugarbeet_weed_segmentation.md) | Semantic Segmentation | 1931 | 
+[rangeland_weeds_australia](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/rangeland_weeds_australia.md) | Image Classification | 17509 | 
+[fruit_detection_worldwide](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/fruit_detection_worldwide.md) | Object Detection | 565 | 
+[leaf_counting_denmark](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/leaf_counting_denmark.md) | Image Classification | 9372 | 
+[apple_detection_usa](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/apple_detection_usa.md) | Object Detection | 2290 | 
+[mango_detection_australia](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/mango_detection_australia.md) | Object Detection | 1730 | 
+[apple_flower_segmentation](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/apple_flower_segmentation.md) | Semantic Segmentation | 148 | 
+[apple_segmentation_minnesota](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/apple_segmentation_minnesota.md) | Semantic Segmentation | 670 | 
+[rice_seedling_segmentation](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/rice_seedling_segmentation.md) | Semantic Segmentation | 224 | 
+[plant_village_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/plant_village_classification.md) | Image Classification | 55448 | 
+[autonomous_greenhouse_regression](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/autonomous_greenhouse_regression.md) | Image Regression | 389 | 
+[grape_detection_syntheticday](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/grape_detection_syntheticday.md) | Object Detection | 448 | 
+[grape_detection_californiaday](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/grape_detection_californiaday.md) | Object Detection | 126 | 
+[grape_detection_californianight](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/grape_detection_californianight.md) | Object Detection | 150 | 
+[guava_disease_pakistan](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/guava_disease_pakistan.md) | Image Classification | 306 | 
+[apple_detection_spain](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/apple_detection_spain.md) | Object Detection | 967 | 
+[apple_detection_drone_brazil](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/apple_detection_drone_brazil.md) | Object Detection | 689 | 
+[plant_doc_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/plant_doc_classification.md) | Image Classification | 2598 | 
+[plant_doc_detection](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/plant_doc_detection.md) | Object Detection | 2598 | 
+[wheat_head_counting](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/wheat_head_counting.md) | Object Detection | 6512 | 
+[peachpear_flower_segmentation](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/peachpear_flower_segmentation.md) | Semantic Segmentation | 42 | 
+[red_grapes_and_leaves_segmentation](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/red_grapes_and_leaves_segmentation.md) | Semantic Segmentation | 258 | 
+[white_grapes_and_leaves_segmentation](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/white_grapes_and_leaves_segmentation.md) | Semantic Segmentation | 273 | 
+[ghai_romaine_detection](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/ghai_romaine_detection.md) | Object Detection | 500 |
+[ghai_green_cabbage_detection](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/ghai_green_cabbage_detection.md) | Object Detection | 500 |
+[ghai_iceberg_lettuce_detection](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/ghai_iceberg_lettuce_detection.md) | Object Detection | 500 |
+[riseholme_strawberry_classification_2021](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/riseholme_strawberry_classification_2021.md) | Image Classification | 3520 |
+[ghai_broccoli_detection](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/ghai_broccoli_detection.md) | Object Detection | 500 |
+[bean_synthetic_earlygrowth_aerial](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/bean_synthetic_earlygrowth_aerial.md) | Semantic Segmentation | 2500 |
+[ghai_strawberry_fruit_detection](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/ghai_strawberry_fruit_detection.md) | Object Detection | 500 |
+[vegann_multicrop_presence_segmentation](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/vegann_multicrop_presence_segmentation.md) | Semantic Segmentation | 3775 |
+[corn_maize_leaf_disease](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/corn_maize_leaf_disease.md) | Image Classification | 4188 |
+[tomato_leaf_disease](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/tomato_leaf_disease.md) | Image Classification | 11000 |
+[vine_virus_photo_dataset](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/vine_virus_photo_dataset.md) | Image Classification | 3866 |
+[tomato_ripeness_detection](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/tomato_ripeness_detection.md) | Object Detection | 804 |
+[embrapa_wgisd_grape_detection](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/embrapa_wgisd_grape_detection.md) | Object Detection | 239 |
+[growliflower_cauliflower_segmentation](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/growliflower_cauliflower_segmentation.md) | Semantic Segmentation | 1542 |
+[strawberry_detection_2023](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/strawberry_detection_2023.md) | Object Detection | 204 |
+[strawberry_detection_2022](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/strawberry_detection_2022.md) | Object Detection | 175 |
+[almond_harvest_2021](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/almond_harvest_2021.md) | Object Detection | 50 |
+[almond_bloom_2023](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/almond_bloom_2023.md) | Object Detection | 100 |
+[gemini_flower_detection_2022](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/gemini_flower_detection_2022.md) | Object Detection | 134 |
+[gemini_leaf_detection_2022](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/gemini_leaf_detection_2022.md) | Object Detection | 25 |
+[gemini_pod_detection_2022](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/gemini_pod_detection_2022.md) | Object Detection | 98 |
+[gemini_plant_detection_2022](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/gemini_plant_detection_2022.md) | Object Detection | 402 |
+[paddy_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/paddy_disease_classification.md) | Image Classification | 10407 |
+[onion_leaf_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/onion_leaf_classification.md) | Image Classification | 4502 |
+[chilli_leaf_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/chilli_leaf_classification.md) | Image Classification | 10974 |
+[orange_leaf_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/orange_leaf_disease_classification.md) | Image Classification | 5813 |
+[papaya_leaf_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/papaya_leaf_disease_classification.md) | Image Classification | 2159 |
+[blackgram_plant_leaf_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/blackgram_plant_leaf_disease_classification.md) | Image Classification | 1007 |
+[arabica_coffee_leaf_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/arabica_coffee_leaf_disease_classification.md) | Image Classification | 58549 |
+[banana_leaf_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/banana_leaf_disease_classification.md) | Image Classification | 1288 |
+[coconut_tree_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/coconut_tree_disease_classification.md) | Image Classification | 5798 |
+[rice_leaf_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/rice_leaf_disease_classification.md) | Image Classification | 3829 |
+[tea_leaf_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/tea_leaf_disease_classification.md) | Image Classification | 5867 |
+[betel_leaf_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/betel_leaf_disease_classification.md) | Image Classification | 3589 |
+[java_plum_leaf_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/java_plum_leaf_disease_classification.md) | Image Classification | 2400 |
+[sunflower_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/sunflower_disease_classification.md) | Image Classification | 2358 |
+[cucumber_disease_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/cucumber_disease_classification.md) | Image Classification | 7689 |
+[iNatAg](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/iNatAg.md) | Image Classification | 4720903 |
+[iNatAg-mini](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/iNatAg-mini.md) | Image Classification | 560844 |
+[soybean_insect_classification](https://github.com/Project-AgML/AgML/blob/main/docs/datasets/soybean_insect_classification.md) | Image Classification | 6410 |
+
+## iNatAg and iNatAg-mini
+
+
+AgML provides an API with direct access to iNatAg (and iNatAg-mini), one of the world's largest collections of agricultural images dedicated for the task of image classification. Collectively, this dataset contains over 4 million images along with detailed species classificaations and enables access to a variety of large-scale agricultural machine learning tasks. You can instantiate the iNatAg (or iNatAg-mini, a smaller variant of iNatAg for smaller-scale applications) dataset as follows:
+
+```python
+# To select a collection of scientific family names.
+loader = agml.data.AgMLDataLoader.from_parent("iNatAg", filters={"family_name": ["...", "..."]})
+
+# To select common names.
+loader = agml.data.AgMLDataLoader.from_parent("iNatAg", filters={"common_name": "..."})
+```
+
+
+## Usage Information
+
+### Using Public Agricultural Data
+
+AgML aims to provide easy access to a range of existing public agricultural datasets The core of AgML's public data pipeline is
+[`AgMLDataLoader`](/agml/data/loader.py). You can use the `AgMLDataLoader` or `agml.data.download_public_dataset()` to download
+the dataset locally from which point it will be automatically loaded from the disk on future runs.
+From this point, the data within the loader can be split into train/val/test sets, batched, have augmentations and transforms
+applied, and be converted into a training-ready dataset (including batching, tensor conversion, and image formatting).
+
+To see the various ways in which you can use AgML datasets in your training pipelines, check out
+the [example notebook](/examples/AgML-Data.ipynb).
+
+## Annotation Formats
+
+A core aim of AgML is to provide datasets in a standardized format, enabling the synthesizing of multiple datasets
+into a single training pipeline. To this end, we provide annotations in the following formats:
+
+- **Image Classification**: Image-To-Label-Number
+- **Object Detection**: [COCO JSON](https://cocodataset.org/#format-data)
+- **Semantic Segmentation**: Dense Pixel-Wise
+
+## Contributions
+
+We welcome contributions! If you would like to contribute a new feature, fix an issue that you've noticed, or even just mention
+a bug or feature that you would like to see implemented, please don't hesitate to use the *Issues* tab to bring it to our attention.
+
+See the [contributing guidelines](/CONTRIBUTING.md) for more information.
+
+## Funding
+This project is partly funded by the [National AI Institute for Food Systems](https://aifs.ucdavis.edu).

AgML/source/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+"""
+AgML Project Package Initialization File
+"""

AgML/source/agml/__init__.py

@@ -0,0 +1,42 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.7.4"
+__all__ = ["data", "synthetic", "backend", "viz", "io"]
+
+
+# If AgML is being imported for the first time, then we need to setup
+# the module, namely prepping the config file.
+def _setup():
+    import json as _json
+    import os as _os
+
+    if not _os.path.exists(_os.path.expanduser("~/.agml")):
+        _os.makedirs(_os.path.expanduser("~/.agml"))
+        with open(_os.path.join(_os.path.expanduser("~/.agml/config.json")), "w") as f:
+            _json.dump(
+                {
+                    "data_path": _os.path.expanduser("~/.agml/datasets"),
+                    "viz_backend": "matplotlib",
+                },
+                f,
+            )
+
+
+_setup()
+del _setup  # noqa
+
+
+# There are no top-level imported functions or classes, only the modules.
+from . import backend, data, io, synthetic, viz

AgML/source/agml/_assets/detector_benchmarks.json

@@ -0,0 +1,1314 @@
+{
+    "apple_detection_drone_brazil+yolo11l": {
+        "metrics/precision(B)": 0.8104349835380228,
+        "metrics/recall(B)": 0.7988342667927443,
+        "metrics/mAP50(B)": 0.8601733668427034,
+        "metrics/mAP50-95(B)": 0.5072494132458618,
+        "fitness": 0.542541808605546,
+        "class_map": {
+            "classes": {
+                "0": "apple"
+            },
+            "maps": [
+                0.5072494132458618
+            ]
+        }
+    },
+    "apple_detection_drone_brazil+yolo11m": {
+        "metrics/precision(B)": 0.8608394830617053,
+        "metrics/recall(B)": 0.7991967871485943,
+        "metrics/mAP50(B)": 0.8992294395315695,
+        "metrics/mAP50-95(B)": 0.5134246428119603,
+        "fitness": 0.5520051224839212,
+        "class_map": {
+            "classes": {
+                "0": "apple"
+            },
+            "maps": [
+                0.5134246428119603
+            ]
+        }
+    },
+    "apple_detection_drone_brazil+yolo11n": {
+        "metrics/precision(B)": 0.8507723684861725,
+        "metrics/recall(B)": 0.8225628237831374,
+        "metrics/mAP50(B)": 0.8956789461187679,
+        "metrics/mAP50-95(B)": 0.5022398615976896,
+        "fitness": 0.5415837700497974,
+        "class_map": {
+            "classes": {
+                "0": "apple"
+            },
+            "maps": [
+                0.5022398615976896
+            ]
+        }
+    },
+    "apple_detection_drone_brazil+yolo11s": {
+        "metrics/precision(B)": 0.7951189701449942,
+        "metrics/recall(B)": 0.8106776508309076,
+        "metrics/mAP50(B)": 0.8609606179092347,
+        "metrics/mAP50-95(B)": 0.4866340253378222,
+        "fitness": 0.5240666845949634,
+        "class_map": {
+            "classes": {
+                "0": "apple"
+            },
+            "maps": [
+                0.4866340253378222
+            ]
+        }
+    },
+    "apple_detection_drone_brazil+yolo11x": {
+        "metrics/precision(B)": 0.8069007869601802,
+        "metrics/recall(B)": 0.8775295985640813,
+        "metrics/mAP50(B)": 0.910928163353364,
+        "metrics/mAP50-95(B)": 0.5166210943431347,
+        "fitness": 0.5560518012441577,
+        "class_map": {
+            "classes": {
+                "0": "apple"
+            },
+            "maps": [
+                0.5166210943431347
+            ]
+        }
+    },
+    "apple_detection_spain+yolo11l": {
+        "metrics/precision(B)": 0.895859485025347,
+        "metrics/recall(B)": 0.8555849489543671,
+        "metrics/mAP50(B)": 0.9371364896175589,
+        "metrics/mAP50-95(B)": 0.579262748646173,
+        "fitness": 0.6150501227433116,
+        "class_map": {
+            "classes": {
+                "0": "apple"
+            },
+            "maps": [
+                0.579262748646173
+            ]
+        }
+    },
+    "apple_detection_spain+yolo11m": {
+        "metrics/precision(B)": 0.886427547531193,
+        "metrics/recall(B)": 0.8813830876330876,
+        "metrics/mAP50(B)": 0.9436276131565261,
+        "metrics/mAP50-95(B)": 0.6027847361615526,
+        "fitness": 0.6368690238610499,
+        "class_map": {
+            "classes": {
+                "0": "apple"
+            },
+            "maps": [
+                0.6027847361615526
+            ]
+        }
+    },
+    "apple_detection_spain+yolo11n": {
+        "metrics/precision(B)": 0.8772810873702708,
+        "metrics/recall(B)": 0.8794449125384377,
+        "metrics/mAP50(B)": 0.9422717351592312,
+        "metrics/mAP50-95(B)": 0.5968358978532009,
+        "fitness": 0.631379481583804,
+        "class_map": {
+            "classes": {
+                "0": "apple"
+            },
+            "maps": [
+                0.5968358978532009
+            ]
+        }
+    },
+    "apple_detection_spain+yolo11s": {
+        "metrics/precision(B)": 0.9028280294123243,
+        "metrics/recall(B)": 0.8805440582444066,
+        "metrics/mAP50(B)": 0.9460649029636068,
+        "metrics/mAP50-95(B)": 0.5950217456217229,
+        "fitness": 0.6301260613559113,
+        "class_map": {
+            "classes": {
+                "0": "apple"
+            },
+            "maps": [
+                0.5950217456217229
+            ]
+        }
+    },
+    "apple_detection_spain+yolo11x": {
+        "metrics/precision(B)": 0.8854823467235927,
+        "metrics/recall(B)": 0.8588322246858833,
+        "metrics/mAP50(B)": 0.9251687653229144,
+        "metrics/mAP50-95(B)": 0.5872286398209259,
+        "fitness": 0.6210226523711248,
+        "class_map": {
+            "classes": {
+                "0": "apple"
+            },
+            "maps": [
+                0.5872286398209259
+            ]
+        }
+    },
+    "apple_detection_usa+yolo11n": {
+        "metrics/precision(B)": 0.9270838048818623,
+        "metrics/recall(B)": 0.9338242039905568,
+        "metrics/mAP50(B)": 0.9748424668962935,
+        "metrics/mAP50-95(B)": 0.7811112753679461,
+        "fitness": 0.8004843945207809,
+        "class_map": {
+            "classes": {
+                "0": "apple"
+            },
+            "maps": [
+                0.7811112753679461
+            ]
+        }
+    },
+    "apple_detection_usa+yolo11s": {
+        "metrics/precision(B)": 0.9470802220658486,
+        "metrics/recall(B)": 0.9347684279191129,
+        "metrics/mAP50(B)": 0.9798209001351015,
+        "metrics/mAP50-95(B)": 0.8303136510701614,
+        "fitness": 0.8452643759766554,
+        "class_map": {
+            "classes": {
+                "0": "apple"
+            },
+            "maps": [
+                0.8303136510701614
+            ]
+        }
+    },
+    "fruit_detection_worldwide+yolo11l": {
+        "metrics/precision(B)": 0.7840810800621841,
+        "metrics/recall(B)": 0.8191062145159602,
+        "metrics/mAP50(B)": 0.8641094337838477,
+        "metrics/mAP50-95(B)": 0.5416057642125035,
+        "fitness": 0.573856131169638,
+        "class_map": {
+            "classes": {
+                "0": "avocado",
+                "1": "rockmelon",
+                "2": "apple",
+                "3": "orange",
+                "4": "strawberry",
+                "5": "mango",
+                "6": "capsicum"
+            },
+            "maps": [
+                0.6937530110895355,
+                0.6599475916860917,
+                0.3072447491193536,
+                0.44985732917887516,
+                0.6684763555021369,
+                0.621381101506044,
+                0.3905802114054871
+            ]
+        }
+    },
+    "fruit_detection_worldwide+yolo11m": {
+        "metrics/precision(B)": 0.7699396847252584,
+        "metrics/recall(B)": 0.7049490920251014,
+        "metrics/mAP50(B)": 0.7917153211939195,
+        "metrics/mAP50-95(B)": 0.4507520456030815,
+        "fitness": 0.48484837316216534,
+        "class_map": {
+            "classes": {
+                "0": "avocado",
+                "1": "rockmelon",
+                "2": "apple",
+                "3": "orange",
+                "4": "strawberry",
+                "5": "mango",
+                "6": "capsicum"
+            },
+            "maps": [
+                0.5338804695616493,
+                0.4643126528314029,
+                0.5596974845150083,
+                0.3994619555034442,
+                0.40974001920122893,
+                0.43729794524602844,
+                0.35087379236280913
+            ]
+        }
+    },
+    "fruit_detection_worldwide+yolo11n": {
+        "metrics/precision(B)": 0.9226956515783662,
+        "metrics/recall(B)": 0.8428619967059985,
+        "metrics/mAP50(B)": 0.9282310037207211,
+        "metrics/mAP50-95(B)": 0.5653289765966472,
+        "fitness": 0.6016191793090545,
+        "class_map": {
+            "classes": {
+                "0": "avocado",
+                "1": "rockmelon",
+                "2": "apple",
+                "3": "orange",
+                "4": "strawberry",
+                "5": "mango",
+                "6": "capsicum"
+            },
+            "maps": [
+                0.5077999713770625,
+                0.7048337662337663,
+                0.6961418200702028,
+                0.5176978995737195,
+                0.5109032371347128,
+                0.5440584978198245,
+                0.47586764396724074
+            ]
+        }
+    },
+    "fruit_detection_worldwide+yolo11s": {
+        "metrics/precision(B)": 0.8749349608578184,
+        "metrics/recall(B)": 0.8491066178862222,
+        "metrics/mAP50(B)": 0.9011810080939855,
+        "metrics/mAP50-95(B)": 0.5785717028098898,
+        "fitness": 0.6108326333382994,
+        "class_map": {
+            "classes": {
+                "0": "avocado",
+                "1": "rockmelon",
+                "2": "apple",
+                "3": "orange",
+                "4": "strawberry",
+                "5": "mango",
+                "6": "capsicum"
+            },
+            "maps": [
+                0.7381820126096492,
+                0.6398140426525528,
+                0.5399714799890161,
+                0.6917575199485568,
+                0.43697906240342893,
+                0.5203213108500946,
+                0.4829764912159303
+            ]
+        }
+    },
+    "fruit_detection_worldwide+yolo11x": {
+        "metrics/precision(B)": 0.78655473099151,
+        "metrics/recall(B)": 0.7043307364199446,
+        "metrics/mAP50(B)": 0.7872813915634435,
+        "metrics/mAP50-95(B)": 0.5012090094387892,
+        "fitness": 0.5298162476512547,
+        "class_map": {
+            "classes": {
+                "0": "avocado",
+                "1": "rockmelon",
+                "2": "apple",
+                "3": "orange",
+                "4": "strawberry",
+                "5": "mango",
+                "6": "capsicum"
+            },
+            "maps": [
+                0.6725356118333934,
+                0.5808266095238095,
+                0.2792468486772157,
+                0.4817921302985327,
+                0.535893342081749,
+                0.4863169982590526,
+                0.47185152539777037
+            ]
+        }
+    },
+    "ghai_broccoli_detection+yolo11l": {
+        "metrics/precision(B)": 0.7409921665653947,
+        "metrics/recall(B)": 0.7277157702805868,
+        "metrics/mAP50(B)": 0.7428713423394218,
+        "metrics/mAP50-95(B)": 0.5635296062946676,
+        "fitness": 0.5814637798991431,
+        "class_map": {
+            "classes": {
+                "0": "canopy",
+                "1": "crown"
+            },
+            "maps": [
+                0.3223404913339708,
+                0.8047187212553645
+            ]
+        }
+    },
+    "ghai_broccoli_detection+yolo11m": {
+        "metrics/precision(B)": 0.7576714485757008,
+        "metrics/recall(B)": 0.7369239726716079,
+        "metrics/mAP50(B)": 0.7584831482913197,
+        "metrics/mAP50-95(B)": 0.5599340879416073,
+        "fitness": 0.5797889939765786,
+        "class_map": {
+            "classes": {
+                "0": "canopy",
+                "1": "crown"
+            },
+            "maps": [
+                0.3324174053329621,
+                0.7874507705502521
+            ]
+        }
+    },
+    "ghai_broccoli_detection+yolo11n": {
+        "metrics/precision(B)": 0.7377016001185307,
+        "metrics/recall(B)": 0.6524970264004617,
+        "metrics/mAP50(B)": 0.6838369761182593,
+        "metrics/mAP50-95(B)": 0.469799986880626,
+        "fitness": 0.4912036858043894,
+        "class_map": {
+            "classes": {
+                "0": "canopy",
+                "1": "crown"
+            },
+            "maps": [
+                0.23939637932948324,
+                0.7002035944317686
+            ]
+        }
+    },
+    "ghai_broccoli_detection+yolo11s": {
+        "metrics/precision(B)": 0.7329678146138658,
+        "metrics/recall(B)": 0.724735813460258,
+        "metrics/mAP50(B)": 0.7388249529707553,
+        "metrics/mAP50-95(B)": 0.5386676297103012,
+        "fitness": 0.5586833620363467,
+        "class_map": {
+            "classes": {
+                "0": "canopy",
+                "1": "crown"
+            },
+            "maps": [
+                0.3160978305795931,
+                0.7612374288410095
+            ]
+        }
+    },
+    "ghai_broccoli_detection+yolo11x": {
+        "metrics/precision(B)": 0.7672524763778574,
+        "metrics/recall(B)": 0.7342813692020289,
+        "metrics/mAP50(B)": 0.7708247492458355,
+        "metrics/mAP50-95(B)": 0.5935923090810958,
+        "fitness": 0.6113155530975698,
+        "class_map": {
+            "classes": {
+                "0": "canopy",
+                "1": "crown"
+            },
+            "maps": [
+                0.38456190936112233,
+                0.8026227088010692
+            ]
+        }
+    },
+    "ghai_green_cabbage_detection+yolo11l": {
+        "metrics/precision(B)": 0.8613838860312255,
+        "metrics/recall(B)": 0.8429330535008768,
+        "metrics/mAP50(B)": 0.9116406496856747,
+        "metrics/mAP50-95(B)": 0.7118640967709261,
+        "fitness": 0.731841752062401,
+        "class_map": {
+            "classes": {
+                "0": "green_cabbage"
+            },
+            "maps": [
+                0.7118640967709261
+            ]
+        }
+    },
+    "ghai_green_cabbage_detection+yolo11m": {
+        "metrics/precision(B)": 0.8673449730257053,
+        "metrics/recall(B)": 0.8342857142857143,
+        "metrics/mAP50(B)": 0.8907064681223883,
+        "metrics/mAP50-95(B)": 0.6944137082286287,
+        "fitness": 0.7140429842180047,
+        "class_map": {
+            "classes": {
+                "0": "green_cabbage"
+            },
+            "maps": [
+                0.6944137082286287
+            ]
+        }
+    },
+    "ghai_green_cabbage_detection+yolo11n": {
+        "metrics/precision(B)": 0.8355977884325566,
+        "metrics/recall(B)": 0.8151426059182874,
+        "metrics/mAP50(B)": 0.8691196766626649,
+        "metrics/mAP50-95(B)": 0.6591389190599897,
+        "fitness": 0.6801369948202572,
+        "class_map": {
+            "classes": {
+                "0": "green_cabbage"
+            },
+            "maps": [
+                0.6591389190599897
+            ]
+        }
+    },
+    "ghai_green_cabbage_detection+yolo11s": {
+        "metrics/precision(B)": 0.8857430201563139,
+        "metrics/recall(B)": 0.8585525715610961,
+        "metrics/mAP50(B)": 0.9102921460534825,
+        "metrics/mAP50-95(B)": 0.7139257965173013,
+        "fitness": 0.7335624314709195,
+        "class_map": {
+            "classes": {
+                "0": "green_cabbage"
+            },
+            "maps": [
+                0.7139257965173013
+            ]
+        }
+    },
+    "ghai_green_cabbage_detection+yolo11x": {
+        "metrics/precision(B)": 0.8884185216941978,
+        "metrics/recall(B)": 0.8343497989006972,
+        "metrics/mAP50(B)": 0.9109117546708069,
+        "metrics/mAP50-95(B)": 0.704366630205667,
+        "fitness": 0.725021142652181,
+        "class_map": {
+            "classes": {
+                "0": "green_cabbage"
+            },
+            "maps": [
+                0.704366630205667
+            ]
+        }
+    },
+    "ghai_iceberg_lettuce_detection+yolo11l": {
+        "metrics/precision(B)": 0.8842813590120964,
+        "metrics/recall(B)": 0.78,
+        "metrics/mAP50(B)": 0.880281737114181,
+        "metrics/mAP50-95(B)": 0.5308984079738271,
+        "fitness": 0.5658367408878625,
+        "class_map": {
+            "classes": {
+                "0": "iceberg_lettuce"
+            },
+            "maps": [
+                0.5308984079738271
+            ]
+        }
+    },
+    "ghai_iceberg_lettuce_detection+yolo11m": {
+        "metrics/precision(B)": 0.8565205192561391,
+        "metrics/recall(B)": 0.852,
+        "metrics/mAP50(B)": 0.8882337959487256,
+        "metrics/mAP50-95(B)": 0.5447845441322091,
+        "fitness": 0.5791294693138608,
+        "class_map": {
+            "classes": {
+                "0": "iceberg_lettuce"
+            },
+            "maps": [
+                0.5447845441322091
+            ]
+        }
+    },
+    "ghai_iceberg_lettuce_detection+yolo11n": {
+        "metrics/precision(B)": 0.9075705684362595,
+        "metrics/recall(B)": 0.8520705116070535,
+        "metrics/mAP50(B)": 0.8951989397183541,
+        "metrics/mAP50-95(B)": 0.5689564386149899,
+        "fitness": 0.6015806887253263,
+        "class_map": {
+            "classes": {
+                "0": "iceberg_lettuce"
+            },
+            "maps": [
+                0.5689564386149899
+            ]
+        }
+    },
+    "ghai_iceberg_lettuce_detection+yolo11s": {
+        "metrics/precision(B)": 0.8848128837651625,
+        "metrics/recall(B)": 0.7824267782426778,
+        "metrics/mAP50(B)": 0.8701965203008429,
+        "metrics/mAP50-95(B)": 0.5381306510663206,
+        "fitness": 0.5713372379897729,
+        "class_map": {
+            "classes": {
+                "0": "iceberg_lettuce"
+            },
+            "maps": [
+                0.5381306510663206
+            ]
+        }
+    },
+    "ghai_iceberg_lettuce_detection+yolo11x": {
+        "metrics/precision(B)": 0.8744060262501543,
+        "metrics/recall(B)": 0.7860519492425313,
+        "metrics/mAP50(B)": 0.8649674277006355,
+        "metrics/mAP50-95(B)": 0.5270984046669671,
+        "fitness": 0.560885306970334,
+        "class_map": {
+            "classes": {
+                "0": "iceberg_lettuce"
+            },
+            "maps": [
+                0.5270984046669671
+            ]
+        }
+    },
+    "ghai_romaine_detection+yolo11l": {
+        "metrics/precision(B)": 0.9368473966694478,
+        "metrics/recall(B)": 0.8546757717214724,
+        "metrics/mAP50(B)": 0.9193170125873877,
+        "metrics/mAP50-95(B)": 0.7699090606095494,
+        "fitness": 0.7848498558073334,
+        "class_map": {
+            "classes": {
+                "0": "romaine"
+            },
+            "maps": [
+                0.7699090606095494
+            ]
+        }
+    },
+    "ghai_romaine_detection+yolo11m": {
+        "metrics/precision(B)": 0.9313772904604823,
+        "metrics/recall(B)": 0.8590308370044053,
+        "metrics/mAP50(B)": 0.9444871705216602,
+        "metrics/mAP50-95(B)": 0.7907666278321335,
+        "fitness": 0.8061386821010862,
+        "class_map": {
+            "classes": {
+                "0": "romaine"
+            },
+            "maps": [
+                0.7907666278321335
+            ]
+        }
+    },
+    "ghai_romaine_detection+yolo11n": {
+        "metrics/precision(B)": 0.9536018902378349,
+        "metrics/recall(B)": 0.8477366255144033,
+        "metrics/mAP50(B)": 0.9274152280108815,
+        "metrics/mAP50-95(B)": 0.7745598409185308,
+        "fitness": 0.7898453796277659,
+        "class_map": {
+            "classes": {
+                "0": "romaine"
+            },
+            "maps": [
+                0.7745598409185308
+            ]
+        }
+    },
+    "ghai_romaine_detection+yolo11s": {
+        "metrics/precision(B)": 0.9421641001234269,
+        "metrics/recall(B)": 0.8611664812460607,
+        "metrics/mAP50(B)": 0.9511326333399283,
+        "metrics/mAP50-95(B)": 0.7972501823711204,
+        "fitness": 0.8126384274680013,
+        "class_map": {
+            "classes": {
+                "0": "romaine"
+            },
+            "maps": [
+                0.7972501823711204
+            ]
+        }
+    },
+    "ghai_romaine_detection+yolo11x": {
+        "metrics/precision(B)": 0.9223680838425339,
+        "metrics/recall(B)": 0.8123984591915938,
+        "metrics/mAP50(B)": 0.9086774326923178,
+        "metrics/mAP50-95(B)": 0.7400014802754252,
+        "fitness": 0.7568690755171145,
+        "class_map": {
+            "classes": {
+                "0": "romaine"
+            },
+            "maps": [
+                0.7400014802754252
+            ]
+        }
+    },
+    "ghai_strawberry_fruit_detection+yolo11l": {
+        "metrics/precision(B)": 0.6448881162125962,
+        "metrics/recall(B)": 0.49763784192355615,
+        "metrics/mAP50(B)": 0.5360449682848161,
+        "metrics/mAP50-95(B)": 0.36858859954370954,
+        "fitness": 0.3853342364178202,
+        "class_map": {
+            "classes": {
+                "0": "Bud",
+                "1": "Calyx",
+                "2": "Detached Fruit",
+                "3": "Flower",
+                "4": "Large green",
+                "5": "Leaf",
+                "6": "Ripe fruit",
+                "7": "Small Green",
+                "8": "Stem",
+                "9": "Unripe fruit"
+            },
+            "maps": [
+                0.36858859954370954,
+                0.469766081987379,
+                0.4586561965811966,
+                0.2744212082247911,
+                0.08640693226385923,
+                0.9919636363636364,
+                0.4135172471752262,
+                0.07613553965837612,
+                0.46867301006156703,
+                0.07775754357735426
+            ]
+        }
+    },
+    "ghai_strawberry_fruit_detection+yolo11m": {
+        "metrics/precision(B)": 0.5757778491011301,
+        "metrics/recall(B)": 0.4415712268148009,
+        "metrics/mAP50(B)": 0.444863565990511,
+        "metrics/mAP50-95(B)": 0.27921732332490995,
+        "fitness": 0.29578194759147003,
+        "class_map": {
+            "classes": {
+                "0": "Bud",
+                "1": "Calyx",
+                "2": "Detached Fruit",
+                "3": "Flower",
+                "4": "Large green",
+                "5": "Leaf",
+                "6": "Ripe fruit",
+                "7": "Small Green",
+                "8": "Stem",
+                "9": "Unripe fruit"
+            },
+            "maps": [
+                0.0,
+                0.5006049591619524,
+                0.23618073593073588,
+                0.31732732321969975,
+                0.04961585492434706,
+                0.9045611089249965,
+                0.2484189806795046,
+                0.06908596496122062,
+                0.4234868976489901,
+                0.042891407797652575
+            ]
+        }
+    },
+    "ghai_strawberry_fruit_detection+yolo11n": {
+        "metrics/precision(B)": 0.43024502781160584,
+        "metrics/recall(B)": 0.47812476497970713,
+        "metrics/mAP50(B)": 0.43768460776658524,
+        "metrics/mAP50-95(B)": 0.26189911059565196,
+        "fitness": 0.2794776603127453,
+        "class_map": {
+            "classes": {
+                "0": "Bud",
+                "1": "Calyx",
+                "2": "Detached Fruit",
+                "3": "Flower",
+                "4": "Large green",
+                "5": "Leaf",
+                "6": "Ripe fruit",
+                "7": "Small Green",
+                "8": "Stem",
+                "9": "Unripe fruit"
+            },
+            "maps": [
+                0.26189911059565196,
+                0.28001587526598604,
+                0.4963485268084572,
+                0.11916920292153717,
+                0.03683740274673995,
+                0.9382351146161799,
+                0.043547383404538696,
+                0.040321322331614076,
+                0.36669891550313993,
+                0.03591825176267435
+            ]
+        }
+    },
+    "ghai_strawberry_fruit_detection+yolo11s": {
+        "metrics/precision(B)": 0.5058797129220138,
+        "metrics/recall(B)": 0.3943899456162283,
+        "metrics/mAP50(B)": 0.39689324528219905,
+        "metrics/mAP50-95(B)": 0.2693731453786905,
+        "fitness": 0.28212515536904137,
+        "class_map": {
+            "classes": {
+                "0": "Bud",
+                "1": "Calyx",
+                "2": "Detached Fruit",
+                "3": "Flower",
+                "4": "Large green",
+                "5": "Leaf",
+                "6": "Ripe fruit",
+                "7": "Small Green",
+                "8": "Stem",
+                "9": "Unripe fruit"
+            },
+            "maps": [
+                0.0,
+                0.3861783354825693,
+                0.4743717311111112,
+                0.18124887953621094,
+                0.015798379333843936,
+                0.9814681574239714,
+                0.1457980084607174,
+                0.04372695172090872,
+                0.42937711916648774,
+                0.03576389155108432
+            ]
+        }
+    },
+    "ghai_strawberry_fruit_detection+yolo11x": {
+        "metrics/precision(B)": 0.5031699374002283,
+        "metrics/recall(B)": 0.4661095882326068,
+        "metrics/mAP50(B)": 0.47375836147430367,
+        "metrics/mAP50-95(B)": 0.3375521022306996,
+        "fitness": 0.35117272815506,
+        "class_map": {
+            "classes": {
+                "0": "Bud",
+                "1": "Calyx",
+                "2": "Detached Fruit",
+                "3": "Flower",
+                "4": "Large green",
+                "5": "Leaf",
+                "6": "Ripe fruit",
+                "7": "Small Green",
+                "8": "Stem",
+                "9": "Unripe fruit"
+            },
+            "maps": [
+                0.0,
+                0.5073074896278351,
+                0.6498271724137932,
+                0.3149322149563958,
+                0.055327773855347165,
+                0.9814479041406694,
+                0.19138336651215965,
+                0.08376537101164086,
+                0.5361975168123582,
+                0.05533221297679599
+            ]
+        }
+    },
+    "grape_detection_californiaday+yolo11l": {
+        "metrics/precision(B)": 0.7359297112238289,
+        "metrics/recall(B)": 0.6666666666666666,
+        "metrics/mAP50(B)": 0.7488084760197763,
+        "metrics/mAP50-95(B)": 0.4349909954379633,
+        "fitness": 0.46637274349614466,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.4349909954379633
+            ]
+        }
+    },
+    "grape_detection_californiaday+yolo11m": {
+        "metrics/precision(B)": 0.7773064938902419,
+        "metrics/recall(B)": 0.7374353226465903,
+        "metrics/mAP50(B)": 0.8077534084598061,
+        "metrics/mAP50-95(B)": 0.4722895973277059,
+        "fitness": 0.505835978440916,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.4722895973277059
+            ]
+        }
+    },
+    "grape_detection_californiaday+yolo11n": {
+        "metrics/precision(B)": 0.7763716212075286,
+        "metrics/recall(B)": 0.6999522564038694,
+        "metrics/mAP50(B)": 0.7950193029970769,
+        "metrics/mAP50-95(B)": 0.4354202972550957,
+        "fitness": 0.4713801978292938,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.4354202972550957
+            ]
+        }
+    },
+    "grape_detection_californiaday+yolo11s": {
+        "metrics/precision(B)": 0.7714644290308007,
+        "metrics/recall(B)": 0.7016129032258065,
+        "metrics/mAP50(B)": 0.7758377164119181,
+        "metrics/mAP50-95(B)": 0.4587987241064469,
+        "fitness": 0.490502623336994,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.4587987241064469
+            ]
+        }
+    },
+    "grape_detection_californiaday+yolo11x": {
+        "metrics/precision(B)": 0.7741723749893685,
+        "metrics/recall(B)": 0.7342657342657343,
+        "metrics/mAP50(B)": 0.7929902314501572,
+        "metrics/mAP50-95(B)": 0.46120749873436406,
+        "fitness": 0.4943857720059434,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.46120749873436406
+            ]
+        }
+    },
+    "grape_detection_californianight+yolo11l": {
+        "metrics/precision(B)": 0.8205106047713361,
+        "metrics/recall(B)": 0.714765100671141,
+        "metrics/mAP50(B)": 0.7789473565331466,
+        "metrics/mAP50-95(B)": 0.38727844233662967,
+        "fitness": 0.4264453337562814,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.38727844233662967
+            ]
+        }
+    },
+    "grape_detection_californianight+yolo11m": {
+        "metrics/precision(B)": 0.7498171113931226,
+        "metrics/recall(B)": 0.6684950129807627,
+        "metrics/mAP50(B)": 0.7597497390103658,
+        "metrics/mAP50-95(B)": 0.3955797325963587,
+        "fitness": 0.43199673323775944,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.3955797325963587
+            ]
+        }
+    },
+    "grape_detection_californianight+yolo11n": {
+        "metrics/precision(B)": 0.8047380714047381,
+        "metrics/recall(B)": 0.6868892702226036,
+        "metrics/mAP50(B)": 0.7710866835242444,
+        "metrics/mAP50-95(B)": 0.38862968136840154,
+        "fitness": 0.42687538158398586,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.38862968136840154
+            ]
+        }
+    },
+    "grape_detection_californianight+yolo11s": {
+        "metrics/precision(B)": 0.7567877803719415,
+        "metrics/recall(B)": 0.7690381054281749,
+        "metrics/mAP50(B)": 0.8017478762899578,
+        "metrics/mAP50-95(B)": 0.437436860925854,
+        "fitness": 0.47386796246226437,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.437436860925854
+            ]
+        }
+    },
+    "grape_detection_californianight+yolo11x": {
+        "metrics/precision(B)": 0.736821555156227,
+        "metrics/recall(B)": 0.7449392712550608,
+        "metrics/mAP50(B)": 0.7811544936516707,
+        "metrics/mAP50-95(B)": 0.45322934116927815,
+        "fitness": 0.48602185641751744,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.45322934116927815
+            ]
+        }
+    },
+    "grape_detection_syntheticday+yolo11l": {
+        "metrics/precision(B)": 0.9479649653275937,
+        "metrics/recall(B)": 0.8096817356076615,
+        "metrics/mAP50(B)": 0.9180702791053456,
+        "metrics/mAP50-95(B)": 0.720114342036841,
+        "fitness": 0.7399099357436915,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.720114342036841
+            ]
+        }
+    },
+    "grape_detection_syntheticday+yolo11m": {
+        "metrics/precision(B)": 0.9209415736623586,
+        "metrics/recall(B)": 0.7480933095611995,
+        "metrics/mAP50(B)": 0.8824423748033108,
+        "metrics/mAP50-95(B)": 0.6451196561441267,
+        "fitness": 0.6688519280100451,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.6451196561441267
+            ]
+        }
+    },
+    "grape_detection_syntheticday+yolo11n": {
+        "metrics/precision(B)": 0.967477944679693,
+        "metrics/recall(B)": 0.7243090046568308,
+        "metrics/mAP50(B)": 0.8866848206786255,
+        "metrics/mAP50-95(B)": 0.6597935907906964,
+        "fitness": 0.6824827137794894,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.6597935907906964
+            ]
+        }
+    },
+    "grape_detection_syntheticday+yolo11s": {
+        "metrics/precision(B)": 0.937753124746368,
+        "metrics/recall(B)": 0.7158730158730159,
+        "metrics/mAP50(B)": 0.8756332872383675,
+        "metrics/mAP50-95(B)": 0.6526962340923247,
+        "fitness": 0.674989939406929,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.6526962340923247
+            ]
+        }
+    },
+    "grape_detection_syntheticday+yolo11x": {
+        "metrics/precision(B)": 0.9046948366020652,
+        "metrics/recall(B)": 0.7638888888888888,
+        "metrics/mAP50(B)": 0.8939851564733988,
+        "metrics/mAP50-95(B)": 0.6670388816374231,
+        "fitness": 0.6897335091210207,
+        "class_map": {
+            "classes": {
+                "0": "grape"
+            },
+            "maps": [
+                0.6670388816374231
+            ]
+        }
+    },
+    "mango_detection_australia+yolo11n": {
+        "metrics/precision(B)": 0.9608107469068398,
+        "metrics/recall(B)": 0.9592006661115737,
+        "metrics/mAP50(B)": 0.9905041965687286,
+        "metrics/mAP50-95(B)": 0.7329004634628975,
+        "fitness": 0.7586608367734806,
+        "class_map": {
+            "classes": {
+                "0": "mango"
+            },
+            "maps": [
+                0.7329004634628975
+            ]
+        }
+    },
+    "mango_detection_australia+yolo11s": {
+        "metrics/precision(B)": 0.9621686178541151,
+        "metrics/recall(B)": 0.9601259181532005,
+        "metrics/mAP50(B)": 0.9897904628631954,
+        "metrics/mAP50-95(B)": 0.7417234866786149,
+        "fitness": 0.7665301842970729,
+        "class_map": {
+            "classes": {
+                "0": "mango"
+            },
+            "maps": [
+                0.7417234866786149
+            ]
+        }
+    },
+    "plant_doc_detection+yolo11n": {
+        "metrics/precision(B)": 0.5699978210221228,
+        "metrics/recall(B)": 0.4667694854146501,
+        "metrics/mAP50(B)": 0.49983071722743755,
+        "metrics/mAP50-95(B)": 0.28933845111216405,
+        "fitness": 0.3103876777236914,
+        "class_map": {
+            "classes": {
+                "0": "Corn leaf blight",
+                "1": "Tomato Early blight leaf",
+                "2": "Potato leaf early blight",
+                "3": "Potato leaf late blight",
+                "4": "Blueberry leaf",
+                "5": "grape leaf black rot",
+                "6": "Bell_pepper leaf spot",
+                "7": "Cherry leaf",
+                "8": "Peach leaf",
+                "9": "Soyabean leaf",
+                "10": "Strawberry leaf",
+                "11": "Apple Scab Leaf",
+                "12": "Corn rust leaf",
+                "13": "Apple leaf",
+                "14": "Corn Gray leaf spot",
+                "15": "Tomato leaf mosaic virus",
+                "16": "Tomato mold leaf",
+                "17": "Tomato leaf yellow virus",
+                "18": "Tomato leaf bacterial spot",
+                "19": "Tomato leaf late blight",
+                "20": "Squash Powdery mildew leaf",
+                "21": "Bell_pepper leaf",
+                "22": "grape leaf",
+                "23": "Apple rust leaf",
+                "24": "Tomato Septoria leaf spot",
+                "25": "Tomato leaf",
+                "26": "Raspberry leaf",
+                "27": "Potato leaf",
+                "28": "Tomato two spotted spider mites leaf"
+            },
+            "maps": [
+                0.21165168785446267,
+                0.2792325697569097,
+                0.1354302448211265,
+                0.263397483670283,
+                0.3782640229715143,
+                0.515134299139252,
+                0.3829545815204622,
+                0.28162911303808413,
+                0.407434444466772,
+                0.4101654916508175,
+                0.6115402640316101,
+                0.2757413754673899,
+                0.3832889745021061,
+                0.29309831862053365,
+                0.3417740747743542,
+                0.14864104802091133,
+                0.09251660230388258,
+                0.22793129430027714,
+                0.1096991079374952,
+                0.3384237564640334,
+                0.4487062911280722,
+                0.46780512902862303,
+                0.21608434755864292,
+                0.25096967753644134,
+                0.17234089042132714,
+                0.34210560135206436,
+                0.39394232324864203,
+                0.010912066666666666,
+                0.0
+            ]
+        }
+    },
+    "plant_doc_detection+yolo11s": {
+        "metrics/precision(B)": 0.5208979589174184,
+        "metrics/recall(B)": 0.5696899647509104,
+        "metrics/mAP50(B)": 0.5618029491774192,
+        "metrics/mAP50-95(B)": 0.3287203928993561,
+        "fitness": 0.3520286485271624,
+        "class_map": {
+            "classes": {
+                "0": "Corn leaf blight",
+                "1": "Tomato Early blight leaf",
+                "2": "Potato leaf early blight",
+                "3": "Potato leaf late blight",
+                "4": "Blueberry leaf",
+                "5": "grape leaf black rot",
+                "6": "Bell_pepper leaf spot",
+                "7": "Cherry leaf",
+                "8": "Peach leaf",
+                "9": "Soyabean leaf",
+                "10": "Strawberry leaf",
+                "11": "Apple Scab Leaf",
+                "12": "Corn rust leaf",
+                "13": "Apple leaf",
+                "14": "Corn Gray leaf spot",
+                "15": "Tomato leaf mosaic virus",
+                "16": "Tomato mold leaf",
+                "17": "Tomato leaf yellow virus",
+                "18": "Tomato leaf bacterial spot",
+                "19": "Tomato leaf late blight",
+                "20": "Squash Powdery mildew leaf",
+                "21": "Bell_pepper leaf",
+                "22": "grape leaf",
+                "23": "Apple rust leaf",
+                "24": "Tomato Septoria leaf spot",
+                "25": "Tomato leaf",
+                "26": "Raspberry leaf",
+                "27": "Potato leaf",
+                "28": "Tomato two spotted spider mites leaf"
+            },
+            "maps": [
+                0.2259487438831727,
+                0.03936366643804077,
+                0.341892241889508,
+                0.07243948582620627,
+                0.43245585022891275,
+                0.5274739759036144,
+                0.43621388674439177,
+                0.35123023909078277,
+                0.2803598679029792,
+                0.4787679330427327,
+                0.5936627836372785,
+                0.2769313350532646,
+                0.4074691558138146,
+                0.45653293038859327,
+                0.463017540106952,
+                0.29188476003679065,
+                0.13341390064905032,
+                0.1527025963641791,
+                0.21154645072713732,
+                0.32613242674334925,
+                0.3667393608026686,
+                0.483387343071833,
+                0.42529072532523193,
+                0.389782799313227,
+                0.2171715838648273,
+                0.020821773218783948,
+                0.47281725221529197,
+                0.3287203928993561,
+                0.3287203928993561
+            ]
+        }
+    },
+    "tomato_ripeness_detection+yolo11n": {
+        "metrics/precision(B)": 0.8604115397443802,
+        "metrics/recall(B)": 0.797066972977987,
+        "metrics/mAP50(B)": 0.8782874784766541,
+        "metrics/mAP50-95(B)": 0.7272230615344513,
+        "fitness": 0.7423295032286716,
+        "class_map": {
+            "classes": {
+                "0": "b_fully_ripened",
+                "1": "b_half_ripened",
+                "2": "b_green",
+                "3": "l_fully_ripened",
+                "4": "l_half_ripened",
+                "5": "l_green"
+            },
+            "maps": [
+                0.7354947193755746,
+                0.6358638892642564,
+                0.7243581208396942,
+                0.8167860030366242,
+                0.6972496245489528,
+                0.7535860121416055
+            ]
+        }
+    },
+    "tomato_ripeness_detection+yolo11s": {
+        "metrics/precision(B)": 0.8229190445960298,
+        "metrics/recall(B)": 0.8835138959024996,
+        "metrics/mAP50(B)": 0.9121278256507654,
+        "metrics/mAP50-95(B)": 0.766808600552848,
+        "fitness": 0.7813405230626399,
+        "class_map": {
+            "classes": {
+                "0": "b_fully_ripened",
+                "1": "b_half_ripened",
+                "2": "b_green",
+                "3": "l_fully_ripened",
+                "4": "l_half_ripened",
+                "5": "l_green"
+            },
+            "maps": [
+                0.7548328307989863,
+                0.7823764546071317,
+                0.8009270920266699,
+                0.7731819691183286,
+                0.7255040982562855,
+                0.7640291585096863
+            ]
+        }
+    },
+    "wheat_head_counting+yolo11l": {
+        "metrics/precision(B)": 0.9245879212203956,
+        "metrics/recall(B)": 0.8779604425952586,
+        "metrics/mAP50(B)": 0.9340138625325242,
+        "metrics/mAP50-95(B)": 0.5784247103757646,
+        "fitness": 0.6139836255914406,
+        "class_map": {
+            "classes": {
+                "0": "Wheat Head"
+            },
+            "maps": [
+                0.5784247103757646
+            ]
+        }
+    },
+    "wheat_head_counting+yolo11m": {
+        "metrics/precision(B)": 0.9183902766224068,
+        "metrics/recall(B)": 0.8840558731996816,
+        "metrics/mAP50(B)": 0.9366175742380312,
+        "metrics/mAP50-95(B)": 0.5713564029858087,
+        "fitness": 0.6078825201110309,
+        "class_map": {
+            "classes": {
+                "0": "Wheat Head"
+            },
+            "maps": [
+                0.5713564029858087
+            ]
+        }
+    },
+    "wheat_head_counting+yolo11n": {
+        "metrics/precision(B)": 0.9171613583006002,
+        "metrics/recall(B)": 0.8632682396662342,
+        "metrics/mAP50(B)": 0.9281931030761819,
+        "metrics/mAP50-95(B)": 0.5556535027937193,
+        "fitness": 0.5929074628219655,
+        "class_map": {
+            "classes": {
+                "0": "Wheat Head"
+            },
+            "maps": [
+                0.5556535027937193
+            ]
+        }
+    },
+    "wheat_head_counting+yolo11s": {
+        "metrics/precision(B)": 0.9254287063231834,
+        "metrics/recall(B)": 0.8788046106055802,
+        "metrics/mAP50(B)": 0.93777255866261,
+        "metrics/mAP50-95(B)": 0.5679245907418896,
+        "fitness": 0.6049093875339617,
+        "class_map": {
+            "classes": {
+                "0": "Wheat Head"
+            },
+            "maps": [
+                0.5679245907418896
+            ]
+        }
+    },
+    "wheat_head_counting+yolo11x": {
+        "metrics/precision(B)": 0.9223821425244931,
+        "metrics/recall(B)": 0.8769885342336543,
+        "metrics/mAP50(B)": 0.9271253514411493,
+        "metrics/mAP50-95(B)": 0.5721022258377911,
+        "fitness": 0.607604538398127,
+        "class_map": {
+            "classes": {
+                "0": "Wheat Head"
+            },
+            "maps": [
+                0.5721022258377911
+            ]
+        }
+    }
+}

AgML/source/agml/_assets/model_benchmarks.json

@@ -0,0 +1,271 @@
+{
+    "carrot_weeds_germany": {
+        "metric": {
+            "mIOU": 52.18
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "image_size": [
+                    512,
+                    512
+                ]
+            },
+            "optimizer_config": {
+                "lr": 0.005
+            }
+        }
+    },
+    "sugarbeet_weed_segmentation": {
+        "metric": {
+            "mIOU": 53.59
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "image_size": [
+                    512,
+                    512
+                ]
+            },
+            "optimizer_config": {
+                "lr": 0.005
+            }
+        }
+    },
+    "apple_flower_segmentation": {
+        "metric": {
+            "mIOU": 68.38
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "image_size": [
+                    512,
+                    512
+                ]
+            },
+            "optimizer_config": {
+                "lr": 0.005
+            }
+        }
+    },
+    "apple_segmentation_minnesota": {
+        "metric": {
+            "mIOU": 79.08
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "image_size": [
+                    512,
+                    512
+                ]
+            },
+            "optimizer_config": {
+                "lr": 0.005
+            }
+        }
+    },
+    "rice_seedling_segmentation": {
+        "metric": {
+            "mIOU": 52.2
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "image_size": [
+                    512,
+                    512
+                ]
+            },
+            "optimizer_config": {
+                "lr": 0.005
+            }
+        }
+    },
+    "peachpear_flower_segmentation": {
+        "metric": {
+            "mIOU": 72.58
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "image_size": [
+                    512,
+                    512
+                ]
+            },
+            "optimizer_config": {
+                "lr": 0.005
+            }
+        }
+    },
+    "red_grapes_and_leaves_segmentation": {
+        "metric": {
+            "mIOU": 49.18
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "image_size": [
+                    512,
+                    512
+                ]
+            },
+            "optimizer_config": {
+                "lr": 0.005
+            }
+        }
+    },
+    "white_grapes_and_leaves_segmentation": {
+        "metric": {
+            "mIOU": 51.93
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "image_size": [
+                    512,
+                    512
+                ]
+            },
+            "optimizer_config": {
+                "lr": 0.005
+            }
+        }
+    },
+    "grape_detection_californiaday": {
+        "metric": {
+            "mAP": 69.01
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "box_loss_weight": 100.0,
+                "image_size": [
+                    512,
+                    512
+                ],
+                "confidence_threshold": 0.3
+            },
+            "optimizer_config": {
+                "lr": 0.0002
+            }
+        }
+    },
+    "grape_detection_californianight": {
+        "metric": {
+            "mAP": 63.99
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "box_loss_weight": 100.0,
+                "image_size": [
+                    512,
+                    512
+                ],
+                "confidence_threshold": 0.3
+            },
+            "optimizer_config": {
+                "lr": 0.0002
+            }
+        }
+    },
+    "fruit_detection_worldwide": {
+        "metric": {
+            "mIOU": 70.35
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "box_loss_weight": 50.0,
+                "image_size": [
+                    512,
+                    512
+                ],
+                "confidence_threshold": 0.3
+            },
+            "optimizer_config": {
+                "lr": 0.0008
+            }
+        }
+    },
+    "mango_detection_australia": {
+        "metric": {
+            "mAP": 95.32
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "box_loss_weight": 100.0,
+                "image_size": [
+                    512,
+                    512
+                ],
+                "confidence_threshold": 0.3
+            },
+            "optimizer_config": {
+                "lr": 0.0002
+            }
+        }
+    },
+    "apple_detection_usa": {
+        "metric": {
+            "mAP": 94.16
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "box_loss_weight": 100.0,
+                "image_size": [
+                    512,
+                    512
+                ],
+                "confidence_threshold": 0.3
+            },
+            "optimizer_config": {
+                "lr": 0.0002
+            }
+        }
+    },
+    "apple_detection_spain": {
+        "metric": {
+            "mAP": 86.65
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "box_loss_weight": 100.0,
+                "image_size": [
+                    512,
+                    512
+                ],
+                "confidence_threshold": 0.3
+            },
+            "optimizer_config": {
+                "lr": 0.0002
+            }
+        }
+    },
+    "apple_detection_drone_brazil": {
+        "metric": {
+            "mAP": 79.62
+        },
+        "hyperparameters": {
+            "epochs": 50,
+            "model_config": {
+                "box_loss_weight": 100.0,
+                "image_size": [
+                    512,
+                    512
+                ],
+                "confidence_threshold": 0.3
+            },
+            "optimizer_config": {
+                "lr": 0.0002
+            }
+        }
+    }
+}

AgML/source/agml/_assets/public_datasources.json

@@ -0,0 +1,2506 @@
+{
+    "bean_disease_uganda": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "africa",
+            "country": "uganda"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "1295",
+        "docs_url": "https://github.com/AI-Lab-Makerere/ibean/",
+        "classes": {
+            "0": "angular_leaf_spot",
+            "1": "bean_rust",
+            "2": "healthy"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.484832227230072,
+                0.5186858773231506,
+                0.31070876121520996
+            ],
+            "std": [
+                0.18201105296611786,
+                0.1994817703962326,
+                0.16856099665164948
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "carrot_weeds_germany": {
+        "ml_task": "semantic_segmentation",
+        "ag_task": "weed_segmentation",
+        "location": {
+            "continent": "europe",
+            "country": "germany"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground_mobile",
+        "input_data_format": "png",
+        "annotation_format": "image",
+        "n_images": "60",
+        "docs_url": "https://github.com/cwfid/dataset",
+        "classes": {
+            "0": "carrot",
+            "1": "weeds"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.2736402750015259,
+                0.31067565083503723,
+                0.2736402750015259
+            ],
+            "std": [
+                0.08643443882465363,
+                0.0995175763964653,
+                0.08643443882465363
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "plant_seedlings_aarhus": {
+        "ml_task": "image_classification",
+        "ag_task": "weed_classification",
+        "location": {
+            "continent": "europe",
+            "country": "denmark"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground_fixed",
+        "input_data_format": "png",
+        "annotation_format": "directory_names",
+        "n_images": "5539",
+        "docs_url": "https://vision.eng.au.dk/plant-seedlings-dataset/",
+        "classes": {
+            "0": "maize",
+            "1": "common_wheat",
+            "2": "sugar_beet",
+            "3": "scentless_mayweed",
+            "4": "common_chickweed",
+            "5": "shepherds_purse",
+            "6": "cleavers",
+            "7": "charlock",
+            "8": "fat_hen",
+            "9": "smallflowered_cranesbill",
+            "10": "black_grass",
+            "11": "loose_silkybent"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.3287692368030548,
+                0.28884512186050415,
+                0.20677343010902405
+            ],
+            "std": [
+                0.09281572699546814,
+                0.09644099324941635,
+                0.10552676767110825
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "soybean_weed_uav_brazil": {
+        "ml_task": "image_classification",
+        "ag_task": "weed_classification",
+        "location": {
+            "continent": "south_america",
+            "country": "brazil"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "tif",
+        "annotation_format": "directory_names",
+        "n_images": "15336",
+        "docs_url": "https://data.mendeley.com/datasets/3fmjm7ncc6/2",
+        "classes": {
+            "0": "broadleaf",
+            "1": "grass",
+            "2": "soil",
+            "3": "soybean"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.32902154326438904,
+                0.33378341794013977,
+                0.1865464597940445
+            ],
+            "std": [
+                0.24514122307300568,
+                0.2491522580385208,
+                0.1570599377155304
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "sugarcane_damage_usa": {
+        "ml_task": "image_classification",
+        "ag_task": "damage_classification",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "platform": "ground_fixed",
+        "input_data_format": "bmp",
+        "annotation_format": "directory_names",
+        "n_images": "153",
+        "docs_url": "https://github.com/The77Lab/SugarcaneBilletsDataset",
+        "classes": {
+            "0": "cracked",
+            "1": "crushed",
+            "2": "no_buds",
+            "3": "two_buds",
+            "4": "single_damaged_buds",
+            "5": "no_damage"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.24388393759727478,
+                0.24691258370876312,
+                0.26203399896621704
+            ],
+            "std": [
+                0.22363615036010742,
+                0.21735334396362305,
+                0.1923859715461731
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "crop_weeds_greece": {
+        "ml_task": "image_classification",
+        "ag_task": "weed_classification",
+        "location": {
+            "continent": "europe",
+            "country": "greece"
+        },
+        "sensor_modality": "rgb",
+        "platform": "handheld",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "508",
+        "docs_url": "https://github.com/AUAgroup/early-crop-weed",
+        "classes": {
+            "0": "black_nightshade",
+            "1": "velvet_leaf",
+            "2": "cotton",
+            "3": "tomato"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.7247000336647034,
+                0.6694437265396118,
+                0.5436030030250549
+            ],
+            "std": [
+                0.16351649165153503,
+                0.15307274460792542,
+                0.13750970363616943
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "sugarbeet_weed_segmentation": {
+        "ml_task": "semantic_segmentation",
+        "ag_task": "weed_segmentation",
+        "location": {
+            "continent": "none",
+            "country": "none"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "png",
+        "annotation_format": "image",
+        "n_images": "125",
+        "docs_url": "https://github.com/inkyusa/weedNet",
+        "stats": {
+            "mean": [
+                0.374398797750473,
+                0.374398797750473,
+                0.374398797750473
+            ],
+            "std": [
+                0.16996806859970093,
+                0.16996806859970093,
+                0.16996806859970093
+            ]
+        },
+        "classes": {
+            "0": "sugar_beet",
+            "1": "random_weeds"
+        },
+        "external_image_sources": [],
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "rangeland_weeds_australia": {
+        "ml_task": "image_classification",
+        "ag_task": "weed_classification",
+        "location": {
+            "continent": "oceania",
+            "country": "australia"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "17509",
+        "docs_url": "https://github.com/AlexOlsen/DeepWeeds",
+        "classes": {
+            "0": "chinee_apple",
+            "1": "lantana",
+            "2": "negative",
+            "3": "parkinsonia",
+            "4": "parthenium",
+            "5": "prickly_acacia",
+            "6": "rubber_vine",
+            "7": "siam_weed",
+            "8": "snake_weed"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.3785800635814667,
+                0.38957375288009644,
+                0.3797687590122223
+            ],
+            "std": [
+                0.22421929240226746,
+                0.22450383007526398,
+                0.22274655103683472
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "fruit_detection_worldwide": {
+        "ml_task": "object_detection",
+        "ag_task": "fruit_detection",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "mixed",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "565",
+        "docs_url": "https://drive.google.com/drive/folders/1CmsZb1caggLRN7ANfika8WuPiywo4mBb",
+        "stats": {
+            "mean": [
+                0.3779049813747406,
+                0.4085286557674408,
+                0.28682708740234375
+            ],
+            "std": [
+                0.2306382656097412,
+                0.22500208020210266,
+                0.2183498740196228
+            ]
+        },
+        "classes": {
+            "1": "avocado",
+            "2": "rockmelon",
+            "3": "apple",
+            "4": "orange",
+            "5": "strawberry",
+            "6": "mango",
+            "7": "capsicum"
+        },
+        "external_image_sources": [],
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "leaf_counting_denmark": {
+        "ml_task": "image_classification",
+        "ag_task": "leaf_counting",
+        "location": {
+            "continent": "europe",
+            "country": "denmark"
+        },
+        "platform": "mixed",
+        "input_data_format": "png",
+        "annotation_format": "directory_names",
+        "n_images": "9372",
+        "docs_url": "https://vision.eng.au.dk/leaf-counting-dataset/",
+        "classes": {
+            "0": "1",
+            "1": "2",
+            "2": "3",
+            "3": "4",
+            "4": "5",
+            "5": "6",
+            "6": "7",
+            "7": "8",
+            "8": "9+"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.42661556601524353,
+                0.427418053150177,
+                0.3417948782444
+            ],
+            "std": [
+                0.12296482920646667,
+                0.14348869025707245,
+                0.1148977279663086
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "apple_detection_usa": {
+        "ml_task": "object_detection",
+        "ag_task": "fruit_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "png",
+        "annotation_format": "coco_json",
+        "n_images": "2290",
+        "docs_url": "https://hdl.handle.net/2376/17721",
+        "stats": {
+            "mean": [
+                0.2810896933078766,
+                0.29005560278892517,
+                0.2775411605834961
+            ],
+            "std": [
+                0.18863406777381897,
+                0.18647761642932892,
+                0.1885077804327011
+            ]
+        },
+        "classes": {
+            "1": "apple"
+        },
+        "external_image_sources": [],
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "mango_detection_australia": {
+        "ml_task": "object_detection",
+        "ag_task": "fruit_detection",
+        "location": {
+            "continent": "oceania",
+            "country": "australia"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "1242",
+        "docs_url": "https://researchdata.edu.au/mangoyolo-set/1697505",
+        "stats": {
+            "mean": [
+                0.07288186252117157,
+                0.1193767637014389,
+                0.050045181065797806
+            ],
+            "std": [
+                0.058673080056905746,
+                0.07924807816743851,
+                0.043408144265413284
+            ]
+        },
+        "classes": {
+            "1": "mango"
+        },
+        "external_image_sources": [],
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "tomato_ripeness_detection": {
+        "ml_task": "object_detection",
+        "ag_task": "fruit_ripeness_detection",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "804",
+        "docs_url": "https://www.kaggle.com/datasets/nexuswho/laboro-tomato",
+        "classes": {
+            "1": "b_fully_ripened",
+            "2": "b_half_ripened",
+            "3": "b_green",
+            "4": "l_fully_ripened",
+            "5": "l_half_ripened",
+            "6": "l_green"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.46756285429000854,
+                0.5048850178718567,
+                0.317764014005661
+            ],
+            "std": [
+                0.21034087240695953,
+                0.19280721247196198,
+                0.22368815541267395
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "apple_flower_segmentation": {
+        "ml_task": "semantic_segmentation",
+        "ag_task": "flower_segmentation",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "image",
+        "n_images": "148",
+        "docs_url": "https://data.nal.usda.gov/dataset/data-multi-species-fruit-flower-detection-using-refined-semantic-segmentation-network",
+        "stats": {
+            "mean": [
+                0.4580204486846924,
+                0.47888585925102234,
+                0.4053090512752533
+            ],
+            "std": [
+                0.2040298879146576,
+                0.19258278608322144,
+                0.2156316637992859
+            ]
+        },
+        "classes": {
+            "0": "apple"
+        },
+        "external_image_sources": [],
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "apple_segmentation_minnesota": {
+        "ml_task": "semantic_segmentation",
+        "ag_task": "fruit_segmentation",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "png",
+        "annotation_format": "image",
+        "n_images": "670",
+        "docs_url": "https://rsn.umn.edu/projects/orchard-monitoring/minneapple",
+        "stats": {
+            "mean": [
+                0.3736153841018677,
+                0.43716397881507874,
+                0.33839237689971924
+            ],
+            "std": [
+                0.2627662420272827,
+                0.28767505288124084,
+                0.33136168122291565
+            ]
+        },
+        "classes": {
+            "0": "apple"
+        },
+        "external_image_sources": [],
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "rice_seedling_segmentation": {
+        "ml_task": "semantic_segmentation",
+        "ag_task": "weed_segmentation",
+        "location": {
+            "continent": "asia",
+            "country": "china"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld",
+        "input_data_format": "jpg",
+        "annotation_format": "image",
+        "n_images": "224",
+        "docs_url": "https://github.com/kabbas570/CED-Net-Crops-and-Weeds-Segmentation-for-Smart-Farming-Using-a-Small-Cascaded-Encoder-Decoder-Archi",
+        "stats": {
+            "mean": [
+                0.6552301049232483,
+                0.7014582753181458,
+                0.6353084444999695
+            ],
+            "std": [
+                0.08477462828159332,
+                0.1075846329331398,
+                0.11058590561151505
+            ]
+        },
+        "classes": {
+            "0": "Rice",
+            "1": "Weed"
+        },
+        "external_image_sources": [],
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "plant_village_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "55448",
+        "docs_url": "https://github.com/spMohanty/PlantVillage-Dataset",
+        "classes": {
+            "0": "Apple___Apple_scab",
+            "1": "Apple___Black_rot",
+            "2": "Apple___Cedar_apple_rust",
+            "3": "Apple___healthy",
+            "4": "Background_without_leaves",
+            "5": "Blueberry___healthy",
+            "6": "Cherry___Powdery_mildew",
+            "7": "Cherry___healthy",
+            "8": "Corn___Cercospora_leaf_spot Gray_leaf_spot",
+            "9": "Corn___Common_rust",
+            "10": "Corn___Northern_Leaf_Blight",
+            "11": "Corn___healthy",
+            "12": "Grape___Black_rot",
+            "13": "Grape___Esca_(Black_Measles)",
+            "14": "Grape___Leaf_blight_(Isariopsis_Leaf_Spot)",
+            "15": "Grape___healthy",
+            "16": "Orange___Haunglongbing_(Citrus_greening)",
+            "17": "Peach___Bacterial_spot",
+            "18": "Peach___healthy",
+            "19": "Pepper,_bell___Bacterial_spot",
+            "20": "Pepper,_bell___healthy",
+            "21": "Potato___Early_blight",
+            "22": "Potato___Late_blight",
+            "23": "Potato___healthy",
+            "24": "Raspberry___healthy",
+            "25": "Soybean___healthy",
+            "26": "Squash___Powdery_mildew",
+            "27": "Strawberry___Leaf_scorch",
+            "28": "Strawberry___healthy",
+            "29": "Tomato___Bacterial_spot",
+            "30": "Tomato___Early_blight",
+            "31": "Tomato___Late_blight",
+            "32": "Tomato___Leaf_Mold",
+            "33": "Tomato___Septoria_leaf_spot",
+            "34": "Tomato___Spider_mites Two-spotted_spider_mite",
+            "35": "Tomato___Target_Spot",
+            "36": "Tomato___Tomato_Yellow_Leaf_Curl_Virus",
+            "37": "Tomato___Tomato_mosaic_virus",
+            "38": "Tomato___healthy"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.46678444743156433,
+                0.4892062544822693,
+                0.4119776785373688
+            ],
+            "std": [
+                0.17720915377140045,
+                0.15169931948184967,
+                0.19375933706760406
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "autonomous_greenhouse_regression": {
+        "ml_task": "image_regression",
+        "ag_task": "ag_regression",
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "png",
+        "annotation_format": "json",
+        "n_images": "389",
+        "docs_url": "https://data.4tu.nl/articles/dataset/3rd_Autonomous_Greenhouse_Challenge_Online_Challenge_Lettuce_Images/15023088/1",
+        "location": {
+            "continent": "europe",
+            "country": "netherlands"
+        },
+        "classes": {
+            "regression": {
+                "0": "FreshWeightShoot",
+                "1": "DryWeightShoot",
+                "2": "Height",
+                "3": "Diameter",
+                "4": "LeafArea"
+            },
+            "classification": {
+                "0": "Aphylion",
+                "1": "Lugano",
+                "2": "Salanova",
+                "3": "Satine"
+            }
+        },
+        "external_image_sources": [
+            "depth"
+        ],
+        "stats": {
+            "mean": [
+                0.4848844110965729,
+                0.45935600996017456,
+                0.4063475430011749
+            ],
+            "std": [
+                0.13606780767440796,
+                0.13940763473510742,
+                0.19247490167617798
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "grape_detection_syntheticday": {
+        "classes": {
+            "1": "grape"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "fruit_detection",
+        "location": {
+            "continent": "digital",
+            "country": "digital"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "synthetic",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "448",
+        "docs_url": "N/A",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.24965472519397736,
+                0.2763761281967163,
+                0.20683138072490692
+            ],
+            "std": [
+                0.15098054707050323,
+                0.18630442023277283,
+                0.2241816222667694
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "grape_detection_californiaday": {
+        "classes": {
+            "1": "grape"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "fruit_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "image",
+        "n_images": "126",
+        "docs_url": "",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.4455319046974182,
+                0.4638977646827698,
+                0.27496835589408875
+            ],
+            "std": [
+                0.2677213251590729,
+                0.25918272137641907,
+                0.2625304162502289
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "grape_detection_californianight": {
+        "classes": {
+            "1": "grape"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "fruit_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "image",
+        "n_images": "150",
+        "docs_url": "",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.22235676646232605,
+                0.21305303275585175,
+                0.22475562989711761
+            ],
+            "std": [
+                0.20471057295799255,
+                0.20105913281440735,
+                0.20582404732704163
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "guava_disease_pakistan": {
+        "classes": {
+            "0": "Canker",
+            "1": "Dot",
+            "2": "Mummification",
+            "3": "Rust"
+        },
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "asia",
+            "country": "pakistan"
+        },
+        "sensor_modality": "rgb",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "306",
+        "docs_url": "https://data.mendeley.com/datasets/s8x6jn5cvr/1",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.44554802775382996,
+                0.45087501406669617,
+                0.32288917899131775
+            ],
+            "std": [
+                0.18661096692085266,
+                0.18359650671482086,
+                0.18744416534900665
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "apple_detection_spain": {
+        "classes": {
+            "1": "apple"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "fruit_detection",
+        "location": {
+            "continent": "europe",
+            "country": "spain"
+        },
+        "sensor_modality": "rgb",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "967",
+        "docs_url": "https://www.grap.udl.cat/en/publications/KFuji_RGBDS_database.html",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.3745550215244293,
+                0.4699203670024872,
+                0.3931747078895569
+            ],
+            "std": [
+                0.2724320888519287,
+                0.27973315119743347,
+                0.2829023003578186
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "apple_detection_drone_brazil": {
+        "classes": {
+            "1": "apple"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "fruit_detection",
+        "location": {
+            "continent": "south_america",
+            "country": "brazil"
+        },
+        "sensor_modality": "rgb",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "689",
+        "docs_url": "https://github.com/thsant/add256/tree/zenodo-1.0",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.3362397849559784,
+                0.48177945613861084,
+                0.32026395201683044
+            ],
+            "std": [
+                0.18435567617416382,
+                0.1873108297586441,
+                0.1734534204006195
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "plant_doc_classification": {
+        "classes": {
+            "0": "Corn leaf blight",
+            "1": "Tomato Early blight leaf",
+            "2": "Potato leaf early blight",
+            "3": "Potato leaf late blight",
+            "4": "Blueberry leaf",
+            "5": "grape leaf black rot",
+            "6": "Bell_pepper leaf spot",
+            "7": "Cherry leaf",
+            "8": "Peach leaf",
+            "9": "Soyabean leaf",
+            "10": "Strawberry leaf",
+            "11": "Apple Scab Leaf",
+            "12": "Corn rust leaf",
+            "13": "Apple leaf",
+            "14": "Corn Gray leaf spot",
+            "15": "Tomato leaf mosaic virus",
+            "16": "Tomato mold leaf",
+            "17": "Tomato leaf yellow virus",
+            "18": "Tomato leaf bacterial spot",
+            "19": "Tomato leaf late blight",
+            "20": "Squash Powdery mildew leaf",
+            "21": "Bell_pepper leaf",
+            "22": "grape leaf",
+            "23": "Apple rust leaf",
+            "24": "Tomato Septoria leaf spot",
+            "25": "Tomato leaf",
+            "26": "Raspberry leaf",
+            "27": "Tomato two spotted spider mites leaf"
+        },
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "2598",
+        "docs_url": "https://github.com/pratikkayal/PlantDoc-Dataset",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.48221418261528015,
+                0.549874484539032,
+                0.3833079934120178
+            ],
+            "std": [
+                0.21240688860416412,
+                0.2036021202802658,
+                0.22107268869876862
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "plant_doc_detection": {
+        "classes": {
+            "1": "Corn leaf blight",
+            "2": "Tomato Early blight leaf",
+            "3": "Potato leaf early blight",
+            "4": "Potato leaf late blight",
+            "5": "Blueberry leaf",
+            "6": "grape leaf black rot",
+            "7": "Bell_pepper leaf spot",
+            "8": "Cherry leaf",
+            "9": "Peach leaf",
+            "10": "Soyabean leaf",
+            "11": "Strawberry leaf",
+            "12": "Apple Scab Leaf",
+            "13": "Corn rust leaf",
+            "14": "Apple leaf",
+            "15": "Corn Gray leaf spot",
+            "16": "Tomato leaf mosaic virus",
+            "17": "Tomato mold leaf",
+            "18": "Tomato leaf yellow virus",
+            "19": "Tomato leaf bacterial spot",
+            "20": "Tomato leaf late blight",
+            "21": "Squash Powdery mildew leaf",
+            "22": "Bell_pepper leaf",
+            "23": "grape leaf",
+            "24": "Apple rust leaf",
+            "25": "Tomato Septoria leaf spot",
+            "26": "Tomato leaf",
+            "27": "Raspberry leaf",
+            "28": "Potato leaf",
+            "29": "Tomato two spotted spider mites leaf"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "disease_detection",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "2346",
+        "docs_url": "https://github.com/pratikkayal/PlantDoc-Object-Detection-Dataset",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.4765770733356476,
+                0.5437355637550354,
+                0.3779796361923218
+            ],
+            "std": [
+                0.21106934547424316,
+                0.2036709189414978,
+                0.21807792782783508
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "wheat_head_counting": {
+        "classes": {
+            "1": "Wheat Head"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "wheat_head_counting",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "6512",
+        "docs_url": "https://zenodo.org/record/5092309",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.371184766292572,
+                0.36074304580688477,
+                0.23014496266841888
+            ],
+            "std": [
+                0.2190379649400711,
+                0.21708978712558746,
+                0.17700935900211334
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "peachpear_flower_segmentation": {
+        "classes": {
+            "1": "Flower"
+        },
+        "ml_task": "semantic_segmentation",
+        "ag_task": "flower_segmentation",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": "jpg",
+        "annotation_format": "image",
+        "n_images": "42",
+        "docs_url": "https://data.nal.usda.gov/dataset/data-multi-species-fruit-flower-detection-using-refined-semantic-segmentation-network",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.4377596378326416,
+                0.4347001910209656,
+                0.4147627353668213
+            ],
+            "std": [
+                0.20811232924461365,
+                0.2141699343919754,
+                0.2441396266222
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "red_grapes_and_leaves_segmentation": {
+        "classes": {
+            "1": "Leaf",
+            "2": "Grape"
+        },
+        "ml_task": "semantic_segmentation",
+        "ag_task": "vineyard_scene_segmentation",
+        "location": {
+            "continent": "europe",
+            "country": "greece"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": "jpg",
+        "annotation_format": "image",
+        "n_images": "258",
+        "docs_url": "https://link.springer.com/chapter/10.1007/978-3-030-48791-1_22",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.392019122838974,
+                0.49319183826446533,
+                0.5254659056663513
+            ],
+            "std": [
+                0.2721235156059265,
+                0.2871488034725189,
+                0.2831265926361084
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "white_grapes_and_leaves_segmentation": {
+        "classes": {
+            "1": "Leaf",
+            "2": "Grape"
+        },
+        "ml_task": "semantic_segmentation",
+        "ag_task": "vineyard_scene_segmentation",
+        "location": {
+            "continent": "europe",
+            "country": "greece"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": "jpg",
+        "annotation_format": "image",
+        "n_images": "273",
+        "docs_url": "https://link.springer.com/chapter/10.1007/978-3-030-48791-1_22",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.30906906723976135,
+                0.47296029329299927,
+                0.5238653421401978
+            ],
+            "std": [
+                0.23329640924930573,
+                0.24588093161582947,
+                0.25335168838500977
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "ghai_romaine_detection": {
+        "classes": {
+            "1": "romaine"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "crop_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "500",
+        "docs_url": "https://github.com/AxisAg/GHAIDatasets/blob/main/datasets/romaine.md",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.3551516532897949,
+                0.4719887971878052,
+                0.33378446102142334
+            ],
+            "std": [
+                0.253168523311615,
+                0.26515713334083557,
+                0.2393222600221634
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "ghai_green_cabbage_detection": {
+        "classes": {
+            "1": "green_cabbage"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "crop_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "500",
+        "docs_url": "https://github.com/AxisAg/GHAIDatasets/blob/main/datasets/green_cabbage.md",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.4649180769920349,
+                0.49569201469421387,
+                0.46829652786254883
+            ],
+            "std": [
+                0.28846681118011475,
+                0.27976617217063904,
+                0.276553213596344
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "ghai_iceberg_lettuce_detection": {
+        "classes": {
+            "1": "iceberg_lettuce"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "crop_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "500",
+        "docs_url": "https://github.com/AxisAg/GHAIDatasets/blob/main/datasets/iceberg.md",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.3193310499191284,
+                0.4143226742744446,
+                0.3003973960876465
+            ],
+            "std": [
+                0.2828165590763092,
+                0.2841264605522156,
+                0.25660744309425354
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "riseholme_strawberry_classification_2021": {
+        "classes": {
+            "0": "anomalous",
+            "1": "occluded",
+            "2": "ripe",
+            "3": "unripe"
+        },
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "input_data_format": "png",
+        "annotation_format": "directory_names",
+        "n_images": "3520",
+        "docs_url": "https://github.com/ctyeong/Riseholme-2021",
+        "location": {
+            "continent": "europe",
+            "country": "uk"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.5753851532936096,
+                0.46378713846206665,
+                0.36370429396629333
+            ],
+            "std": [
+                0.19837889075279236,
+                0.19222107529640198,
+                0.1773509532213211
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "ghai_broccoli_detection": {
+        "classes": {
+            "1": "canopy",
+            "2": "crown"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "crop_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "500",
+        "docs_url": "https://github.com/AxisAg/GHAIDatasets/blob/main/datasets/broccoli.md",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.42832377552986145,
+                0.45015057921409607,
+                0.43816232681274414
+            ],
+            "std": [
+                0.24546495079994202,
+                0.23975253105163574,
+                0.23829665780067444
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "bean_synthetic_earlygrowth_aerial": {
+        "classes": {
+            "1": "leaves",
+            "2": "branches"
+        },
+        "ml_task": "semantic_segmentation",
+        "ag_task": "plant_segmentation",
+        "location": {
+            "continent": "digital",
+            "country": "digital"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "synthetic",
+        "platform": "aerial",
+        "input_data_format": "jpg",
+        "annotation_format": "image",
+        "n_images": "2500",
+        "docs_url": "https://github.com/Project-AgML/AgML/blob/main/docs/datasets/bean_synthetic_earlygrowth_aerial.md",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.24804304540157318,
+                0.2218194305896759,
+                0.16611839830875397
+            ],
+            "std": [
+                0.10789012163877487,
+                0.12054657936096191,
+                0.08992248773574829
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "ghai_strawberry_fruit_detection": {
+        "classes": {
+            "1": "Bud",
+            "2": "Calyx",
+            "3": "Detached Fruit",
+            "4": "Flower",
+            "5": "Large green",
+            "6": "Leaf",
+            "7": "Ripe fruit",
+            "8": "Small Green",
+            "9": "Stem",
+            "10": "Unripe fruit"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "crop_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "500",
+        "docs_url": "https://github.com/AxisAg/GHAIDatasets/blob/main/datasets/strawberry.md",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.49159616231918335,
+                0.5238277316093445,
+                0.4485996663570404
+            ],
+            "std": [
+                0.18163496255874634,
+                0.16137710213661194,
+                0.18042609095573425
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "vegann_multicrop_presence_segmentation": {
+        "classes": {
+            "1": "plant"
+        },
+        "ml_task": "semantic_segmentation",
+        "ag_task": "vegetation_segmentation",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "aerial",
+        "input_data_format": "png",
+        "annotation_format": "image",
+        "n_images": "3775",
+        "docs_url": "https://zenodo.org/records/7636408",
+        "stats": {
+            "mean": [
+                16.391704559326172,
+                18.685382843017578,
+                14.235113143920898
+            ],
+            "std": [
+                4.5903120040893555,
+                5.818264007568359,
+                4.189508438110352
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "corn_maize_leaf_disease": {
+        "classes": {
+            "0": "Blight",
+            "1": "Common_Rust",
+            "2": "Gray_Leaf_Spot",
+            "3": "Healthy"
+        },
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld",
+        "input_data_format": [
+            "jpg",
+            "png",
+            "jpeg",
+            "JPG"
+        ],
+        "annotation_format": "directory_names",
+        "n_images": "4188",
+        "docs_url": "https://www.kaggle.com/datasets/smaranjitghose/corn-or-maize-leaf-disease-dataset/data",
+        "stats": {
+            "mean": [
+                0.4375976622104645,
+                0.49800175428390503,
+                0.3756123483181
+            ],
+            "std": [
+                0.17942173779010773,
+                0.1671891063451767,
+                0.17422494292259216
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "tomato_leaf_disease": {
+        "classes": {
+            "0": "Bacterial Spot",
+            "1": "Early Blight",
+            "2": "Healthy",
+            "3": "Late Blight",
+            "4": "Leaf Mold",
+            "5": "Septoria Leaf Spot",
+            "6": "Spider Mites Two-spotted Spider Mite",
+            "7": "Target Spot",
+            "8": "Tomato Mosaic Virus",
+            "9": "Tomato Yellow Leaf Curl Virus"
+        },
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld",
+        "input_data_format": [
+            "jpeg"
+        ],
+        "annotation_format": "directory_names",
+        "n_images": "11000",
+        "docs_url": "https://www.kaggle.com/datasets/kaustubhb999/tomatoleaf?resource=download",
+        "stats": {
+            "mean": [
+                0.4527473449707031,
+                0.4633065462112427,
+                0.41868826746940613
+            ],
+            "std": [
+                0.16920849680900574,
+                0.1477183699607849,
+                0.18538573384284973
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "vine_virus_photo_dataset": {
+        "classes": {
+            "0": "Leafroll 3",
+            "1": "No Virus",
+            "2": "Other Red",
+            "3": "Red Blotch"
+        },
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": [
+            "jpg",
+            "png",
+            "jpeg",
+            "JPG"
+        ],
+        "annotation_format": "directory_names",
+        "n_images": "3866",
+        "docs_url": "",
+        "stats": {
+            "mean": [
+                0.44989314675331116,
+                0.4489104151725769,
+                0.29500243067741394
+            ],
+            "std": [
+                0.2310592532157898,
+                0.2280450314283371,
+                0.23016904294490814
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "embrapa_wgisd_grape_detection": {
+        "classes": {
+            "0": "Chardonnay",
+            "1": "PinotGris",
+            "2": "PinotNoir"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "fruit_detection",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": "jpg",
+        "annotation_format": "coco_json",
+        "n_images": "239",
+        "docs_url": "https://github.com/thsant/wgisd/tree/master",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.5015993118286133,
+                0.5361655354499817,
+                0.4303760826587677
+            ],
+            "std": [
+                0.2352389246225357,
+                0.2119981199502945,
+                0.2218495011329651
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "growliflower_cauliflower_segmentation": {
+        "classes": {
+            "1": "maskLeaves",
+            "2": "maskPlants",
+            "3": "maskStems",
+            "4": "maskVoid"
+        },
+        "ml_task": "semantic_segmentation",
+        "ag_task": "crop_segmentation",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "input_data_format": "jpg",
+        "annotation_format": "image",
+        "n_images": "1542",
+        "docs_url": "https://phenoroam.phenorob.de/geonetwork/srv/eng/catalog.search#/metadata/cb328232-31f5-4b84-a929-8e1ee551d66a",
+        "stats": {
+            "mean": [
+                0.48514866828918457,
+                0.4264565408229828,
+                0.33096641302108765
+            ],
+            "std": [
+                0.12302214652299881,
+                0.1260126680135727,
+                0.11423511058092117
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "strawberry_detection_2023": {
+        "classes": {
+            "0": "strawberry",
+            "1": "flower",
+            "2": "green",
+            "3": "large_white",
+            "4": "pink",
+            "5": "red",
+            "6": "small_white"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "fruit_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "annotation_format": "coco_json",
+        "input_data_format": "jpg",
+        "n_images": "204",
+        "docs_url": "https://universe.roboflow.com/strawberrydet/strawberry_2023/3",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.2719569802284241,
+                0.2755374312400818,
+                0.2003764510154724
+            ],
+            "std": [
+                0.14823083579540253,
+                0.1477075070142746,
+                0.14832836389541626
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "strawberry_detection_2022": {
+        "classes": {
+            "0": "strawberry",
+            "1": "flower",
+            "2": "green",
+            "3": "large_white",
+            "4": "pink",
+            "5": "red",
+            "6": "small_white"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "fruit_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "handheld/ground",
+        "annotation_format": "coco_json",
+        "input_data_format": "jpg",
+        "n_images": "175",
+        "docs_url": "https://universe.roboflow.com/strawberrydet/strawberry_2022/dataset/4",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.287511944770813,
+                0.3060769736766815,
+                0.29882678389549255
+            ],
+            "std": [
+                0.21001286804676056,
+                0.19917000830173492,
+                0.20504336059093475
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "almond_harvest_2021": {
+        "classes": {
+            "0": "Visible Fruit/Flower",
+            "1": "Partially Visible Fruit/Flower",
+            "2": "Visible Occluded Fruit/Flower",
+            "3": "Partially Visible Occluded Fruit/Flower"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "fruit_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "annotation_format": "coco_json",
+        "input_data_format": "jpg",
+        "n_images": "50",
+        "docs_url": "",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.1509094387292862,
+                0.15222010016441345,
+                0.14836227893829346
+            ],
+            "std": [
+                0.16955046355724335,
+                0.1702313870191574,
+                0.1667901575565338
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "almond_bloom_2023": {
+        "classes": {
+            "0": "Visible Fruit/Flower"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "flower_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "annotation_format": "coco_json",
+        "input_data_format": "jpg",
+        "n_images": "100",
+        "docs_url": "",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.32803982496261597,
+                0.34916892647743225,
+                0.3486247956752777
+            ],
+            "std": [
+                0.23713527619838715,
+                0.24729833006858826,
+                0.23989765346050262
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "gemini_flower_detection_2022": {
+        "classes": {
+            "1": "object"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "flower_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "annotation_format": "coco_json",
+        "input_data_format": "jpg",
+        "n_images": "134",
+        "docs_url": "http://gemini-breeding.github.io/",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.43233251571655273,
+                0.46158429980278015,
+                0.3074590265750885
+            ],
+            "std": [
+                0.15803606808185577,
+                0.1518802046775818,
+                0.1447635143995285
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "gemini_leaf_detection_2022": {
+        "classes": {
+            "1": "object"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "leaf_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "annotation_format": "coco_json",
+        "input_data_format": "jpg",
+        "n_images": "25",
+        "docs_url": "http://gemini-breeding.github.io/",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.5152409672737122,
+                0.48080527782440186,
+                0.4170548915863037
+            ],
+            "std": [
+                0.17544656991958618,
+                0.1531846821308136,
+                0.13160519301891327
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "gemini_pod_detection_2022": {
+        "classes": {
+            "1": "object"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "pod_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "annotation_format": "coco_json",
+        "input_data_format": "jpg",
+        "n_images": "98",
+        "docs_url": "http://gemini-breeding.github.io/",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.4158003032207489,
+                0.47277751564979553,
+                0.28544333577156067
+            ],
+            "std": [
+                0.16562676429748535,
+                0.1610204428434372,
+                0.1648048311471939
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "gemini_plant_detection_2022": {
+        "classes": {
+            "1": "plant",
+            "2": "weed"
+        },
+        "ml_task": "object_detection",
+        "ag_task": "plant_detection",
+        "location": {
+            "continent": "north_america",
+            "country": "usa"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "ground",
+        "annotation_format": "coco_json",
+        "input_data_format": "jpg",
+        "n_images": "402",
+        "docs_url": "http://gemini-breeding.github.io/",
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.5137307047843933,
+                0.47968026995658875,
+                0.4163574278354645
+            ],
+            "std": [
+                0.17421172559261322,
+                0.15248580276966095,
+                0.13018731772899628
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "paddy_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "asia",
+            "country": "india"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "10407",
+        "docs_url": "https://www.kaggle.com/competitions/paddy-disease-classification/data",
+        "classes": {
+            "0": "bacterial_leaf_blight",
+            "1": "bacterial_leaf_streak",
+            "2": "bacterial_panicle_blight",
+            "3": "blast",
+            "4": "brown_spot",
+            "5": "dead_heart",
+            "6": "downy_mildew",
+            "7": "hispa",
+            "8": "normal",
+            "9": "tungro"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.49716252088546753,
+                0.5882205963134766,
+                0.2304224669933319
+            ],
+            "std": [
+                0.23286384344100952,
+                0.23301087319850922,
+                0.1900700032711029
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "onion_leaf_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "leaf_classification",
+        "location": {
+            "continent": "asia",
+            "country": "india"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "4502",
+        "docs_url": "https://www.researchgate.net/publication/380611658_Dataset_of_Chilli_and_Onion_Plant_Leaf_Images_for_Classification_and_Detection",
+        "classes": {
+            "0": "healthy",
+            "1": "iris_yellow_virus",
+            "2": "leaf_blight",
+            "3": "purple_blotch"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.49259838461875916,
+                0.5055249929428101,
+                0.33485615253448486
+            ],
+            "std": [
+                0.18601194024085999,
+                0.14629681408405304,
+                0.15651942789554596
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "chilli_leaf_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "leaf_classification",
+        "location": {
+            "continent": "asia",
+            "country": "india"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "10974",
+        "docs_url": "https://www.researchgate.net/publication/380611658_Dataset_of_Chilli_and_Onion_Plant_Leaf_Images_for_Classification_and_Detection",
+        "classes": {
+            "0": "cercospora",
+            "1": "healthy",
+            "2": "mites_and_trips",
+            "3": "nutritional",
+            "4": "powdery_mildew"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.5395056009292603,
+                0.5616798996925354,
+                0.4004570543766022
+            ],
+            "std": [
+                0.17784084379673004,
+                0.1770888715982437,
+                0.18177233636379242
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "orange_leaf_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "asia",
+            "country": "bangladesh"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpeg",
+        "annotation_format": "directory_names",
+        "n_images": "5813",
+        "docs_url": "https://www.sciencedirect.com/science/article/pii/S2352340924006802#sec0004",
+        "classes": {
+            "0": "citrus_canker",
+            "1": "citrus_greening",
+            "2": "citrus_mealybugs",
+            "3": "die_back",
+            "4": "foliage_damaged",
+            "5": "healthy_leaf",
+            "6": "powdery_mildew",
+            "7": "shot_hole",
+            "8": "spiny_whitefly",
+            "9": "yellow_dragon",
+            "10": "yellow_leaves"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.641664981842041,
+                0.645190954208374,
+                0.5412733554840088
+            ],
+            "std": [
+                0.21496868133544922,
+                0.19639450311660767,
+                0.2651123106479645
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "papaya_leaf_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "asia",
+            "country": "bangladesh"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "2159",
+        "docs_url": "https://www.sciencedirect.com/science/article/pii/S2352340924008734",
+        "classes": {
+            "0": "Anthracnose",
+            "1": "BacterialSpot",
+            "2": "Curl",
+            "3": "Healthy",
+            "4": "RingSpot"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.4037705361843109,
+                0.5427719950675964,
+                0.29178404808044434
+            ],
+            "std": [
+                0.1690509021282196,
+                0.15967564284801483,
+                0.14206627011299133
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "blackgram_plant_leaf_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "asia",
+            "country": "india"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "1007",
+        "docs_url": "https://www.sciencedirect.com/science/article/pii/S2352340922009295",
+        "classes": {
+            "0": "anthracnose",
+            "1": "healthy",
+            "2": "leaf_crinckle",
+            "3": "powdery_mildew",
+            "4": "yellow_mosaic"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.4454658031463623,
+                0.5082071423530579,
+                0.3293055593967438
+            ],
+            "std": [
+                0.19813409447669983,
+                0.1792915314435959,
+                0.16840000450611115
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "arabica_coffee_leaf_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "africa",
+            "country": "kenya"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": [
+            "jpg",
+            "jpeg"
+        ],
+        "annotation_format": "directory_names",
+        "n_images": "58549",
+        "docs_url": "https://www.sciencedirect.com/science/article/pii/S2352340921004261?via%3Dihub#sec0001",
+        "classes": {
+            "0": "Cerscospora",
+            "1": "Healthy",
+            "2": "Leaf_rust",
+            "3": "Miner",
+            "4": "Phoma"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.4663342535495758,
+                0.6128407716751099,
+                0.40549468994140625
+            ],
+            "std": [
+                0.1319405734539032,
+                0.10137982666492462,
+                0.13884715735912323
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "banana_leaf_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "africa",
+            "country": "ethiopia"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "1288",
+        "docs_url": "https://www.researchgate.net/publication/380900090_Sigatoka_and_Xanthomonas_Banana_Leaf_Disease_Detection_Via_Transfer_Learning",
+        "classes": {
+            "0": "healthy",
+            "1": "segatoka",
+            "2": "xamthomonas"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.4403895437717438,
+                0.4788191020488739,
+                0.22997669875621796
+            ],
+            "std": [
+                0.2052917778491974,
+                0.20397576689720154,
+                0.18596293032169342
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "coconut_tree_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "asia",
+            "country": "india"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "5798",
+        "docs_url": "https://www.sciencedirect.com/science/article/pii/S2352340923007692#sec0003",
+        "classes": {
+            "0": "Bud_Root_Dropping",
+            "1": "Bud_Rot",
+            "2": "Gray_Leaf_Spot",
+            "3": "Leaf_Rot",
+            "4": "Stem_Bleeding"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.49172350764274597,
+                0.49519237875938416,
+                0.4346628189086914
+            ],
+            "std": [
+                0.2121831327676773,
+                0.20774872601032257,
+                0.22041980922222137
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "rice_leaf_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "worldwide",
+            "country": "worldwide"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "3829",
+        "docs_url": "https://www.kaggle.com/datasets/anshulm257/rice-disease-dataset/data",
+        "classes": {
+            "0": "Bacterial_Leaf_Blight",
+            "1": "Brown_Spot",
+            "2": "Healthy_Rice_Leaf",
+            "3": "Leaf_Blast",
+            "4": "Leaf_Scald",
+            "5": "Sheath_Blight"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.3801848888397217,
+                0.5112283229827881,
+                0.24517497420310974
+            ],
+            "std": [
+                0.1857737898826599,
+                0.1747269332408905,
+                0.13470299541950226
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "tea_leaf_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "asia",
+            "country": "india"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "5867",
+        "docs_url": "https://www.kaggle.com/datasets/saikatdatta1994/tea-leaf-disease",
+        "classes": {
+            "0": "algal_spot",
+            "1": "brown_blight",
+            "2": "gray_blight",
+            "3": "healthy",
+            "4": "helopeltis",
+            "5": "red_spot"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.720391035079956,
+                0.7148104310035706,
+                0.7214117646217346
+            ],
+            "std": [
+                0.2642996311187744,
+                0.24272359907627106,
+                0.32473450899124146
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "betel_leaf_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "asia",
+            "country": "bangladesh"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "3589",
+        "docs_url": "https://www.semanticscholar.org/paper/Betel-Leaf-Diseases-Classification-using-Machine-A-David-Mukunthan/38208c9d2306444e3b9f8593715a46e2dcf26f44#paper-topics",
+        "classes": {
+            "0": "Bacterial_Leaf_Disease",
+            "1": "Dried_Leaf",
+            "2": "Fungal_Brown_Spot_Disease",
+            "3": "Healthy_Leaf"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.44302278757095337,
+                0.5203492045402527,
+                0.32380297780036926
+            ],
+            "std": [
+                0.18618987500667572,
+                0.19697120785713196,
+                0.21828916668891907
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "java_plum_leaf_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "asia",
+            "country": "bangladesh"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "2400",
+        "docs_url": "https://www.sciencedirect.com/science/article/pii/S2772375524001059#sec0003",
+        "classes": {
+            "0": "Bacterial_Spot",
+            "1": "Brown_Blight",
+            "2": "Dry",
+            "3": "Healthy",
+            "4": "Powdery_Mildew",
+            "5": "Sooty_Mold"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.6888663172721863,
+                0.7057309746742249,
+                0.6495749950408936
+            ],
+            "std": [
+                0.21865394711494446,
+                0.20790858566761017,
+                0.3019416928291321
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "sunflower_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "asia",
+            "country": "bangladesh"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "2358",
+        "docs_url": "https://www.sciencedirect.com/science/article/pii/S2352340922002542",
+        "classes": {
+            "0": "Downy_mildew",
+            "1": "Fresh_leaf",
+            "2": "Gray_mold",
+            "3": "Leaf_scars"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.45603200793266296,
+                0.45415663719177246,
+                0.2436216175556183
+            ],
+            "std": [
+                0.22495068609714508,
+                0.21411556005477905,
+                0.17952653765678406
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "cucumber_disease_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "disease_classification",
+        "location": {
+            "continent": "asia",
+            "country": "bangladesh"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "7689",
+        "docs_url": "https://www.sciencedirect.com/science/article/pii/S2352340923004389",
+        "classes": {
+            "0": "Anthracnose",
+            "1": "Bacterial_Wilt",
+            "2": "Belly_Rot",
+            "3": "Downy_Mildew",
+            "4": "Fresh_Cucumber",
+            "5": "Fresh_Leaf",
+            "6": "Gummy_Stem_Blight",
+            "7": "Pythium_Fruit_Rot"
+        },
+        "external_image_sources": [],
+        "stats": {
+            "mean": [
+                0.46584174036979675,
+                0.5267260074615479,
+                0.3983174264431
+            ],
+            "std": [
+                0.19197316467761993,
+                0.18358547985553741,
+                0.17932511866092682
+            ]
+        },
+        "parent_dataset": "",
+        "extra_metadata": {}
+    },
+    "soybean_insect_classification": {
+        "ml_task": "image_classification",
+        "ag_task": "pest_classification",
+        "location": {
+            "continent": "south_america",
+            "country": "brazil"
+        },
+        "sensor_modality": "rgb",
+        "real_synthetic": "real",
+        "platform": "uav",
+        "input_data_format": "jpg",
+        "annotation_format": "directory_names",
+        "n_images": "6410",
+        "docs_url": "https://data.mendeley.com/datasets/bycbh73438/1",
+        "classes": {
+            "0": "Caterpillar",
+            "1": "Diabrotica_speciosa",
+            "2": "Healthy"
+        },
+        "external_image_sources": [],
+        "parent_dataset": "",
+        "extra_metadata": {},
+        "stats": {
+            "mean": [
+                0.4129256308078766,
+                0.515846848487854,
+                0.3347107470035553
+            ],
+            "std": [
+                0.22007715702056885,
+                0.22993944585323334,
+                0.20302338898181915
+            ]
+        }
+    }
+}

AgML/source/agml/_assets/shape_info.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:483dfc00dac1e09bc963e6b2c9ae30556c60425164b1a3bb9929e4452d8b3229
+size 499283

AgML/source/agml/_assets/source_citations.json

@@ -0,0 +1,270 @@
+{
+    "bean_disease_uganda": {
+        "license": "MIT",
+        "citation": ""
+    },
+    "carrot_weeds_germany": {
+        "license": "",
+        "citation": "@inproceedings{haug15,\n  author={Haug, Sebastian and Ostermann, J{\\\"o}rn},\n  title={A Crop/Weed Field Image Dataset for the Evaluation of Computer Vision Based Precision Agriculture Tasks},\n  year={2015},\n  booktitle={Computer Vision - ECCV 2014 Workshops},\n  doi={10.1007/978-3-319-16220-1_8},\n  url={http://dx.doi.org/10.1007/978-3-319-16220-1_8},\n  pages={105--116}\n}"
+    },
+    "plant_seedlings_aarhus": {
+        "license": "CC BY-SA 4.0",
+        "citation": "@article{Giselsson2017,\n  author = {Giselsson, Thomas Mosgaard and Dyrmann, Mads and J{\\o}rgensen, Rasmus Nyholm and Jensen, Peter Kryger and Midtiby, Henrik Skov},\n  journal = {arXiv preprint},\n  keywords = {benchmark,database,plant seedlings,segmentation,site-specific weed control},\n  title = {{A Public Image Database for Benchmark of Plant Seedling Classification Algorithms}},\n  year = {2017}\n}"
+    },
+    "soybean_weed_uav_brazil": {
+        "license": "CC BY-NC 3.0",
+        "citation": "dos Santos Ferreira, Alessandro; Pistori, Hemerson; Matte Freitas, Daniel; Gon\u00e7alves da Silva, Gercina (2017), \u201cData for: Weed Detection in Soybean Crops Using ConvNets\u201d, Mendeley Data, V2, doi: 10.17632/3fmjm7ncc6.2"
+    },
+    "sugarcane_damage_usa": {
+        "license": "",
+        "citation": "@ARTICLE{8412587,\n  author={Alencastre-Miranda, Moises and Davidson, Joseph R. and Johnson, Richard M. and Waguespack, Herman and Krebs, Hermano Igo},\n  journal={IEEE Robotics and Automation Letters}, \n  title={Robotics for Sugarcane Cultivation: Analysis of Billet Quality using Computer Vision}, \n  year={2018},\n  volume={3},\n  number={4},\n  pages={3828-3835},\n  doi={10.1109/LRA.2018.2856999}}"
+    },
+    "crop_weeds_greece": {
+        "license": "MIT",
+        "citation": "@article{ESPEJOGARCIA2020105306,\n  title = {Towards weeds identification assistance through transfer learning},\n  journal = {Computers and Electronics in Agriculture},\n  volume = {171},\n  pages = {105306},\n  year = {2020},\n  issn = {0168-1699},\n  doi = {https://doi.org/10.1016/j.compag.2020.105306},\n  url = {https://www.sciencedirect.com/science/article/pii/S0168169919319854},\n  author = {Borja Espejo-Garcia and Nikos Mylonas and Loukas Athanasakos and Spyros Fountas and Ioannis Vasilakoglou},\n  keywords = {Weed identification, Deep learning, Transfer learning, Open data, Precision agriculture},\n  abstract = {Reducing the use of pesticides through selective spraying is an important component towards a more sustainable computer-assisted agriculture. Weed identification at early growth stage contributes to reduced herbicide rates. However, while computer vision alongside deep learning have overcome the performance of approaches that use hand-crafted features, there are still some open challenges in the development of a reliable automatic plant identification system. These type of systems have to take into account different sources of variability, such as growth stages and soil conditions, with the added constraint of the limited size of usual datasets. This study proposes a novel crop/weed identification system that relies on a combination of fine-tuning pre-trained convolutional networks (Xception, Inception-Resnet, VGNets, Mobilenet and Densenet) with the \u201ctraditional\u201d machine learning classifiers (Support Vector Machines, XGBoost and Logistic Regression) trained with the previously deep extracted features. The aim of this approach was to avoid overfitting and to obtain a robust and consistent performance. To evaluate this approach, an open access dataset of two crop [tomato (Solanum lycopersicum L.) and cotton (Gossypium hirsutum L.)] and two weed species [black nightshade (Solanum nigrum L.) and velvetleaf (Abutilon theophrasti Medik.)] was generated. The pictures were taken by different production sites across Greece under natural variable light conditions from RGB cameras. The results revealed that a combination of fine-tuned Densenet and Support Vector Machine achieved a micro F1 score of 99.29% with a very low performance difference between train and test sets. Other evaluated approaches also obtained repeatedly more than 95% F1 score. Additionally, our results analysis provides some heuristics for designing transfer-learning based systems to avoid overfitting without decreasing performance.}\n}"
+    },
+    "sugarbeet_weed_segmentation": {
+        "license": "GPL-3.0",
+        "citation": "@ARTICLE{8115245,\n  author={I. Sa and Z. Chen and M. Popovi\u0107 and R. Khanna and F. Liebisch and J. Nieto and R. Siegwart},\n  journal={IEEE Robotics and Automation Letters},\n  title={weedNet: Dense Semantic Weed Classification Using Multispectral Images and MAV for Smart Farming},\n  year={2018},\n  volume={3},\n  number={1},\n  pages={588-595},\n  keywords={agriculture;agrochemicals;autonomous aerial vehicles;control engineering computing;convolution;crops;feature extraction;image classification;learning (artificial intelligence);neural nets;vegetation;MAV;SegNet;convolutional neural network;crop health;crop management;curve classification metrics;dense semantic classes;dense semantic weed classification;encoder-decoder;input image channels;multispectral images;selective weed treatment;vegetation index;weed detection;Agriculture;Cameras;Image segmentation;Robots;Semantics;Training;Vegetation mapping;Aerial systems;agricultural automation;applications;robotics in agriculture and forestry},\n  doi={10.1109/LRA.2017.2774979},\n  ISSN={},\n  month={Jan}\n}"
+    },
+    "rangeland_weeds_australia": {
+        "license": "CC BY-SA 4.0",
+        "citation": "@Article{Olsen2019,\n  author={Olsen, Alex and Konovalov, Dmitry A. and Philippa, Bronson and Ridd, Peter and Wood, Jake C. and Johns, Jamie and Banks, Wesley and Girgenti, Benjamin and Kenny, Owen and Whinney, James and Calvert, Brendan and Azghadi, Mostafa Rahimi and White, Ronald D.},\n  title={DeepWeeds: A Multiclass Weed Species Image Dataset for Deep Learning},\n  journal={Scientific Reports},\n  year={2019},\n  month={Feb},\n  day={14},\n  volume={9},\n  number={1},\n  pages={2058},\n  abstract={Robotic weed control has seen increased research of late with its potential for boosting productivity in agriculture. Majority of works focus on developing robotics for croplands, ignoring the weed management problems facing rangeland stock farmers. Perhaps the greatest obstacle to widespread uptake of robotic weed control is the robust classification of weed species in their natural environment. The unparalleled successes of deep learning make it an ideal candidate for recognising various weed species in the complex rangeland environment. This work contributes the first large, public, multiclass image dataset of weed species from the Australian rangelands; allowing for the development of robust classification methods to make robotic weed control viable. The DeepWeeds dataset consists of 17,509 labelled images of eight nationally significant weed species native to eight locations across northern Australia. This paper presents a baseline for classification performance on the dataset using the benchmark deep learning models, Inception-v3 and ResNet-50. These models achieved an average classification accuracy of 95.1{\\%} and 95.7{\\%}, respectively. We also demonstrate real time performance of the ResNet-50 architecture, with an average inference time of 53.4 ms per image. These strong results bode well for future field implementation of robotic weed control methods in the Australian rangelands.},\n  issn={2045-2322},\n  doi={10.1038/s41598-018-38343-3},\n  url={https://doi.org/10.1038/s41598-018-38343-3}\n}"
+    },
+    "fruit_detection_worldwide": {
+        "license": "",
+        "citation": "@Article{s16081222,\n  AUTHOR = {Sa, Inkyu and Ge, Zongyuan and Dayoub, Feras and Upcroft, Ben and Perez, Tristan and McCool, Chris},\n  TITLE = {DeepFruits: A Fruit Detection System Using Deep Neural Networks},\n  JOURNAL = {Sensors},\n  VOLUME = {16},\n  YEAR = {2016},\n  NUMBER = {8},\n  ARTICLE-NUMBER = {1222},\n  URL = {https://www.mdpi.com/1424-8220/16/8/1222},\n  ISSN = {1424-8220},\n  ABSTRACT = {This paper presents a novel approach to fruit detection using deep convolutional neural networks. The aim is to build an accurate, fast and reliable fruit detection system, which is a vital element of an autonomous agricultural robotic platform; it is a key element for fruit yield estimation and automated harvesting. Recent work in deep neural networks has led to the development of a state-of-the-art object detector termed Faster Region-based CNN (Faster R-CNN). We adapt this model, through transfer learning, for the task of fruit detection using imagery obtained from two modalities: colour (RGB) and Near-Infrared (NIR). Early and late fusion methods are explored for combining the multi-modal (RGB and NIR) information. This leads to a novel multi-modal Faster R-CNN model, which achieves state-of-the-art results compared to prior work with the F1 score, which takes into account both precision and recall performances improving from     0 . 807     to     0 . 838     for the detection of sweet pepper. In addition to improved accuracy, this approach is also much quicker to deploy for new fruits, as it requires bounding box annotation rather than pixel-level annotation (annotating bounding boxes is approximately an order of magnitude quicker to perform). The model is retrained to perform the detection of seven fruits, with the entire process taking four hours to annotate and train the new model per fruit.},\n  DOI = {10.3390/s16081222}\n}"
+    },
+    "leaf_counting_denmark": {
+        "license": "CC BY-SA 4.0",
+        "citation": "@Article{s18051580,\n  author = {Teimouri, Nima and Dyrmann, Mads and Nielsen, Per  Rydahl and Mathiassen, Solvejg  Kopp and Somerville, Gayle  J. and J\u00f8rgensen, Rasmus  Nyholm},\n  title = {Weed Growth Stage Estimator Using Deep Convolutional Neural Networks},\n  journal = {Sensors},\n  volume = {18},\n  year = {2018},\n  number = {5},\n  url = {http://www.mdpi.com/1424-8220/18/5/1580},\n  issn = {1424-8220}\n}"
+    },
+    "apple_detection_usa": {
+        "license": "",
+        "citation": "@article{karkee2019apple,\n  title={Apple Dataset Benchmark from Orchard Environment in Modern Fruiting Wall},\n  author={Karkee, Manoj and Bhusal, Santosh and Zhang, Qin},\n  year={2019}\n}"
+    },
+    "mango_detection_australia": {
+        "license": "",
+        "citation": "@Misc{Koirala2019,\n  author={Koirala, Anand and Walsh, Kerry and Wang, Z. and McCarthy, C.},\n  title={MangoYOLO data set},\n  year={2019},\n  month={2021},\n  day={10-19},\n  publisher={Central Queensland University},\n  keywords={Mango images; Fruit detection; Yield estimation; Mango; Agricultural Land Management; Horticultural Crop Growth and Development},\n  abstract={Datasets and directories are structured similar to the PASCAL VOC dataset, avoiding the need to change scripts already available, with the detection frameworks ready to parse PASCAL VOC annotations into their format. The sub-directory JPEGImages consist of 1730 images (612x512 pixels) used for train, test and validation. Each image has at least one annotated fruit. The sub-directory Annotations consists of all the annotation files (record of bounding box coordinates for each image) in xml format and have the same name as the image name. The sub-directory Main consists of the text file that contains image names (without extension) used for train, test and validation. Training set (train.txt) lists 1300 train images  Validation set (val.txt) lists 130 validation images Test set (test.txt) lists 300 test images Each image has an XML annotation file (filename = image name) and each image set (training validation and test set) has associated text files (train.txt, val.txt and test.txt) containing the list of image names to be used for training and testing.  The XML annotation file contains the image attributes (name, width, height), the object attributes (class name, object bounding box co-ordinates (xmin, ymin, xmax, ymax)). (xmin, ymin) and (xmax, ymax) are the pixel co-ordinates of the bounding box's top-left corner and bottom-right corner respectively.},\n  note={CC-BY-4.0},\n  url={https://figshare.com/articles/dataset/MangoYOLO_data_set/13450661, https://researchdata.edu.au/mangoyolo-set},\n  language={English}\n}"
+    },
+    "tomato_ripeness_detection": {
+        "license": "CC BY-NC-SA 4.0",
+        "citation": ""
+    },
+    "apple_flower_segmentation": {
+        "license": "US Public Domain",
+        "citation": "@ARTICLE{8392727,\n  author={Dias, Philipe A. and Tabb, Amy and Medeiros, Henry},\n  journal={IEEE Robotics and Automation Letters}, \n  title={Multispecies Fruit Flower Detection Using a Refined Semantic Segmentation Network}, \n  year={2018},\n  volume={3},\n  number={4},\n  pages={3003-3010},\n  doi={10.1109/LRA.2018.2849498}}"
+    },
+    "apple_segmentation_minnesota": {
+        "license": "MIT",
+        "citation": "@misc{hani2019minneapple,\n    title={MinneApple: A Benchmark Dataset for Apple Detection and Segmentation},\n    author={Nicolai H\u00e4ni and Pravakar Roy and Volkan Isler}\n    year={2019},\n    eprint={1909.06441},\n    archivePrefix={arXiv},\n    primaryClass={cs.CV}\n}"
+    },
+    "rice_seedling_segmentation": {
+        "license": "",
+        "citation": "@Article{electronics9101602,\n  AUTHOR = {Khan, Abbas and Ilyas, Talha and Umraiz, Muhammad and Mannan, Zubaer Ibna and Kim, Hyongsuk},\n  TITLE = {CED-Net: Crops and Weeds Segmentation for Smart Farming Using a Small Cascaded Encoder-Decoder Architecture},\n  JOURNAL = {Electronics},\n  VOLUME = {9},\n  YEAR = {2020},\n  NUMBER = {10},\n  ARTICLE-NUMBER = {1602},\n  URL = {https://www.mdpi.com/2079-9292/9/10/1602},\n  ISSN = {2079-9292},\n  ABSTRACT = {Convolutional neural networks (CNNs) have achieved state-of-the-art performance in numerous aspects of human life and the agricultural sector is no exception. One of the main objectives of deep learning for smart farming is to identify the precise location of weeds and crops on farmland. In this paper, we propose a semantic segmentation method based on a cascaded encoder-decoder network, namely CED-Net, to differentiate weeds from crops. The existing architectures for weeds and crops segmentation are quite deep, with millions of parameters that require longer training time. To overcome such limitations, we propose an idea of training small networks in cascade to obtain coarse-to-fine predictions, which are then combined to produce the final results. Evaluation of the proposed network and comparison with other state-of-the-art networks are conducted using four publicly available datasets: rice seeding and weed dataset, BoniRob dataset, carrot crop vs. weed dataset, and a paddy–millet dataset. The experimental results and their comparisons proclaim that the proposed network outperforms state-of-the-art architectures, such as U-Net, SegNet, FCN-8s, and DeepLabv3, over intersection over union (IoU), F1-score, sensitivity, true detection rate, and average precision comparison metrics by utilizing only (1/5.74 × U-Net), (1/5.77 × SegNet), (1/3.04 × FCN-8s), and (1/3.24 × DeepLabv3) fractions of total parameters.},\n  DOI = {10.3390/electronics9101602}\n}"
+    },
+    "plant_village_classification": {
+        "license": "",
+        "citation": "@article{DBLP:journals/corr/HughesS15,\n  author    = {David P. Hughes and\n               Marcel Salath{'{e} } },\n  title     = {An open access repository of images on plant health to enable the\n               development of mobile disease diagnostics through machine\n               learning and crowdsourcing},\n  journal   = {CoRR},\n  volume    = {abs/1511.08060},\n  year      = {2015},\n  url       = {http://arxiv.org/abs/1511.08060},\n  archivePrefix = {arXiv},\n  eprint    = {1511.08060},\n  timestamp = {Mon, 13 Aug 2018 16:48:21 +0200},\n  biburl    = {https://dblp.org/rec/bib/journals/corr/HughesS15},\n  bibsource = {dblp computer science bibliography, https://dblp.org}\n}"
+    },
+    "plant_doc_classification": {
+        "license": "CC BY-SA 4.0",
+        "citation": "@inproceedings{10.1145/3371158.3371196,\n  author = {Singh, Davinder and Jain, Naman and Jain, Pranjali and Kayal, Pratik and Kumawat, Sudhakar and Batra, Nipun},\n  title = {PlantDoc: A Dataset for Visual Plant Disease Detection},\n  year = {2020},\n  isbn = {9781450377386},\n  publisher = {Association for Computing Machinery},\n  address = {New York, NY, USA},\n  url = {https://doi.org/10.1145/3371158.3371196},\n  doi = {10.1145/3371158.3371196},\n  booktitle = {Proceedings of the 7th ACM IKDD CoDS and 25th COMAD},\n  pages = {249–253},\n  numpages = {5},\n  keywords = {Deep Learning, Object Detection, Image Classification},\n  location = {Hyderabad, India},\n  series = {CoDS COMAD 2020}\n  }"
+    },
+    "autonomous_greenhouse_regression": {
+        "license": "CC BY-SA 4.0",
+        "citation": "@misc{https://doi.org/10.4121/15023088.v1,\n  doi = {10.4121/15023088.V1},\n  url = {https://data.4tu.nl/articles/_/15023088/1},\n  author = {Hemming,  S. (Silke) and de Zwart,  H.F. (Feije) and Elings,  A. (Anne) and bijlaard,  monique and Marrewijk,  van,  Bart and Petropoulou,  Anna},\n  keywords = {Horticultural Crops,  Mechanical Engineering,  FOS: Mechanical engineering,  Artificial Intelligence and Image Processing,  FOS: Computer and information sciences,  Horticultural Production,  FOS: Agriculture,  forestry and fisheries,  Autonomous Greenhouse Challenge,  autonomous greenhouse,  Artificial Intelligence,  image processing,  computer vision,  Horticulture,  Lettuce,  sensors,  non-destructive sensing},\n  title = {3rd Autonomous Greenhouse Challenge: Online Challenge Lettuce Images},\n  publisher = {4TU.ResearchData},\n  year = {2021},\n  copyright = {Creative Commons Attribution 4.0 International}\n}"
+    },
+    "grape_detection_syntheticday": {
+        "license": "",
+        "citation": "@ARTICLE{10.3389/fpls.2019.01185,\n  \nAUTHOR={Bailey, Brian N.},   \n\t \nTITLE={Helios: A Scalable 3D Plant and Environmental Biophysical Modeling Framework},      \n\t\nJOURNAL={Frontiers in Plant Science},      \n\t\nVOLUME={10},      \n\t\nYEAR={2019},      \n\t  \nURL={https://www.frontiersin.org/article/10.3389/fpls.2019.01185},       \n\t\nDOI={10.3389/fpls.2019.01185},      \n\t\nISSN={1664-462X},   \n   \nABSTRACT={This article presents an overview of Helios, a new three-dimensional (3D) plant and environmental modeling framework. Helios is a model coupling framework designed to provide maximum flexibility in integrating and running arbitrary 3D environmental system models. Users interact with Helios through a well-documented open-source C++ API. Version 1.0 comes with model plug-ins for radiation transport, the surface energy balance, stomatal conductance, photosynthesis, solar position, and procedural tree generation. Additional plug-ins are also available for visualizing model geometry and data and for processing and integrating LiDAR scanning data. Many of the plug-ins perform calculations on the graphics processing unit, which allows for efficient simulation of very large domains with high detail. An example modeling study is presented in which leaf-level heterogeneity in water usage and photosynthesis of an orchard is examined to understand how this leaf-scale variability contributes to whole-tree and -canopy fluxes.}\n}"
+    },
+    "grape_detection_californiaday": {
+        "license": "",
+        "citation": "@misc{GrapeDay,\n  author    = {Plant AI and Biophysics Lab},\n  title     = {Grape Detection 2019 Day},\n  year      = {2019},\n  url       = {https://github.com/plant-ai-biophysics-lab/AgML} \n "
+    },
+    "grape_detection_californianight": {
+        "license": "",
+        "citation": "@misc{GrapeNight,\n  author    = {Plant AI and Biophysics Lab},\n  title     = {Grape Detection 2020 Night},\n  year      = {2020},\n  url       = {https://github.com/plant-ai-biophysics-lab/AgML} \n "
+    },
+    "guava_disease_pakistan": {
+        "license": "",
+        "citation": "@article{Rauf_Lali_2021, \n    title={A Guava Fruits and Leaves Dataset for Detection and Classification of Guava Diseases through Machine Learning}, \n    volume={1}, \n    url={https://data.mendeley.com/datasets/s8x6jn5cvr/1}, \n    DOI={10.17632/s8x6jn5cvr.1}, \n    abstractNote={(1) Plant diseases are the primary cause of reduced productivity in agriculture, which results in economic losses. Guava is a big source of nutrients for humans all over the world. Guava diseases, on the other hand, harm the yield and quality of the crop. (2) For the identification and classification of plant diseases, computer vision and image processing methods have been commonly used. (3) The dataset includes an image gallery of healthy and unhealthy Guava fruits and leaves that could be used by researchers to adopt advanced computer vision techniques to protect plants from disease. Dot, Canker, Mummification, and Rust are the diseases targeted in the data sets. (4) The dataset contains 306 images of healthy and unhealthy images for both Guava fruits and leaves collectively. Each image contains 6000 * 4000 dimensions with 300 dpi resolution. (5) All images were acquired from the tropical areas of Pakistan under the supervision of Prof. Dr. Ikramullah Lali. (6) All images were annotated manually by the domain expert such as For Guava fruits and leaves; Dot (76), Canker (77), Mummification (83), and Rust (70) Note: The data labeling was manual and can be updated by automatic labeling through machine learning. In the meantime, the authors can also use the data set for the clustering problem.}, \n    author={Rauf, Hafiz Tayyab and Lali, Muhammad Ikram Ullah}, \n    year={2021}, month={Apr} \n}\n"
+    },
+    "apple_detection_spain": {
+        "license": "",
+        "citation": "@article{GENEMOLA2019104289,\ntitle = {KFuji RGB-DS database: Fuji apple multi-modal images for fruit detection with color, depth and range-corrected IR data},\njournal = {Data in Brief},\nvolume = {25},\npages = {104289},\nyear = {2019},\nissn = {2352-3409},\ndoi = {https://doi.org/10.1016/j.dib.2019.104289},\nurl = {https://www.sciencedirect.com/science/article/pii/S2352340919306432},\nauthor = {Jordi Gené-Mola and Verónica Vilaplana and Joan R. Rosell-Polo and Josep-Ramon Morros and Javier Ruiz-Hidalgo and Eduard Gregorio},\nkeywords = {Multi-modal dataset, Fruit detection, Depth cameras, RGB-D, Fruit reflectance, Fuji apple},\nabstract = {This article contains data related to the research article entitle “Multi-modal Deep Learning for Fruit Detection Using RGB-D Cameras and their Radiometric Capabilities” [1]. The development of reliable fruit detection and localization systems is essential for future sustainable agronomic management of high-value crops. RGB-D sensors have shown potential for fruit detection and localization since they provide 3D information with color data. However, the lack of substantial datasets is a barrier for exploiting the use of these sensors. This article presents the KFuji RGB-DS database which is composed by 967 multi-modal images of Fuji apples on trees captured using Microsoft Kinect v2 (Microsoft, Redmond, WA, USA). Each image contains information from 3 different modalities: color (RGB), depth (D) and range corrected IR intensity (S). Ground truth fruit locations were manually annotated, labeling a total of 12,839 apples in all the dataset. The current dataset is publicly available at http://www.grap.udl.cat/publicacions/datasets.html.}\n}"
+    },
+    "apple_detection_drone_brazil": {
+        "license": "CC BY-SA 4.0",
+        "citation": "@article{DBLP:journals/corr/abs-2110-12331,\n  author    = {Thiago T. Santos and\n               Luciano Gebler},\n  title     = {A methodology for detection and localization of fruits in apples orchards\n               from aerial images},\n  journal   = {CoRR},\n  volume    = {abs/2110.12331},\n  year      = {2021},\n  url       = {https://arxiv.org/abs/2110.12331},\n  eprinttype = {arXiv},\n  eprint    = {2110.12331},\n  timestamp = {Thu, 28 Oct 2021 15:25:31 +0200},\n  biburl    = {https://dblp.org/rec/journals/corr/abs-2110-12331.bib},\n  bibsource = {dblp computer science bibliography, https://dblp.org}\n}"
+    },
+    "plant_doc_detection": {
+        "license": "CC BY-SA 4.0",
+        "citation": "@inproceedings{10.1145/3371158.3371196,\n  author = {Singh, Davinder and Jain, Naman and Jain, Pranjali and Kayal, Pratik and Kumawat, Sudhakar and Batra, Nipun},\n  title = {PlantDoc: A Dataset for Visual Plant Disease Detection},\n  year = {2020},\n  isbn = {9781450377386},\n  publisher = {Association for Computing Machinery},\n  address = {New York, NY, USA},\n  url = {https://doi.org/10.1145/3371158.3371196},\n  doi = {10.1145/3371158.3371196},\n  booktitle = {Proceedings of the 7th ACM IKDD CoDS and 25th COMAD},\n  pages = {249–253},\n  numpages = {5},\n  keywords = {Deep Learning, Object Detection, Image Classification},\n  location = {Hyderabad, India},\n  series = {CoDS COMAD 2020}\n  }"
+    },
+    "wheat_head_counting": {
+        "license": "CC BY-SA 4.0",
+        "citation": "@article{david2020global,\n  title={Global Wheat Head Detection (GWHD) dataset: a large and diverse dataset of high-resolution RGB-labelled images to develop and benchmark wheat head detection methods},\n  author={David, Etienne and Madec, Simon and Sadeghi-Tehran, Pouria and Aasen, Helge and Zheng, Bangyou and Liu, Shouyang and Kirchgessner, Norbert and Ishikawa, Goro and Nagasawa, Koichi and Badhon, Minhajul A and others},\n  journal={Plant Phenomics},\n  volume={2020},\n  year={2020},\n  publisher={Science Partner Journal}\n  }"
+    },
+    "peachpear_flower_segmentation": {
+        "license": "US Public Domain",
+        "citation": "@ARTICLE{8392727,\n  author={Dias, Philipe A. and Tabb, Amy and Medeiros, Henry},\n  journal={IEEE Robotics and Automation Letters}, \n  title={Multispecies Fruit Flower Detection Using a Refined Semantic Segmentation Network}, \n  year={2018},\n  volume={3},\n  number={4},\n  pages={3003-3010},\n  doi={10.1109/LRA.2018.2849498}}"
+    },
+    "red_grapes_and_leaves_segmentation": {
+        "license": "",
+        "citation": "@inproceedings{kalampokas2020semantic,\n  title={Semantic segmentation of vineyard images using convolutional neural networks},\n  author={Kalampokas, Theofanis and Tziridis, Konstantinos and Nikolaou, Alexandros and Vrochidou, Eleni and Papakostas, George A and Pachidis, Theodore and Kaburlasos, Vassilis G},\n  booktitle={International Conference on Engineering Applications of Neural Networks},\n  pages={292--303},\n  year={2020},\n  organization={Springer}}"
+    },
+    "white_grapes_and_leaves_segmentation": {
+        "license": "",
+        "citation": "@inproceedings{kalampokas2020semantic,\n  title={Semantic segmentation of vineyard images using convolutional neural networks},\n  author={Kalampokas, Theofanis and Tziridis, Konstantinos and Nikolaou, Alexandros and Vrochidou, Eleni and Papakostas, George A and Pachidis, Theodore and Kaburlasos, Vassilis G},\n  booktitle={International Conference on Engineering Applications of Neural Networks},\n  pages={292--303},\n  year={2020},\n  organization={Springer}}"
+    },
+    "ghai_romaine_detection": {
+        "license": "CC BY-SA 4.0",
+        "citation": ""
+    },
+    "ghai_green_cabbage_detection": {
+        "license": "CC BY-SA 4.0",
+        "citation": ""
+    },
+    "ghai_iceberg_lettuce_detection": {
+        "license": "CC BY-SA 4.0",
+        "citation": ""
+    },
+    "riseholme_strawberry_classification_2021": {
+        "license": "",
+        "citation": "@inproceedings{CWSC21,\n  title={Self-supervised Representation Learning for Reliable Robotic Monitoring of Fruit Anomalies},\n  author={Choi, Taeyeong and Would, Owen and Salazar-Gomez, Adrian and Cielniak, Grzegorz},\n  booktitle={2022 International Conference on Robotics and Automation (ICRA)},\n  pages={2266--2272},\n  year={2022},\n  organization={IEEE}\n}"
+    },
+    "ghai_broccoli_detection": {
+        "license": "CC BY-SA 4.0",
+        "citation": ""
+    },
+    "bean_synthetic_earlygrowth_aerial": {
+        "license": "MIT",
+        "citation": "@ARTICLE{10.3389/fpls.2019.01185,\n  \nAUTHOR={Bailey, Brian N.},   \n\t \nTITLE={Helios: A Scalable 3D Plant and Environmental Biophysical Modeling Framework},      \n\t\nJOURNAL={Frontiers in Plant Science},      \n\t\nVOLUME={10},      \n\t\nYEAR={2019},      \n\t  \nURL={https://www.frontiersin.org/article/10.3389/fpls.2019.01185},       \n\t\nDOI={10.3389/fpls.2019.01185},      \n\t\nISSN={1664-462X},   \n   \nABSTRACT={This article presents an overview of Helios, a new three-dimensional (3D) plant and environmental modeling framework. Helios is a model coupling framework designed to provide maximum flexibility in integrating and running arbitrary 3D environmental system models. Users interact with Helios through a well-documented open-source C++ API. Version 1.0 comes with model plug-ins for radiation transport, the surface energy balance, stomatal conductance, photosynthesis, solar position, and procedural tree generation. Additional plug-ins are also available for visualizing model geometry and data and for processing and integrating LiDAR scanning data. Many of the plug-ins perform calculations on the graphics processing unit, which allows for efficient simulation of very large domains with high detail. An example modeling study is presented in which leaf-level heterogeneity in water usage and photosynthesis of an orchard is examined to understand how this leaf-scale variability contributes to whole-tree and -canopy fluxes.}\n}"
+    },
+    "ghai_strawberry_fruit_detection": {
+        "license": "CC BY-SA 4.0",
+        "citation": ""
+    },
+    "vegann_multicrop_presence_segmentation": {
+        "license": "CC BY-SA 4.0",
+        "citation": " @article{Madec_Irfan_Velumani_Baret_David_Daubige_Samatan_Serouart_Smith_James_et al._2023, title={VegAnn, Vegetation Annotation of multi-crop RGB images acquired under diverse conditions for segmentation}, volume={10}, ISSN={2052-4463}, url={https://www.nature.com/articles/s41597-023-02098-y}, DOI={10.1038/s41597-023-02098-y}, abstractNote={Abstract\n            \n              Applying deep learning to images of cropping systems provides new knowledge and insights in research and commercial applications. Semantic segmentation or pixel-wise classification, of RGB images acquired at the ground level, into vegetation and background is a critical step in the estimation of several canopy traits. Current state of the art methodologies based on convolutional neural networks (CNNs) are trained on datasets acquired under controlled or indoor environments. These models are unable to generalize to real-world images and hence need to be fine-tuned using new labelled datasets. This motivated the creation of the VegAnn -\n              Veg\n              etation\n              Ann\n              otation - dataset, a collection of 3775 multi-crop RGB images acquired for different phenological stages using different systems and platforms in diverse illumination conditions. We anticipate that VegAnn will help improving segmentation algorithm performances, facilitate benchmarking and promote large-scale crop vegetation segmentation research.}, number={1}, journal={Scientific Data}, author={Madec, Simon and Irfan, Kamran and Velumani, Kaaviya and Baret, Frederic and David, Etienne and Daubige, Gaetan and Samatan, Lucas Bernigaud and Serouart, Mario and Smith, Daniel and James, Chrisbin and Camacho, Fernando and Guo, Wei and De Solan, Benoit and Chapman, Scott C. and Weiss, Marie}, year={2023}, month=may, pages={302}, language={en} }\n"
+    },
+    "corn_maize_leaf_disease": {
+        "license": "",
+        "citation": "Singh D, Jain N, Jain P, Kayal P, Kumawat S, Batra N. PlantDoc: a dataset for visual plant disease detection. InProceedings of the 7th ACM IKDD CoDS and 25th COMAD 2020 Jan 5 (pp. 249-253)."
+    },
+    "tomato_leaf_disease": {
+        "license": "CC0: Public Domain",
+        "citation": ""
+    },
+    "vine_virus_photo_dataset": {
+        "license": "Apache 2.0",
+        "citation": ""
+    },
+    "embrapa_wgisd_grape_detection": {
+        "license": "CC BY-NC 4.0",
+        "citation": ""
+    },
+    "growliflower_cauliflower_segmentation": {
+        "license": "",
+        "citation": "Kierdorf, Jana & Junker-Frohn, Laura & Delaney, Mike & Olave, Mariele & Burkart, Andreas & Jaenicke, Hannah & Muller, Onno & Roscher, Ribana. (2022). GrowliFlower: An image time‐series dataset for GROWth analysis of cauLIFLOWER. Journal of Field Robotics. 40. 10.1002/rob.22122. "
+    },
+    "strawberry_detection_2023": {
+        "license": "CC BY 4.0",
+        "citation": ""
+    },
+    "strawberry_detection_2022": {
+        "license": "CC BY 4.0",
+        "citation": ""
+    },
+    "almond_harvest_2021": {
+        "license": "Apache 2.0",
+        "citation": ""
+    },
+    "almond_bloom_2023": {
+        "license": "Apache 2.0",
+        "citation": ""
+    },
+    "gemini_flower_detection_2022": {
+        "license": "Apache 2.0",
+        "citation": ""
+    },
+    "gemini_leaf_detection_2022": {
+        "license": "Apache 2.0",
+        "citation": ""
+    },
+    "gemini_pod_detection_2022": {
+        "license": "Apache 2.0",
+        "citation": ""
+    },
+    "gemini_plant_detection_2022": {
+        "license": "Apache 2.0",
+        "citation": ""
+    },
+    "paddy_disease_classification": {
+        "license": "CC BY 4.0",
+        "citation": "Petchiammal A, Briskline Kiruba S, Murugan D, Pandarasamy Arjunan. (2022). Paddy Doctor: A Visual Image Dataset for Automated Paddy Disease Classification and Benchmarking. IEEE Dataport. https://dx.doi.org/10.21227/hz4v-af08"
+    },
+    "onion_leaf_classification": {
+        "license": "CC BY 4.0",
+        "citation": "Aishwarya, M.P & Reddy, A.. (2024). Dataset of Chilli and Onion Plant Leaf Images for Classification and Detection. Data in Brief. 54. 110524. 10.1016/j.dib.2024.110524. "
+    },
+    "chilli_leaf_classification": {
+        "license": "CC BY 4.0",
+        "citation": "Aishwarya, M.P & Reddy, A.. (2024). Dataset of Chilli and Onion Plant Leaf Images for Classification and Detection. Data in Brief. 54. 110524. 10.1016/j.dib.2024.110524. "
+    },
+    "orange_leaf_disease_classification": {
+        "license": "CC BY 4.0",
+        "citation": "Emon, Yousuf Rayhan; Ahad, Md Taimur (2023), “Multi-format open-source sweet orange leaf dataset for disease detection, classification, and analysis.”, Mendeley Data, V1, doi: 10.17632/f7cr74mwpj.1"
+    },
+    "papaya_leaf_disease_classification": {
+        "license": "CC BY 4.0",
+        "citation": "Sarker, Arpita ; Mustofa, Sumaya; Ahad, Md Taimur  (2023), “BDPapayaLeaf: A annotation based image dataset of papaya leaf disease.”, Mendeley Data, V1, doi: 10.17632/p997fvf526.1"
+    },
+    "blackgram_plant_leaf_disease_classification": {
+        "license": "CC BY 4.0",
+        "citation": "Talasila, Srinivas; Rawal, Kirti; Sethi, Gaurav; MSS, Sanjay; M, Surya Prakash Reddy (2022), “Blackgram Plant Leaf Disease Dataset”, Mendeley Data, V3, doi: 10.17632/zfcv9fmrgv.3"
+    },
+    "arabica_coffee_leaf_disease_classification": {
+        "license": "CC BY 4.0",
+        "citation": "@article{JEPKOECH2021107142, title = {Arabica coffee leaf images dataset for coffee leaf disease detection and classification}, journal = {Data in Brief}, volume = {36}, pages = {107142}, year = {2021}, issn = {2352-3409}, doi = {https://doi.org/10.1016/j.dib.2021.107142}, url = {https://www.sciencedirect.com/science/article/pii/S2352340921004261}, author = {Jennifer Jepkoech and David Muchangi Mugo and Benson K. Kenduiywo and Edna Chebet Too}}"
+    },
+    "banana_leaf_disease_classification": {
+        "license": "CC BY 4.0",
+        "citation": "hailu, yordanos (2021), “Banana Leaf Disease Images”, Mendeley Data, V1, doi: 10.17632/rjykr62kdh.1"
+    },
+    "coconut_tree_disease_classification": {
+        "license": "CC BY 4.0",
+        "citation": "PATIL, Kailas; Thite, Sandip; Suryawanshi, Yogesh; chumchu, prawit (2023), “Coconut Tree Disease Dataset”, Mendeley Data, V1, doi: 10.17632/gh56wbsnj5.1"
+    },
+    "rice_leaf_disease_classification": {
+        "license": "",
+        "citation": ""
+    },
+    "tea_leaf_disease_classification": {
+        "license": "CC BY-NC 4.0",
+        "citation": "@article{BALASUNDARAM2025103784, title = {Tea leaf disease detection using segment anything model and deep convolutional neural networks}, journal = {Results in Engineering}, volume = {25}, pages = {103784}, year = {2025}, issn = {2590-1230}, doi = {https://doi.org/10.1016/j.rineng.2024.103784}, url = {https://www.sciencedirect.com/science/article/pii/S2590123024020279}, author = {Ananthakrishnan Balasundaram and Prem Sundaresan and Aryan Bhavsar and Mishti Mattu and Muthu Subash Kavitha and Ayesha Shaik}}"
+    },
+    "betel_leaf_disease_classification": {
+        "license": "CC BY 4.0",
+        "citation": "Rashid, Mohammad Rifat Ahmmad; Hossain, Md. Miskat ; Biswas,  Joy ; Majumder, Hredoy  (2024), “Betel Leaf Image Dataset from Bangladesh”, Mendeley Data, V2, doi: 10.17632/g7fpgj57wc.2"
+    },
+    "java_plum_leaf_disease_classification": {
+        "license": "CC BY 4.0",
+        "citation": "Bhowmik, Auvick Chandra; Ahad, Taimur (2024), “Java Plum Leaf Disease Dataset”, Mendeley Data, V3, doi: 10.17632/43d75vptz4.3"
+    },
+    "sunflower_disease_classification": {
+        "license": "CC BY 4.0",
+        "citation": "Rajbongshi, Aditya; Sara, Umme ; Akter, Bonna ; Shakil, Rashiduzzaman ; Sazzad, Sadia (2022), “Sun Flower Fruits and Leaves dataset for Sunflower Disease Classification through Machine Learning and Deep Learning”, Mendeley Data, V1, doi: 10.17632/b83hmrzth8.1"
+    },
+    "cucumber_disease_classification": {
+        "license": "CC BY 4.0",
+        "citation": "Sultana, Nusrat; Shorif, Sumaita Binte ; Akter, Morium ; Uddin, Mohammad Shorif  (2022), “Cucumber Disease Recognition Dataset”, Mendeley Data, V1, doi: 10.17632/y6d3z6f8z9.1"
+    },
+    "soybean_insect_classification": {
+        "license": "CC BY 4.0",
+        "citation": "Mignoni, Maria Eloisa (2021), “Images of Soybean Leaves”, Mendeley Data, V1, doi: 10.17632/bycbh73438.1"
+    }
+}

AgML/source/agml/_assets/viz_colormaps.json

@@ -0,0 +1,18 @@
+{
+    "default": [
+        [0, 255, 0],
+        [255, 0, 0],
+        [0, 0, 255],
+        [255, 255, 0],
+        [255, 0, 255],
+        [0, 255, 255]
+    ],
+    "agriculture": [
+        [0, 255, 0],
+        [50, 168, 82],
+        [42, 84, 24],
+        [17, 71, 4],
+        [57, 66, 53],
+        [181, 204, 171]
+    ]
+}

AgML/source/agml/_helios/helios_install.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+
+# This script will almost certainly not be run manually, and instead
+# be called by the AgML Helios module. So, we can't rely on using the
+# existing parent directory to install Helios. Instead, we need to get
+# the path of this actual file itself, and then
+PATH_TO_ME="$(readlink -nf "$0")"
+INSTALL_PATH="$(dirname "$(dirname "$PATH_TO_ME")")/_helios/Helios"
+
+# Install or Update Helios, depending on whether the directory for Helios
+# already exists. While the actual Python installation script which calls
+# this shell script has slightly more complex logic (for figuring out
+# version Helios is on, and in turn, whether it needs an update or not),
+# this simply installs/updates based on the existence of the directory.
+if [ ! -d "$INSTALL_PATH" ]; then
+  git clone -b master https://github.com/PlantSimulationLab/Helios.git "$INSTALL_PATH"
+else
+  ORIGINAL_DIR="$PWD"
+  cd "$INSTALL_PATH"
+  git pull https://github.com/PlantSimulationLab/Helios.git master
+  cd "$ORIGINAL_DIR" || echo "Issue when trying to update Helios. Please report this to the AgML team."; exit
+fi
+
+
+

AgML/source/agml/_internal/preprocess.py

@@ -0,0 +1,1609 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Preprocessing code for AgML public data sources.
+This file stores the preprocessing code used to preprocess a public
+dataset when added to AgML's public data sources.
+If you want to use this preprocessing code, run `pip install agml[dev]`
+to install the necessary preprocessing packages.
+"""
+
+import argparse
+import csv
+import glob
+import json
+import os
+import shutil
+import sys
+
+import cv2
+import numpy as np
+import pandas as pd
+from PIL import Image
+
+from agml._internal.process_utils import (
+    convert_bbox_to_coco,
+    convert_xmls_to_cocojson,
+    get_coco_annotation_from_obj,
+    get_image_info,
+    get_label2id,
+    move_segmentation_dataset,
+    read_txt_file,
+)
+from agml.utils.data import load_public_sources
+from agml.utils.io import create_dir, get_dir_list, get_file_list, nested_dir_list
+from agml.utils.logging import tqdm
+
+def yolo_to_coco_bbox(x_center, y_center, width, height, img_width, img_height):
+    """Convert YOLO format (x_center, y_center, width, height) to COCO format (xmin, ymin, width, height)."""
+    xmin = (x_center - (width / 2)) * img_width
+    ymin = (y_center - (height / 2)) * img_height
+    width = width * img_width
+    height = height * img_height
+    return [xmin, ymin, width, height]
+
+
+class PublicDataPreprocessor(object):
+    """Internal data preprocessing class.
+
+    Parameters
+    ----------
+    data_dir : str
+        The directory with a folder `original` and `processed` to hold
+        the original and processed datasets, respectively.
+    """
+
+    def __init__(self, data_dir):
+        self.data_dir = os.path.abspath(data_dir)
+        self.data_original_dir = os.path.join(self.data_dir, "original")
+        self.data_processed_dir = os.path.join(self.data_dir, "processed")
+        self.data_sources = load_public_sources()
+
+    def preprocess(self, dataset_name):
+        """Preprocesses the provided dataset.
+        Parameters
+        ----------
+        dataset_name : str
+            name of dataset to preprocess
+        """
+        getattr(self, dataset_name)(dataset_name)
+
+    def vine_virus_photo_dataset(self, dataset_name):
+        """Preprocesses the Vine Virus Photo Dataset."""
+        # Get the dataset directory directly (no need for 'original' directory)
+        base_path = self.data_dir
+        classes = sorted([d for d in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, d))])
+
+        # Create output directory
+        output_path = os.path.join(self.data_processed_dir, dataset_name)
+        if not os.path.exists(output_path):
+            os.makedirs(output_path)
+
+        # Create subdirectories for each class inside the output directory
+        for cls in classes:
+            class_output_path = os.path.join(output_path, cls)
+            if not os.path.exists(class_output_path):
+                os.makedirs(class_output_path)
+
+        # Process and copy the dataset images to the processed directory
+        for cls in classes:
+            class_path = os.path.join(base_path, cls)
+            for img in os.listdir(class_path):
+                if img.endswith(("jpg", "png", "jpeg", "JPG")):
+                    img_path = os.path.join(class_path, img)
+                    shutil.copyfile(img_path, os.path.join(output_path, cls, img))
+
+        print(f"Dataset {dataset_name} has been preprocessed and saved to {output_path}")
+
+    def corn_maize_leaf_disease(self, dataset_name):
+        """Preprocesses the Corn or Maize Leaf Disease Dataset."""
+        # Get the dataset directory directly (no need for 'original' directory)
+        base_path = self.data_dir
+        classes = sorted([d for d in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, d))])
+
+        # Create output directory
+        output_path = os.path.join(self.data_processed_dir, dataset_name)
+        if not os.path.exists(output_path):
+            os.makedirs(output_path)
+
+        # Create subdirectories for each class inside the output directory
+        for cls in classes:
+            class_output_path = os.path.join(output_path, cls)
+            if not os.path.exists(class_output_path):
+                os.makedirs(class_output_path)
+
+        # Process and copy the dataset images to the processed directory
+        for cls in classes:
+            class_path = os.path.join(base_path, cls)
+            for img in os.listdir(class_path):
+                if img.endswith(("jpg", "png", "jpeg", "JPG")):
+                    img_path = os.path.join(class_path, img)
+
+                    # Open the image using Pillow
+                    with Image.open(img_path) as image:
+                        # Convert image to a NumPy array
+                        img_array = np.array(image)
+
+                        # Ensure the image is in range [0, 255] and dtype is uint8
+                        if img_array.dtype != np.uint8:
+                            # If the image is in float [0-1], scale it to [0-255]
+                            img_array = (img_array * 255).astype(np.uint8)
+
+                        # Convert back to PIL Image to save
+                        processed_image = Image.fromarray(img_array)
+
+                        if processed_image.mode == "RGBA":
+                            processed_image = processed_image.convert("RGB")
+
+                        # Save the processed image to the output directory
+                        processed_image.save(os.path.join(output_path, cls, img))
+
+        print(f"Dataset {dataset_name} has been preprocessed and saved to {output_path}")
+
+    def tomato_leaf_disease(self, dataset_name):
+        """Preprocesses the Tomato Leaf Disease Dataset."""
+        # Get the dataset directory directly (no need for 'original' directory)
+        base_path = self.data_dir
+        classes = sorted([d for d in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, d))])
+
+        # Create output directory
+        output_path = os.path.join(self.data_processed_dir, dataset_name)
+        if not os.path.exists(output_path):
+            os.makedirs(output_path)
+
+        # Create subdirectories for each class inside the output directory
+        for cls in classes:
+            class_output_path = os.path.join(output_path, cls)
+            if not os.path.exists(class_output_path):
+                os.makedirs(class_output_path)
+
+        # Process and copy the dataset images to the processed directory
+        for cls in classes:
+            class_path = os.path.join(base_path, cls)
+            for img in os.listdir(class_path):
+                if img.endswith(("jpg", "png", "jpeg", "JPG")):
+                    img_path = os.path.join(class_path, img)
+
+                    # Open the image using Pillow
+                    with Image.open(img_path) as image:
+                        # Convert image to a NumPy array
+                        img_array = np.array(image)
+
+                        # Ensure the image is in range [0, 255] and dtype is uint8
+                        if img_array.dtype != np.uint8:
+                            # If the image is in float [0-1], scale it to [0-255]
+                            img_array = (img_array * 255).astype(np.uint8)
+
+                        # Convert back to PIL Image to save
+                        processed_image = Image.fromarray(img_array)
+
+                        if processed_image.mode == "RGBA":
+                            processed_image = processed_image.convert("RGB")
+
+                        # Save the processed image to the output directory
+                        processed_image.save(os.path.join(output_path, cls, img))
+
+        print(f"Dataset {dataset_name} has been preprocessed and saved to {output_path}")
+
+    def bean_disease_uganda(self, dataset_name):
+        # Get the dataset classes and paths
+        base_path = os.path.join(self.data_original_dir, dataset_name)
+        dirs = ["train", "validation", "test"]
+        classes = sorted(os.listdir(os.path.join(base_path, dirs[0])))[1:]
+
+        # Construct output directories
+        output = os.path.join(self.data_processed_dir, dataset_name)
+        os.makedirs(output, exist_ok=True)
+        for cls in classes:
+            os.makedirs(os.path.join(output, cls), exist_ok=True)
+
+        # Move the dataset
+        for dir_ in dirs:
+            for cls in classes:
+                path = os.path.join(base_path, dir_, cls)
+                for p in os.listdir(path):
+                    if p.endswith("jpg") or p.endswith("png"):
+                        img = os.path.join(base_path, dir_, cls, p)
+                        shutil.copyfile(img, os.path.join(output, cls, p))
+
+    def leaf_counting_denmark(self, dataset_name):
+        pass
+
+    def plant_seedlings_aarhus(self, dataset_name):
+        pass
+
+    def crop_weeds_greece(self, dataset_name):
+        pass
+
+    def rangeland_weeds_australia(self, dataset_name):
+        # Get the file information.
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        images = get_file_list(os.path.join(dataset_dir, "images"))
+        df = pd.read_csv(os.path.join(dataset_dir, "labels.csv"))
+
+        # Construct the new structure.
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        unique_labels = np.unique(df["Species"])
+        for unique_label in unique_labels:
+            os.makedirs(os.path.join(processed_dir, unique_label.title()), exist_ok=True)
+        for file in tqdm(images, desc="Moving Images", file=sys.stdout):
+            save_dir = df.loc[df["Filename"] == os.path.basename(file)]["Species"].values[0].title()
+            shutil.copyfile(
+                os.path.join(dataset_dir, "images", file),
+                os.path.join(processed_dir, save_dir, os.path.basename(file)),
+            )
+
+    def fruit_detection_worldwide(self, dataset_name):
+        # Get the dataset directory
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name, "datasets")
+
+        # Get folder list
+        dataset_folders = get_dir_list(dataset_dir)
+        label2id = get_label2id(dataset_folders)
+        anno_data_all = []
+        for folder in dataset_folders:
+            annotations = ["test_RGB.txt", "train_RGB.txt"]
+            dataset_path = os.path.join(dataset_dir, folder)
+            # @TODO: Make separate json files for train and test?
+            for anno_file_name in annotations:
+                # Read annotations
+                try:
+                    anno_data = read_txt_file(os.path.join(dataset_path, anno_file_name))
+                except:
+                    try:
+                        anno_data = read_txt_file(os.path.join(dataset_path, anno_file_name + ".txt"))
+                    except Exception as e:
+                        raise e
+
+                # Concat fruit name at head of line
+                for i, anno in enumerate(anno_data):
+                    # Change to test path if the text file is test
+                    if "test" in anno_file_name and "TRAIN" in anno[0]:
+                        anno_data[i][0] = anno[0].replace("TRAIN", "TEST")
+                    anno_data[i][0] = os.path.join(dataset_path, anno_data[i][0])
+
+                anno_data_all += anno_data
+
+        # Process annotation files
+        save_dir_anno = os.path.join(self.data_processed_dir, dataset_name, "annotations")
+        create_dir(save_dir_anno)
+        output_json_file = os.path.join(save_dir_anno, "instances.json")
+
+        general_info = {
+            "description": "fruits dataset",
+            "url": "https://drive.google.com/drive/folders/1CmsZb1caggLRN7ANfika8WuPiywo4mBb",
+            "version": "1.0",
+            "year": 2018,
+            "contributor": "Inkyu Sa",
+            "date_created": "2018/11/12",
+        }
+
+        # Process image files
+        output_img_path = os.path.join(self.data_processed_dir, dataset_name, "images")
+        create_dir(output_img_path)
+
+        convert_bbox_to_coco(anno_data_all, label2id, output_json_file, output_img_path, general_info)
+
+    def apple_detection_usa(self, dataset_name, fix=False):
+        # Just a quick fix to clip over-sized bounding boxes.
+        if fix:
+            # Load in the annotations.
+            dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+            with open(os.path.join(dataset_dir, "annotations.json"), "r") as f:
+                annotations = json.load(f)
+
+            # Get the images and all of their heights/widths.
+            images = annotations["images"]
+            image_id_content_map = {}
+            for image in images:
+                image_id_content_map[image["id"]] = (image["height"], image["width"])
+
+            # Load all of the annotations.
+            new_annotations = []
+            for a in annotations["annotations"]:
+                new_a = a.copy()
+                height, width = image_id_content_map[a["image_id"]]
+                (x, y, w, h) = a["bbox"]
+                x1, y1, x2, y2 = x, y, x + w, y + h
+                x1 = np.clip(x1, 0, width)
+                x2 = np.clip(x2, 0, width)
+                y1 = np.clip(y1, 0, height)
+                y2 = np.clip(y2, 0, height)
+                new_a["bbox"] = [int(i) for i in [x1, y1, x2 - x1, y2 - y1]]
+                new_annotations.append(new_a)
+
+            # Save the annotations.
+            annotations["annotations"] = new_annotations
+            with open(os.path.join(dataset_dir, "annotations.json"), "w") as f:
+                json.dump(annotations, f)
+            return
+
+        # resize the dataset
+        resize = 1.0
+
+        # Read public_datasources.json to get class information
+        category_info = self.data_sources[dataset_name]["classes"]
+        labels_str = []
+        labels_ids = []
+        for info in category_info:
+            labels_str.append(category_info[info])
+            labels_ids.append(int(info))
+
+        label2id = dict(zip(labels_str, labels_ids))
+
+        # Task 1: Image classification
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        obj_Detection_data = os.path.join(dataset_dir, "Dataset")
+
+        # get folders
+        plant_folders = nested_dir_list(obj_Detection_data)
+
+        # do tasks along folders
+        anno_data_all = []
+        for folder in plant_folders:
+            # Get image file and xml file
+            full_path = os.path.join(obj_Detection_data, folder)
+            all_files = get_file_list(full_path)
+            anno_files = [x for x in all_files if "txt" in x]
+            for anno_file in anno_files:
+                anno_line = []
+                anno_path = os.path.join(full_path, anno_file)
+                # Opening annotation file
+                anno_data = read_txt_file(anno_path, delimiter=",")[0]
+
+                for i, anno in enumerate(anno_data):
+                    new_anno = [os.path.join(dataset_dir, anno_data[i][0])]
+                    # Add bbox count
+                    # Update image file path to abs path
+                    bbox_cnt = int((len(anno_data[i]) - 1) / 4)
+                    new_anno.append(str(bbox_cnt))
+                    for idx in range(bbox_cnt):
+                        xmin = int(anno[1 + 4 * idx])
+                        ymin = int(anno[1 + 4 * idx + 1])
+                        w = int(anno[1 + 4 * idx + 2])
+                        h = int(anno[1 + 4 * idx + 3])
+
+                        new_anno.append(str(xmin))  # xmin
+                        new_anno.append(str(ymin))  # ymin
+                        new_anno.append(str(xmin + w))  # xmax
+                        new_anno.append(str(ymin + h))  # ymax
+                        new_anno.append(str(1))  # label
+                    anno_data[i] = new_anno
+                anno_data_all += anno_data
+
+        # Process annotation files
+        save_dir_anno = os.path.join(self.data_processed_dir, dataset_name, "annotations")
+        create_dir(save_dir_anno)
+        output_json_file = os.path.join(save_dir_anno, "instances.json")
+
+        general_info = {
+            "description": "apple dataset",
+            "url": "https://research.libraries.wsu.edu:8443/xmlui/handle/2376/17721",
+            "version": "1.0",
+            "year": 2019,
+            "contributor": "Bhusal, Santosh, Karkee, Manoj, Zhang, Qin",
+            "date_created": "2019/04/20",
+        }
+
+        # Process image files
+        output_img_path = os.path.join(self.data_processed_dir, dataset_name, "images")
+        create_dir(output_img_path)
+        convert_bbox_to_coco(
+            anno_data_all,
+            label2id,
+            output_json_file,
+            output_img_path,
+            general_info,
+            None,
+            None,
+            get_label_from_folder=False,
+            resize=resize,
+            add_foldername=True,
+        )
+
+    def mango_detection_australia(self, dataset_name):
+        # resize the dataset
+        resize = 1.0
+
+        # Read public_datasources.json to get class information
+        datasource_file = os.path.join(os.path.dirname(__file__), "../_assets/public_datasources.json")
+        with open(datasource_file) as f:
+            data = json.load(f)
+            category_info = data[dataset_name]["crop_types"]
+            labels_str = []
+            labels_ids = []
+            for info in category_info:
+                labels_str.append(category_info[info])
+                labels_ids.append(int(info))
+
+            name_converter = dict(zip(["M"], ["mango"]))  # src -> dst
+            label2id = dict(zip(labels_str, labels_ids))
+
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        ann_dir = os.path.join(dataset_dir, "VOCDevkit/VOC2007/Annotations")
+
+        # Get image file and xml file
+        all_files = get_file_list(ann_dir)
+        anno_files = [os.path.join(ann_dir, x) for x in all_files if "xml" in x]
+        img_files = [x.replace(".xml", ".jpg").replace("Annotations", "JPEGImages") for x in anno_files]
+
+        # Process annotation files
+        save_dir_anno = os.path.join(self.data_processed_dir, dataset_name, "annotations")
+        create_dir(save_dir_anno)
+        output_json_file = os.path.join(save_dir_anno, "instances.json")
+
+        # Process image files
+        output_img_path = os.path.join(self.data_processed_dir, dataset_name, "images")
+        create_dir(output_img_path)
+
+        general_info = {
+            "description": "MangoYOLO data set",
+            "url": "https://researchdata.edu.au/mangoyolo-set/1697505",
+            "version": "1.0",
+            "year": 2019,
+            "contributor": "Anand Koirala, Kerry Walsh, Z Wang, C McCarthy",
+            "date_created": "2019/02/25",
+        }
+
+        convert_xmls_to_cocojson(
+            general_info,
+            annotation_paths=anno_files,
+            img_paths=img_files,
+            label2id=label2id,
+            name_converter=name_converter,
+            output_jsonpath=output_json_file,
+            output_imgpath=output_img_path,
+            extract_num_from_imgid=True,
+        )
+
+    def tomato_ripeness_detection(self, dataset_name):
+        pass
+
+    def cotton_seedling_counting(self, dataset_name):
+        # Get all of the relevant data
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        image_dir = os.path.join(dataset_dir, "Images")
+        images = sorted([os.path.join(image_dir, i) for i in os.listdir(image_dir)])
+        with open(os.path.join(dataset_dir, "Images.json"), "r") as f:
+            annotations = json.load(f)
+
+        # Get all of the unique labels
+        labels = []
+        for label_set in annotations["frames"].values():
+            for individual_set in label_set:
+                labels.extend(individual_set["tags"])
+        labels = np.unique(labels).tolist()
+        label2id = get_label2id(labels)  # noqa
+
+        # Extract all of the bounding boxes and images
+        image_data = []
+        annotation_data = []
+        valid_paths = []  # some paths are not in the annotations, track the ones which are
+        for indx, (img_path, annotation) in enumerate(
+            zip(
+                tqdm(images, file=sys.stdout, desc="Generating Data"),
+                annotations["frames"].values(),
+            )
+        ):
+            image_data.append(get_image_info(img_path, indx))
+            valid_paths.append(img_path)
+            for a_set in annotation:
+                formatted_set = [
+                    a_set["x1"],
+                    a_set["y1"],
+                    a_set["x2"],
+                    a_set["y2"],
+                    label2id[a_set["tags"][0]],
+                ]
+                base_annotation_data = get_coco_annotation_from_obj(formatted_set, a_set["name"])
+                base_annotation_data["image_id"] = indx + 1
+                annotation_data.append(base_annotation_data)
+
+        # Set up the annotation dictionary
+        all_annotation_data = {
+            "images": [],
+            "type": "instances",
+            "annotations": [],
+            "categories": [],
+            "info": {
+                "description": "cotton seedling counting dataset",
+                "url": "https://figshare.com/s/616956f8633c17ceae9b",
+                "version": "1.0",
+                "year": 2019,
+                "contributor": "Yu Jiang",
+                "date_created": "2019/11/23",
+            },
+        }
+
+        # Populate the annotation dictionary
+        for label, label_id in label2id.items():
+            category_info = {"supercategory": "none", "id": label_id, "name": label}
+            all_annotation_data["categories"].append(category_info)
+        all_annotation_data["images"] = image_data
+        all_annotation_data["annotations"] = annotation_data
+
+        # Recreate the dataset and zip it
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_img_dir = os.path.join(processed_dir, "images")
+        if os.path.exists(processed_dir):
+            shutil.rmtree(processed_dir)
+        os.makedirs(processed_dir, exist_ok=True)
+        os.makedirs(processed_img_dir, exist_ok=True)
+        for path in images:
+            if path not in valid_paths:
+                continue
+            shutil.copyfile(path, os.path.join(processed_img_dir, os.path.basename(path)))
+        with open(os.path.join(processed_dir, "annotations.json"), "w") as f:
+            json.dump(all_annotation_data, f, indent=4)
+
+        # Zip the dataset
+        shutil.make_archive(processed_dir, "zip", os.path.dirname(processed_dir))
+
+    def apple_flower_segmentation(self, dataset_name):
+        # Get all of the relevant data.
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        apple_a_dir = os.path.join(dataset_dir, "FlowerImages")
+        apple_a_images = os.listdir(apple_a_dir)
+        apple_a_label_dir = os.path.join(dataset_dir, "AppleA_Labels")
+        apple_a_labels = os.listdir(apple_a_label_dir)
+        apple_b_dir = os.path.join(dataset_dir, "AppleB")
+        apple_b_images = os.listdir(apple_b_dir)
+        apple_b_label_dir = os.path.join(dataset_dir, "AppleB_Labels")
+        apple_b_labels = os.listdir(apple_b_label_dir)
+
+        # Map image filenames with their corresponding labels.
+        fname_map_a, fname_map_b = {}, {}
+        for fname in apple_a_images:
+            fname_id = str(int(float(os.path.splitext(fname)[0].split("_")[-1]))) + ".png"
+            if fname_id in apple_a_labels:
+                fname_map_a[os.path.join(apple_a_dir, fname)] = os.path.join(apple_a_label_dir, fname_id)
+        for fname in apple_b_images:
+            fname_id = str(int(float(os.path.splitext(fname)[0].split("_")[-1]))) + ".png"
+            if fname_id in apple_b_labels:
+                fname_map_b[os.path.join(apple_b_dir, fname)] = os.path.join(apple_b_label_dir, fname_id)
+
+        # Process and move the images.
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        os.makedirs(processed_dir, exist_ok=True)
+        processed_image_dir = os.path.join(processed_dir, "images")
+        os.makedirs(processed_image_dir, exist_ok=True)
+        processed_annotation_dir = os.path.join(processed_dir, "annotations")
+        os.makedirs(processed_annotation_dir, exist_ok=True)
+        for image_path, label_path in tqdm(fname_map_a.items(), desc="Processing Part A", file=sys.stdout):
+            image = cv2.resize(cv2.imread(image_path), (2074, 1382))
+            label = cv2.resize(cv2.imread(label_path), (2074, 1382)) // 255
+            label_path = os.path.basename(label_path)
+            out_image_path = os.path.join(processed_image_dir, label_path)
+            out_label_path = os.path.join(processed_annotation_dir, label_path)
+            cv2.imwrite(out_image_path.replace(".png", ".jpg"), image)
+            cv2.imwrite(out_label_path, label)
+        for image_path, label_path in tqdm(fname_map_b.items(), desc="Processing Part B", file=sys.stdout):
+            image = cv2.resize(cv2.imread(image_path), (2074, 1382))
+            label = cv2.resize(cv2.imread(label_path), (2074, 1382)) // 255
+            label_path = os.path.basename(label_path)
+            out_image_path = os.path.join(processed_image_dir, label_path)
+            out_label_path = os.path.join(processed_annotation_dir, label_path)
+            cv2.imwrite(out_image_path.replace(".png", ".jpg"), image)
+            cv2.imwrite(out_label_path, label)
+
+    def sugarbeet_weed_segmentation(self, dataset_name):
+        # Get all of the relevant data
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        train_dir = os.path.join(dataset_dir, "train")
+        train_images = sorted(get_file_list(train_dir))
+        annotation_dir = os.path.join(dataset_dir, "trainannot")  # noqa
+        annotation_images = sorted(get_file_list(annotation_dir))
+
+        # Move the images to the new directory
+        move_segmentation_dataset(
+            self.data_processed_dir,
+            dataset_name,
+            train_images,
+            annotation_images,
+            train_dir,
+            annotation_dir,
+        )
+
+    def carrot_weeds_germany(self, dataset_name):
+        # Get all of the relevant data.
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        train_dir = os.path.join(dataset_dir, "images")
+        train_images = sorted(get_file_list(train_dir))
+        annotation_dir = os.path.join(dataset_dir, "annotations")
+        annotation_images = sorted(get_file_list(annotation_dir, ext="png"))
+
+        # Move the images to the new directory.
+        def _annotation_preprocess_fn(annotation_path, out_path):
+            an_img = cv2.cvtColor(cv2.imread(annotation_path), cv2.COLOR_BGR2RGB)
+            crop, weed = (0, 255, 0), (255, 0, 0)
+            out_annotation = np.zeros(shape=an_img.shape[:-1])
+            crop_indices = np.stack(np.where(np.all(an_img == crop, axis=-1))).T
+            weed_indices = np.stack(np.where(np.all(an_img == weed, axis=-1))).T
+            for indxs in crop_indices:
+                out_annotation[indxs[0]][indxs[1]] = 1
+            for indxs in weed_indices:
+                out_annotation[indxs[0]][indxs[1]] = 2
+            return cv2.imwrite(out_path, out_annotation.astype(np.int8))
+
+        move_segmentation_dataset(
+            self.data_processed_dir,
+            dataset_name,
+            train_images,
+            annotation_images,
+            train_dir,
+            annotation_dir,
+            annotation_preprocess_fn=_annotation_preprocess_fn,
+        )
+
+    def apple_segmentation_minnesota(self, dataset_name):
+        # Get all of the relevant data.
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        train_dir = os.path.join(dataset_dir, "train", "images")
+        train_images = sorted(get_file_list(train_dir))
+        masks_dir = os.path.join(dataset_dir, "train", "masks")
+        mask_images = sorted(get_file_list(masks_dir))
+
+        # Move the images to the new directory.
+        def _annotation_preprocess_fn(annotation_path, out_path):
+            mask = cv2.imread(annotation_path, cv2.IMREAD_GRAYSCALE)
+            ids = np.unique(mask)[1:]
+            masks = mask == ids[:, np.newaxis, np.newaxis]
+            masks = masks.astype(np.int32)
+            if len(masks) == 1:
+                mask = mask
+            elif len(masks) >= 2:
+                mask = np.logical_or(masks[0], masks[1])
+                if len(masks) > 2:
+                    for mask_ in masks[2:]:
+                        mask = np.logical_or(mask, mask_)
+            mask = mask.astype(np.int32)
+            return cv2.imwrite(out_path, mask)
+
+        move_segmentation_dataset(
+            self.data_processed_dir,
+            dataset_name,
+            train_images,
+            mask_images,
+            train_dir,
+            masks_dir,
+            annotation_preprocess_fn=_annotation_preprocess_fn,
+        )
+
+    def rice_seedling_segmentation(self, dataset_name, fix=False):
+        # Re-mapping labels to remove the `Background` class.
+        if fix:
+            data_dir = os.path.join(self.data_original_dir, dataset_name)
+            annotations = sorted(
+                [os.path.join(data_dir, "annotations", i) for i in os.listdir(os.path.join(data_dir, "annotations"))]
+            )
+            os.makedirs(os.path.join(data_dir, "new_annotations"))
+
+            # Create the remap.
+            for annotation in tqdm(annotations):
+                a = cv2.imread(annotation)
+                a[a == 2] = 0
+                a[a == 3] = 2
+                cv2.imwrite(
+                    os.path.join(data_dir, "new_annotations", os.path.basename(annotation)),
+                    a,
+                )
+            return
+
+        # Get all of the relevant data.
+        data_dir = os.path.join(self.data_original_dir, dataset_name)
+        images = sorted(glob.glob(os.path.join(data_dir, "image_*.jpg")))
+        labels = sorted(glob.glob(os.path.join(data_dir, "Label_*.png")))
+        images = [os.path.basename(p) for p in images]
+        labels = [os.path.basename(p) for p in labels]
+
+        # Move the images to the new directory.
+        move_segmentation_dataset(self.data_processed_dir, dataset_name, images, labels, data_dir, data_dir)
+
+    def sugarcane_damage_usa(self, dataset_name):
+        pass
+
+    def soybean_weed_uav_brazil(self, dataset_name):
+        pass
+
+    def plant_village_classification(self, dataset_name):
+        pass
+
+    def autonomous_greenhouse_regression(self, dataset_name):
+        # Get all of the data paths.
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        image_dir = os.path.join(dataset_dir, "RGBImages")
+        depth_dir = os.path.join(dataset_dir, "DepthImages")
+        with open(os.path.join(dataset_dir, "dataset.json"), "r") as f:
+            contents = json.load(f)
+
+        # Construct the output annotation JSON file.
+        out = []
+        for sample in contents["labels"].values():
+            out.append(
+                {
+                    "image": sample["rgb_image_path"],
+                    "depth_image": sample["depth_image_path"],
+                    "outputs": {
+                        "regression": sample["regression_outputs"],
+                        "classification": sample["classification_outputs"]["Variety"],
+                    },
+                }
+            )
+
+        # Copy the images over.
+        out_dir = os.path.join(self.data_processed_dir, dataset_name)
+        out_image_dir = os.path.join(out_dir, "images")
+        os.makedirs(out_image_dir, exist_ok=True)
+        out_depth_dir = os.path.join(out_dir, "depth_images")
+        os.makedirs(out_depth_dir, exist_ok=True)
+        for image in tqdm(os.listdir(image_dir), desc="Moving Images", file=sys.stdout):
+            shutil.copyfile(os.path.join(image_dir, image), os.path.join(out_image_dir, image))
+        for depth in tqdm(os.listdir(depth_dir), desc="Moving Depth Images", file=sys.stdout):
+            shutil.copyfile(os.path.join(depth_dir, depth), os.path.join(out_depth_dir, depth))
+
+        # Save the annotation file.
+        with open(os.path.join(out_dir, "annotations.json"), "w") as f:
+            json.dump(out, f)
+
+    def guava_disease_pakistan(self, dataset_name):
+        # Get all of the images.
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        classes = os.listdir(dataset_dir)
+        all_images = []
+        for cls in classes:
+            all_images.extend([os.path.join(dataset_dir, cls, i) for i in os.listdir(os.path.join(dataset_dir, cls))])
+
+        # Resize all of the images.
+        out_dir = os.path.join(self.data_processed_dir, dataset_name)
+        os.makedirs(out_dir, exist_ok=True)
+        for cls in classes:
+            os.makedirs(os.path.join(out_dir, cls), exist_ok=True)
+        for image in tqdm(all_images, "Resizing Images"):
+            out_image = image.replace("/original/", "/processed/")
+            im = cv2.imread(image, cv2.IMREAD_UNCHANGED)
+            im = cv2.resize(im, (im.shape[1] // 5, im.shape[0] // 5), cv2.INTER_LINEAR)
+            cv2.imwrite(out_image, im)
+
+    def apple_detection_spain(self, dataset_name):
+        # resize the dataset
+        resize = 1.0
+
+        # Read public_datasources.json to get class information
+        datasource_file = os.path.join(os.path.dirname(__file__), "../_assets/public_datasources.json")
+        with open(datasource_file) as f:
+            data = json.load(f)
+            category_info = data[dataset_name]["crop_types"]
+            labels_str = []
+            labels_ids = []
+            for info in category_info:
+                labels_str.append(category_info[info])
+                labels_ids.append(int(info))
+
+            name_converter = dict(zip(["Poma"], ["apple"]))  # src -> dst
+            label2id = dict(zip(labels_str, labels_ids))
+
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        ann_dir = os.path.join(dataset_dir, "preprocessed data/square_annotations1")
+
+        # Get image file and xml file
+        all_files = get_file_list(ann_dir)
+        anno_files = [os.path.join(ann_dir, x) for x in all_files if "xml" in x]
+        img_files = [x.replace(".xml", "hr.jpg").replace("square_annotations1", "images") for x in anno_files]
+
+        # Process annotation files
+        save_dir_anno = os.path.join(self.data_processed_dir, dataset_name, "annotations")
+        create_dir(save_dir_anno)
+        output_json_file = os.path.join(save_dir_anno, "instances.json")
+
+        # Process image files
+        output_img_path = os.path.join(self.data_processed_dir, dataset_name, "images")
+        create_dir(output_img_path)
+
+        general_info = {
+            "description": "KFuji RGB-DS database",
+            "url": "http://www.grap.udl.cat/en/publications/KFuji_RGBDS_database.html",
+            "version": "1.0",
+            "year": 2018,
+            "contributor": "Gené-Mola J, Vilaplana V, Rosell-Polo JR, Morros JR, Ruiz-Hidalgo J, Gregorio E",
+            "date_created": "2018/10/19",
+        }
+
+        convert_xmls_to_cocojson(
+            general_info,
+            annotation_paths=anno_files,
+            img_paths=img_files,
+            label2id=label2id,
+            name_converter=name_converter,
+            output_jsonpath=output_json_file,
+            output_imgpath=output_img_path,
+            extract_num_from_imgid=True,
+        )
+
+    def apple_detection_drone_brazil(self, dataset_name):
+        # Get the data directory and rename it if necessary.
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        if not os.path.exists(dataset_dir):
+            fallback = os.path.join(self.data_original_dir, "thsant-add256-68d2f88")  # noqa
+            if os.path.exists(fallback):
+                os.rename(fallback, dataset_dir)
+
+        # Get all of the images which have valid annotations.
+        with open(os.path.join(dataset_dir, "all.json"), "r") as f:
+            original_annotations = json.load(f)
+        valid_annotations = {k: v for k, v in original_annotations.items() if v != []}
+
+        # Construct the `images` part of the COCO JSON.
+        image_coco = []
+        image_id_map = {}
+        image_dir = os.path.join(dataset_dir, "images")
+        for idx, image_name in tqdm(
+            enumerate(valid_annotations.keys()),
+            desc="Parsing Images",
+            total=len(valid_annotations),
+        ):
+            height, width = cv2.imread(os.path.join(image_dir, image_name)).shape[:2]
+            image_coco.append({"file_name": image_name, "height": height, "width": width, "id": idx})
+            image_id_map[image_name] = idx
+
+        # Construct the `annotations` part of the COCO JSON.
+        annotation_idx = 0
+        annotation_coco = []
+        for image_name, annotation_list in valid_annotations.items():
+            for annotation in annotation_list:
+                # Coordinates are in form (center_x, center_y, radius). We convert
+                # these to (top left x, top left y, width, height)
+                x_c, y_c, r = annotation["cx"], annotation["cy"], annotation["r"]
+                x, y = x_c - r, y_c - r
+                w = h = r * 2
+                annotation_coco.append(
+                    {
+                        "area": w * h,
+                        "iscrowd": 0,
+                        "bbox": [x, y, w, h],
+                        "category_id": 1,
+                        "ignore": 0,
+                        "segmentation": 0,
+                        "image_id": image_id_map[image_name],
+                        "id": annotation_idx,
+                    }
+                )
+                annotation_idx += 1
+
+        # Set up the annotation dictionary.
+        category_info = [{"supercategory": "none", "id": 1, "name": "apple"}]
+        all_annotation_data = {
+            "images": image_coco,
+            "type": "instances",
+            "annotations": annotation_coco,
+            "categories": category_info,
+            "info": {
+                "description": "apple detection dataset with drone imagery",
+                "url": "https://github.com/thsant/add256/tree/zenodo-1.0",
+                "version": "1.0",
+                "year": 2021,
+                "contributor": "Thiago T. Santos and Luciano Gebler",
+                "date_created": "2021/10/2021",
+            },
+        }
+
+        # Recreate the dataset and zip it
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_img_dir = os.path.join(processed_dir, "images")
+        if os.path.exists(processed_dir):
+            shutil.rmtree(processed_dir)
+        os.makedirs(processed_dir, exist_ok=True)
+        os.makedirs(processed_img_dir, exist_ok=True)
+        for path in tqdm(valid_annotations.keys(), desc="Moving Images"):
+            full_path = os.path.join(image_dir, path)
+            shutil.copyfile(full_path, os.path.join(processed_img_dir, os.path.basename(path)))
+        with open(os.path.join(processed_dir, "annotations.json"), "w") as f:
+            json.dump(all_annotation_data, f)
+
+    def plant_doc_classification(self, dataset_name):
+        category_info = self.data_sources[dataset_name]["classes"]
+
+        # paths to original files
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        train_dir = os.path.join(dataset_dir, "train")
+        test_dir = os.path.join(dataset_dir, "test")
+
+        # make output dir
+        output = os.path.join(self.data_processed_dir, dataset_name)
+        os.makedirs(output)
+
+        for key in category_info:
+            category = category_info[key]
+
+            # make output dir for each crop type
+            output_catg_dir = os.path.join(output, category)
+            os.makedirs(output_catg_dir)
+
+            # put train and test images of same category into same folder
+            train_catg_dir = os.path.join(train_dir, category)
+            test_catg_dir = os.path.join(test_dir, category)
+
+            for img_name in get_file_list(train_catg_dir):
+                img = os.path.join(train_catg_dir, img_name)
+                shutil.copyfile(img, os.path.join(output_catg_dir, img_name))
+
+            if os.path.exists(test_catg_dir):
+                for img_name in get_file_list(test_catg_dir):
+                    img = os.path.join(test_catg_dir, img_name)
+                    shutil.copyfile(img, os.path.join(output_catg_dir, img_name))
+
+    def wheat_head_counting(self, dataset_name):
+        label2id = {"Wheat Head": 1}
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+        anno_files = [
+            os.path.join(dataset_dir, "competition_train.csv"),
+            os.path.join(dataset_dir, "competition_test.csv"),
+            os.path.join(dataset_dir, "competition_val.csv"),
+        ]
+
+        annotations = []
+        for anno_file in anno_files:
+            with open(anno_file, "r") as file:
+                reader = csv.reader(file)
+                for row in reader:
+                    img_path = os.path.join(dataset_dir, "images", row[0])
+                    anno = [img_path]
+                    bboxs = row[1].split(";")
+                    anno.append(len(bboxs))
+                    for bbox in bboxs:
+                        if bbox != "no_box":
+                            bbox = bbox.split(" ")
+                            bbox.append("1")
+                            anno.append(bbox)
+                    annotations.append(anno)
+
+        # Define path to processed annotation files
+        output_json_file = os.path.join(self.data_processed_dir, dataset_name, "annotations.json")
+
+        # Create directory for processed image files
+        output_img_path = os.path.join(self.data_processed_dir, dataset_name, "images")
+        create_dir(output_img_path)
+
+        general_info = {
+            "description": "Global Wheat Head Detection (GWHD) dataset",
+            "url": "http://www.global-wheat.com/",
+            "version": "1.0",
+            "year": 2021,
+            "contributor": "David, Etienne and Madec, Simon and Sadeghi-Tehran, Pouria and Aasen, Helge and Zheng, Bangyou and Liu, Shouyang and Kirchgessner, Norbert and Ishikawa, Goro and Nagasawa, Koichi and Badhon, Minhajul A and others",
+            "date_created": "2021/7/12",
+        }
+
+        convert_bbox_to_coco(
+            annotations,
+            label2id,
+            output_json_file,
+            output_img_path,
+            general_info,
+            resize=512 / 1024,
+        )
+
+    def peachpear_flower_segmentation(self, dataset_name):
+        # Create processed directories
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        os.makedirs(processed_dir, exist_ok=True)
+        processed_image_dir = os.path.join(processed_dir, "images")
+        os.makedirs(processed_image_dir, exist_ok=True)
+        processed_annotation_dir = os.path.join(processed_dir, "annotations")
+        os.makedirs(processed_annotation_dir, exist_ok=True)
+
+        dataset_dir = os.path.join(self.data_original_dir, dataset_name)
+
+        # Get image files
+        img_dirs = ["Peach", "Pear"]
+        img_paths = []
+        for img_dir in img_dirs:
+            img_paths += [
+                os.path.join(dataset_dir, img_dir, file_name)
+                for file_name in get_file_list(os.path.join(dataset_dir, img_dir))
+            ]
+
+        # Save all images as jpg in processed directory
+        for img_path in img_paths:
+            processed_path = os.path.join(processed_image_dir, img_path.split("/")[-1].replace(".bmp", ".jpg"))
+            img = cv2.imread(img_path)
+            cv2.imwrite(processed_path, img)
+
+        # Get annotation files
+        anno_dirs = ["PeachLabels", "PearLabels"]
+        anno_paths = []
+        for anno_dir in anno_dirs:
+            anno_paths += [
+                os.path.join(dataset_dir, anno_dir, file_name)
+                for file_name in get_file_list(os.path.join(dataset_dir, anno_dir))
+            ]
+
+        # Transform mask and save to processed directory
+        for anno_path in anno_paths:
+            img = cv2.imread(anno_path, cv2.IMREAD_GRAYSCALE)
+            img = np.where(img[:] == 255, 1, 0)
+            processed_path = os.path.join(processed_annotation_dir, anno_path.split("/")[-1])
+            cv2.imwrite(processed_path, img)
+
+    def ghai_romaine_detection(self, dataset_name):
+        # Create processed directories
+        original_dir = os.path.join(self.data_original_dir, dataset_name)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, "images")
+        os.makedirs(processed_image_dir, exist_ok=True)
+
+        # Move images
+        for image in glob.glob(os.path.join(original_dir, "*.jpg")):
+            shutil.move(image, processed_image_dir)
+        shutil.move(
+            os.path.join(original_dir, "coco.json"),
+            os.path.join(processed_dir, "annotations.json"),
+        )
+
+    def ghai_green_cabbage_detection(self, dataset_name):
+        # Create processed directories
+        original_dir = os.path.join(self.data_original_dir, dataset_name)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, "images")
+        os.makedirs(processed_image_dir, exist_ok=True)
+
+        # Move images
+        for image in glob.glob(os.path.join(original_dir, "*.jpg")):
+            shutil.move(image, processed_image_dir)
+        shutil.move(
+            os.path.join(original_dir, "coco.json"),
+            os.path.join(processed_dir, "annotations.json"),
+        )
+
+    def ghai_iceberg_lettuce_detection(self, dataset_name):
+        # Create processed directories
+        original_dir = os.path.join(self.data_original_dir, dataset_name)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, "images")
+        os.makedirs(processed_image_dir, exist_ok=True)
+
+        # Move images
+        for image in glob.glob(os.path.join(original_dir, "*.jpg")):
+            shutil.move(image, processed_image_dir)
+        shutil.move(
+            os.path.join(original_dir, "coco.json"),
+            os.path.join(processed_dir, "annotations.json"),
+        )
+
+    def riseholme_strawberry_classification_2021(self, dataset_name):
+        # Create processed data directory.
+        original_dir = os.path.join(self.data_original_dir, "Riseholme-2021-main", "Data")
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+
+        # Load all of the individual images and keep a mapping to their corresponding directory.
+        images = {
+            "anomalous": get_file_list(os.path.join(original_dir, "Anomalous")),
+            **{
+                dir_.lower(): get_file_list(os.path.join(original_dir, "Normal", dir_))
+                for dir_ in os.listdir(os.path.join(original_dir, "Normal"))
+            },
+        }
+
+        # Create the output file structure.
+        for class_name, image_set in images.items():
+            class_dir = os.path.join(processed_dir, class_name)
+            os.makedirs(class_dir, exist_ok=True)
+            for file in image_set:
+                shutil.copyfile(file, os.path.join(class_dir, os.path.basename(file)))
+
+    def ghai_broccoli_detection(self, dataset_name):
+        # Create processed directories
+        original_dir = os.path.join(self.data_original_dir, dataset_name)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, "images")
+        os.makedirs(processed_image_dir, exist_ok=True)
+
+        # Move images
+        for image in tqdm(glob.glob(os.path.join(original_dir, "*.jpg"))):
+            shutil.move(image, processed_image_dir)
+        shutil.move(
+            os.path.join(original_dir, "coco.json"),
+            os.path.join(processed_dir, "annotations.json"),
+        )
+
+    def ghai_strawberry_fruit_detection(self, dataset_name):
+        # Create processed directories
+        original_dir = os.path.join(self.data_original_dir, dataset_name)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, "images")
+        os.makedirs(processed_image_dir, exist_ok=True)
+
+        # Move images
+        for image in tqdm(glob.glob(os.path.join(original_dir, "*.jpg"))):
+            shutil.move(image, processed_image_dir)
+        shutil.move(
+            os.path.join(original_dir, "coco.json"),
+            os.path.join(processed_dir, "annotations.json"),
+        )
+
+    def vegann_multicrop_presence_segmentation(self, dataset_name):
+        # Create processed directories
+        original_dir = os.path.join(self.data_original_dir, dataset_name)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, "images")
+        os.makedirs(processed_image_dir, exist_ok=True)
+        processed_annotation_dir = os.path.join(processed_dir, "annotations")
+        os.makedirs(processed_annotation_dir, exist_ok=True)
+
+        # Move images
+        for image in tqdm(glob.glob(os.path.join(original_dir, "images", "*.png"))):
+            shutil.copyfile(image, os.path.join(processed_image_dir, os.path.basename(image)))
+
+        # Read annotations
+        for annotation_file in tqdm(glob.glob(os.path.join(original_dir, "annotations", "*.png"))):
+            annotation = cv2.imread(annotation_file, cv2.IMREAD_UNCHANGED)
+            annotation = np.where(annotation == 255, 1, 0)
+            cv2.imwrite(
+                os.path.join(processed_annotation_dir, os.path.basename(annotation_file)),
+                annotation,
+            )
+
+        # Read the CSV file containing the splits
+        split_csv = pd.read_csv(os.path.join(original_dir, "VegAnn_dataset.csv"), sep=";")
+
+        # Get the `Name` and `TVT-split{n}` columns for each n, and save the splits to a folder
+        splits_folder = os.path.join(processed_dir, ".splits")
+        os.makedirs(splits_folder, exist_ok=True)
+        column_pairs = [["Name", f"TVT-split{i}"] for i in range(1, 5 + 1)]
+
+        splits = {}
+        for column_pair in column_pairs:
+            columns = split_csv[column_pair]
+            train_images = columns[columns[column_pair[1]] == "Training"]["Name"]
+            test_images = columns[columns[column_pair[1]] == "Test"]["Name"]
+            splits[column_pair[1]] = {
+                "train": {os.path.join("images", i): os.path.join("annotations", i) for i in train_images},
+                "val": {},
+                "test": {os.path.join("images", i): os.path.join("annotations", i) for i in test_images},
+            }
+
+        # Save each split to a JSON file
+        for split_name, split in splits.items():
+            with open(os.path.join(splits_folder, f"{split_name}.json"), "w") as f:
+                json.dump(split, f)
+
+    def embrapa_wgisd_grape_detection(self, dataset_name):
+        """Preprocesses your grape dataset (Chardonnay, PinotGris, PinotNoir) from YOLO format to COCO format."""
+        base_path = os.path.join(self.data_original_dir)
+
+        # Get all the dataset folders (Chardonnay, PinotGris, PinotNoir)
+        dataset_folders = ['Chardonnay', 'PinotGris', 'PinotNoir']
+
+        # Prepare the COCO annotation dictionary
+        coco_annotation = {
+            "images": [],
+            "annotations": [],
+            "categories": [{
+                "id": 1,  # Category ID for the object (e.g., grape)
+                "name": "grape",  # You can change this to match your dataset's object
+                "supercategory": "fruit"
+            }]
+        }
+
+        annotation_id = 1
+        image_id = 1
+
+        # Create the output directories for processed images and annotations
+        output_base_dir = os.path.join(self.data_processed_dir, dataset_name)
+        output_img_dir = os.path.join(output_base_dir, 'images')
+        os.makedirs(output_img_dir, exist_ok=True)
+
+        # Iterate through each dataset folder (Chardonnay, PinotGris, PinotNoir)
+        for folder in dataset_folders:
+            images_dir = os.path.join(base_path, folder, 'images')
+            labels_dir = os.path.join(base_path, folder, 'labels')
+
+            # Check if the directories exist
+            if not os.path.exists(images_dir) or not os.path.exists(labels_dir):
+                print(f"Error: Images or labels directory does not exist in {images_dir} or {labels_dir}")
+                continue
+
+            # Create output directory for this folder inside the processed folder
+            output_folder_img_dir = os.path.join(output_img_dir, folder)
+            os.makedirs(output_folder_img_dir, exist_ok=True)
+
+            # Process all images and their corresponding YOLO labels
+            for img_file in tqdm(os.listdir(images_dir), desc=f"Processing images for {folder}"):
+                if img_file.endswith(('jpg', 'jpeg', 'png')):
+                    image_path = os.path.join(images_dir, img_file)
+                    label_file = img_file.replace('.jpg', '.txt').replace('.jpeg', '.txt').replace('.png', '.txt')
+                    label_path = os.path.join(labels_dir, label_file)
+
+                    # Read the image to get its dimensions
+                    img = cv2.imread(image_path)
+                    if img is None:
+                        print(f"Error: Could not open image file {image_path}")
+                        continue
+                    img_height, img_width = img.shape[:2]
+
+                    # Add image information to COCO structure
+                    image_info = {
+                        "file_name": f"{folder}/{img_file}",  # Include folder name in file path
+                        "height": img_height,
+                        "width": img_width,
+                        "id": image_id
+                    }
+                    coco_annotation['images'].append(image_info)
+
+                    # Read the YOLO label file and convert to COCO format
+                    if os.path.exists(label_path):
+                        with open(label_path, 'r') as f:
+                            lines = f.readlines()
+
+                        # Prepare annotations in COCO format
+                        for line in lines:
+                            elements = line.strip().split()
+                            class_id = int(elements[0])  # class_id from YOLO
+                            x_center, y_center, bbox_width, bbox_height = map(float, elements[1:])
+
+                            # Convert to COCO format bounding box
+                            bbox = yolo_to_coco_bbox(x_center, y_center, bbox_width, bbox_height, img_width, img_height)
+
+                            annotation = {
+                                "image_id": image_id,  # Reference to the image ID
+                                "bbox": bbox,  # COCO bounding box [xmin, ymin, width, height]
+                                "category_id": 1,  # Assuming a single category "grape"
+                                "id": annotation_id,  # Unique annotation ID
+                                "area": bbox[2] * bbox[3],  # width * height
+                                "iscrowd": 0,
+                                "segmentation": []
+                            }
+                            coco_annotation['annotations'].append(annotation)
+                            annotation_id += 1
+                    else:
+                        print(f"Warning: No label file found for {img_file}")
+
+                    # Copy the image to the processed directory
+                    shutil.copyfile(image_path, os.path.join(output_folder_img_dir, img_file))
+
+                    # Increment image ID
+                    image_id += 1
+
+        # Save the final COCO annotations to the correct path inside the processed folder
+        output_json_file = os.path.join(output_base_dir, 'annotations.json')
+        with open(output_json_file, 'w') as json_file:
+            json.dump(coco_annotation, json_file, indent=4)
+
+        print(f"COCO annotations saved to {output_json_file}")
+
+    def plant_doc_detection(self, dataset_name):
+        # Resize the dataset (if necessary)
+        resize = 1.0
+
+        # Read public_datasources.json to get class information
+        datasource_file = os.path.join(os.path.dirname(__file__), "../_assets/public_datasources.json")
+        with open(datasource_file) as f:
+            data = json.load(f)
+            category_info = data[dataset_name]['classes']  # This will give us the class information
+            labels_str = []
+            labels_ids = []
+            for info in category_info:
+                labels_str.append(category_info[info])
+                labels_ids.append(int(info))
+
+            # No name conversion in this case, unless you need to remap class names
+            name_converter = None
+            label2id = dict(zip(labels_str, labels_ids))  # Map class names to their respective IDs
+
+        # Set paths to dataset and annotations
+        dataset_dir = os.path.join(self.data_original_dir)
+        ann_dir = dataset_dir  # Both images and XMLs are in the same directory
+
+        # Get image file and xml file
+        all_files = os.listdir(ann_dir)
+        anno_files = [os.path.join(ann_dir, x) for x in all_files if x.endswith("xml")]
+        img_files = [x.replace(".xml", ".jpg") for x in anno_files]  # Assuming images are in JPG format
+
+        # # Process annotation files
+        # save_dir_anno = os.path.join(self.data_processed_dir, dataset_name, 'annotations')
+        # create_dir(save_dir_anno)
+        # output_json_file = os.path.join(save_dir_anno, 'instances.json')
+
+        # Process image files
+        output_img_path = os.path.join(self.data_processed_dir, dataset_name, 'images')
+        create_dir(output_img_path)
+
+        # General information for the COCO JSON format
+        general_info = {
+            "description": "PlantDoc Object Detection Dataset",
+            "url": "https://github.com/pratikkayal/PlantDoc-Object-Detection-Dataset",
+            "version": "1.0",
+            "year": 2024,
+            "contributor": "PlantDoc",
+            "date_created": "2024/10/17"
+        }
+
+        # Save the COCO JSON file directly in the processed directory
+        output_json_file = os.path.join(self.data_processed_dir, dataset_name, 'annotations.json')
+
+        # Convert the XML annotations to COCO format using your existing function
+        convert_xmls_to_cocojson(
+            general_info=general_info,
+            annotation_paths=anno_files,
+            img_paths=img_files,
+            label2id=label2id,
+            name_converter=name_converter,
+            output_jsonpath=output_json_file,
+            output_imgpath=output_img_path,
+            extract_num_from_imgid=False
+        )
+
+        print(f"Preprocessing completed! Annotations saved to {output_json_file}") 
+
+    def growliflower_cauliflower_segmentation(self, dataset_name):
+        """
+        Preprocess the cauliflower dataset by merging mask images and moving them to the new directory.
+        
+        Args:
+            self: Instance of the class where this function belongs.
+            dataset_name (str): Name of the dataset.
+        """
+        # Paths to the dataset
+        dataset_dir = os.path.join(self.data_original_dir)
+        images_dir = os.path.join(dataset_dir, 'images')  # Folder with original images
+        masks_dir = os.path.join(dataset_dir, 'annotations')  # Folder with the mask subfolders
+        
+        # Output directory where preprocessed data will be stored
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, 'images')
+        processed_annotation_dir = os.path.join(processed_dir, 'annotations')
+        
+        # Create output directories if they don't exist
+        os.makedirs(processed_image_dir, exist_ok=True)
+        os.makedirs(processed_annotation_dir, exist_ok=True)
+        
+        # Define the mask class mappings (assign unique labels to each type)
+        mask_classes = {
+            'maskLeaves': 1,
+            'maskPlants': 2,
+            'maskStems': 3,
+            'maskVoid': 4
+        }
+
+        # Get the list of image filenames
+        image_files = sorted(os.listdir(images_dir))
+        
+        for idx, image_file in enumerate(tqdm(image_files, desc="Processing Cauliflower Dataset")):
+            image_name = os.path.splitext(image_file)[0]  # Get the base image name without extension
+            image_path = os.path.join(images_dir, image_file)
+
+            # Find the corresponding masks for each class
+            mask_paths = {}
+            for mask_class in mask_classes.keys():
+                # Build mask filenames with both naming conventions
+                mask_filename_no_plants = f"{image_name}_Label_NoPlants_{mask_class}.png"
+                mask_filename = f"{image_name}_Label_{mask_class}.png"
+                
+                # Set the correct mask path
+                mask_path_no_plants = os.path.join(masks_dir, mask_class, mask_filename_no_plants)
+                mask_path = os.path.join(masks_dir, mask_class, mask_filename)
+                
+                if os.path.exists(mask_path_no_plants):
+                    mask_paths[mask_class] = mask_path_no_plants
+                else:
+                    mask_paths[mask_class] = mask_path
+
+            # Load the original image to get its shape (we need the height and width)
+            orig_image = cv2.imread(image_path)
+            if orig_image is None:
+                print(f"Warning: Unable to read image {image_path}")
+                continue
+            image_shape = orig_image.shape[:2]  # (height, width)
+
+            # Initialize the merged mask as a blank image (background class 0)
+            merged_mask = np.zeros(image_shape, dtype=np.uint8)
+
+            # Visualize and debug individual masks, if needed
+            individual_masks = {}
+
+            for mask_name, class_value in mask_classes.items():
+                mask = cv2.imread(mask_paths[mask_name], cv2.IMREAD_GRAYSCALE)
+                if mask is None:
+                    print(f"Warning: Unable to read mask {mask_paths[mask_name]}")
+                    continue
+                individual_masks[mask_name] = mask  # Store for visualization
+                
+                # Debugging: Check unique values in each mask
+                print(f"{mask_name} unique values for {image_name}: {np.unique(mask)}")
+                
+                # Binarize the mask for leaf mask based on the value 60
+                if mask_name == 'maskLeaves':
+                    binarized_mask = np.where(mask == 60, 1, 0).astype(np.uint8)
+                else:
+                    # For other masks, threshold based on the condition > 60
+                    binarized_mask = np.where(mask > 60, 1, 0).astype(np.uint8)
+                # Merge logic: void mask only applies where there's no other mask
+                if class_value == 4:  # Void class
+                    merged_mask[(merged_mask == 0) & (binarized_mask > 0)] = class_value
+                else:
+                    merged_mask[(binarized_mask > 0)] = class_value
+
+            # Debugging Step: Check unique values in the merged mask
+            unique_values = np.unique(merged_mask)
+            print(f"Image {idx+1}/{len(image_files)}: {image_file} - Unique mask values: {unique_values}")
+
+            # Save the original image and the merged mask to the new directory
+            out_image_path = os.path.join(processed_image_dir, image_file)
+            out_mask_path = os.path.join(processed_annotation_dir, f"{image_name}.png")
+
+            cv2.imwrite(out_image_path, orig_image)  
+            cv2.imwrite(out_mask_path, merged_mask)  
+
+    def strawberry_detection_2023(self, dataset_name):
+        # Create processed directories
+        original_dir = os.path.join(self.data_original_dir)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, 'images')
+        os.makedirs(processed_image_dir, exist_ok = True)
+        # Move images
+        for image in tqdm(glob.glob(os.path.join(original_dir, '*.jpg'))):
+            shutil.move(image, processed_image_dir)
+        shutil.move(os.path.join(original_dir, 'coco.json'),
+                    os.path.join(processed_dir, 'annotations.json'))
+        
+    def strawberry_detection_2022(self, dataset_name):
+        # Create processed directories
+        original_dir = os.path.join(self.data_original_dir)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, 'images')
+        os.makedirs(processed_image_dir, exist_ok = True)
+        # Move images
+        for image in tqdm(glob.glob(os.path.join(original_dir, '*.jpg'))):
+            shutil.move(image, processed_image_dir)
+        shutil.move(os.path.join(original_dir, 'coco.json'),
+                    os.path.join(processed_dir, 'annotations.json'))
+        
+    def almond_harvest_2021(self, dataset_name):
+        # Create processed directories
+        original_dir = os.path.join(self.data_original_dir)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, 'images')
+        os.makedirs(processed_image_dir, exist_ok = True)
+        # Move images
+        for image in tqdm(glob.glob(os.path.join(original_dir, '*.jpg'))):
+            shutil.move(image, processed_image_dir)
+        shutil.move(os.path.join(original_dir, 'coco.json'),
+                    os.path.join(processed_dir, 'annotations.json'))
+
+    def almond_bloom_2023(self, dataset_name):
+        # Create processed directories
+        original_dir = os.path.join(self.data_original_dir)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, 'images')
+        os.makedirs(processed_image_dir, exist_ok = True)
+        # Move images
+        for image in tqdm(glob.glob(os.path.join(original_dir, '*.jpg'))):
+            shutil.move(image, processed_image_dir)
+        shutil.move(os.path.join(original_dir, 'coco.json'),
+                    os.path.join(processed_dir, 'annotations.json'))
+        
+    def gemini_flower_detection_2022(self, dataset_name):
+        original_dir = os.path.join(self.data_original_dir)
+        print(original_dir)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, 'images')
+        os.makedirs(processed_image_dir, exist_ok = True)
+        # Move images
+        for image in tqdm(glob.glob(os.path.join(original_dir, 'images', '*.jpg'))):
+            shutil.move(image, processed_image_dir)
+        shutil.move(os.path.join(original_dir, 'coco.json'),
+                    os.path.join(processed_dir, 'annotations.json'))
+        
+    def gemini_leaf_detection_2022(self, dataset_name):
+        original_dir = os.path.join(self.data_original_dir)
+        print(original_dir)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, 'images')
+        os.makedirs(processed_image_dir, exist_ok = True)
+        # Move images
+        for image in tqdm(glob.glob(os.path.join(original_dir, 'images', '*.jpg'))):
+            shutil.move(image, processed_image_dir)
+        shutil.move(os.path.join(original_dir, 'coco.json'),
+                    os.path.join(processed_dir, 'annotations.json'))
+    
+    def gemini_pod_detection_2022(self, dataset_name):
+        original_dir = os.path.join(self.data_original_dir)
+        print(original_dir)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, 'images')
+        os.makedirs(processed_image_dir, exist_ok = True)
+        # Move images
+        for image in tqdm(glob.glob(os.path.join(original_dir, 'images', '*.jpg'))):
+            shutil.move(image, processed_image_dir)
+        shutil.move(os.path.join(original_dir, 'coco.json'),
+                    os.path.join(processed_dir, 'annotations.json'))
+        
+    def gemini_plant_detection_2022(self, dataset_name):
+        original_dir = os.path.join(self.data_original_dir)
+        print(original_dir)
+        processed_dir = os.path.join(self.data_processed_dir, dataset_name)
+        processed_image_dir = os.path.join(processed_dir, 'images')
+        os.makedirs(processed_image_dir, exist_ok = True)
+        # Move images
+        for image in tqdm(glob.glob(os.path.join(original_dir, 'images', '*.jpg'))):
+            shutil.move(image, processed_image_dir)
+        shutil.move(os.path.join(original_dir, 'coco.json'),
+                    os.path.join(processed_dir, 'annotations.json'))
+
+    def paddy_disease_classification(self, dataset_name):
+        pass
+    
+    def onion_leaf_classification(self, dataset_name):
+        pass
+
+    def chilli_leaf_classification(self, dataset_name):
+        pass
+
+    def orange_leaf_disease_classification(self, dataset_name):
+        pass
+
+    def papaya_leaf_disease_classification(self, dataset_name):
+        pass
+
+    def blackgram_plant_leaf_disease_classification(self, dataset_name):
+        pass
+
+    def arabica_coffee_leaf_disease_classification(self, dataset_name):
+        pass
+
+    def banana_leaf_disease_classification(self, dataset_name):
+        pass
+
+    def coconut_tree_disease_classification(self, dataset_name):
+        pass
+
+    def rice_leaf_disease_classification(self, dataset_name):
+        pass
+
+    def tea_leaf_disease_classification(self, dataset_name):
+        pass
+
+    def betel_leaf_disease_classification(self, dataset_name):
+        pass
+
+    def java_plum_leaf_disease_classification(self, dataset_name):
+        pass
+
+    def sunflower_disease_classification(self, dataset_name):
+        pass
+
+    def cucumber_disease_classification(self, dataset_name):
+        pass
+
+    def soybean_insect_classification(self, dataset_name):
+        pass
+
+if __name__ == '__main__':
+    # Initialize program arguments.
+    ap = argparse.ArgumentParser()
+    ap.add_argument(
+        "--data_dir",
+        type=str,
+        default="../../data_new",
+        help="The directory containing two sub-directories, " "`original` and `processed`, with the data.",
+    )
+    ap.add_argument("--dataset", type=str, help="The dataset to process.")
+    args = ap.parse_args()
+
+    # Execute the preprocessing.
+    p = PublicDataPreprocessor(os.path.abspath(args.data_dir))
+    print("Processing dataset")
+    p.preprocess(args.dataset)
+    print("Converting dataset")
+    # os.chdir(f'{args.data_dir}/processed')
+    # os.system(f'zip -r {args.dataset}.zip {args.dataset} -x ".*" -x "__MACOSX"')

AgML/source/agml/_internal/process_utils.py

@@ -0,0 +1,576 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Preprocess utilities for the code in `preprocess.py`.
+"""
+
+import os
+import sys
+import csv
+import json
+import shutil
+from typing import Dict, List
+import xml.etree.ElementTree as ET
+
+import cv2
+import numpy as np
+
+from PIL import Image
+from skimage import measure
+from shapely.geometry import Polygon, MultiPolygon
+
+from agml.utils.logging import tqdm
+
+
+def read_txt_file(file_name, delimiter=" ", header=False):
+    with open(file_name, newline="\n") as txt_file:
+        txt_reader = csv.reader(txt_file, delimiter=delimiter)
+        txt_lines = []
+        headline = None
+        for idx, line in enumerate(txt_reader):
+            if header and idx == 0:
+                headline = line
+                continue
+            line = [x.strip() for x in line if x.strip()]  # To remove blank elements
+            txt_lines.append(line)
+
+        return txt_lines, headline
+
+
+def get_label2id(labels_str: str) -> Dict[str, int]:
+    """id is 1 start"""
+    labels_ids = list(range(1, len(labels_str) + 1))
+    return dict(zip(labels_str, labels_ids))
+
+
+def get_annpaths(
+    ann_dir_path: str = None, ann_ids_path: str = None, ext: str = "", annpaths_list_path: str = None
+) -> List[str]:
+    # If wanting to use annotation paths list
+    if annpaths_list_path is not None:
+        with open(annpaths_list_path, "r") as f:
+            ann_paths = f.read().split()
+        return ann_paths
+
+    # If wanting to use annotation ids list
+    ext_with_dot = "." + ext if ext != "" else ""
+    with open(ann_ids_path, "r") as f:
+        ann_ids = f.read().split()
+    ann_paths = [os.path.join(ann_dir_path, aid + ext_with_dot) for aid in ann_ids]
+    return ann_paths
+
+
+def get_image_info_from_annoline(annotation_root, idx, resize=1.0, add_foldername=False):
+    filename = annotation_root[0].split("/")[-1]
+    try:
+        img = cv2.imread(annotation_root[0])
+
+        if resize != 1.0:
+            dsize = (int(img.shape[1] * resize), int(img.shape[0] * resize))
+            img = cv2.resize(img, dsize)
+
+        size = img.shape
+        width = size[1]
+        height = size[0]
+
+        if add_foldername:
+            filename = "{folder}_{img_name}".format(
+                folder=annotation_root[0].split("/")[-2], img_name=annotation_root[0].split("/")[-1]
+            )
+
+        image_info = {
+            "file_name": filename,
+            "height": height,
+            "width": width,
+            "id": idx,  # Use image order
+        }
+
+    except Exception as e:
+        print(e)
+        print("Cannot open {file}".format(file=annotation_root[0]))
+        image_info = None
+        img = None
+
+    return image_info, img
+
+
+def get_image_info(annotation_root, idx, resize=1.0, add_foldername=False):
+    path = annotation_root.findtext("path")
+    if path is None or True:
+        filename = annotation_root.findtext("filename")
+    else:
+        filename = os.path.basename(path)
+
+    try:
+        img = cv2.imread(filename)
+
+        if resize != 1.0:
+            dsize = [int(img.shape[1] * resize), int(img.shape[0] * resize)]
+            img = cv2.resize(img, dsize)
+
+        size = img.shape
+        width = size[1]
+        height = size[0]
+
+        if add_foldername:
+            filename = "{folder}_{img_name}".format(folder=filename.split("/")[-2], img_name=filename.split("/")[-1])
+
+        image_info = {
+            "file_name": filename.split("/")[-1],
+            "height": height,
+            "width": width,
+            "id": idx,  # Use image order
+        }
+
+    except Exception as e:
+        print(e)
+        print("Cannot open {file}".format(file=annotation_root[0]))
+        image_info = None
+        img = None
+
+    return image_info, img
+
+
+# Sourced from https://github.com/roboflow-ai/voc2coco.git
+def get_coco_annotation_from_annoline(obj, resize=1.0):
+    # Try the sub-label first
+    category_id = int(obj[4])
+    xmin = int(float(obj[0]) * resize)
+    ymin = int(float(obj[1]) * resize)
+    xmax = int(float(obj[2]) * resize)
+    ymax = int(float(obj[3]) * resize)
+    assert xmax > xmin and ymax > ymin, f"Box size error !: (xmin, ymin, xmax, ymax): {xmin, ymin, xmax, ymax}"
+    o_width = xmax - xmin + 1
+    o_height = ymax - ymin + 1
+    return {
+        "area": o_width * o_height,
+        "iscrowd": 0,
+        "bbox": [xmin, ymin, o_width, o_height],
+        "category_id": category_id,
+        "ignore": 0,
+        "segmentation": [],  # This script is not for segmentation
+    }
+
+
+def get_coco_annotation_from_obj(obj, label2id, name_converter=None):
+    # Try to sub-label first
+    label = obj.findtext("subname")
+    if label is None:
+        label = obj.findtext("name")
+        if name_converter:
+            if label in name_converter:
+                label = name_converter[label]
+        if label not in label2id:
+            return []
+    category_id = label2id[label]
+
+    bndbox = obj.find("bndbox")
+    if bndbox is None:
+        bndbox = obj.find("bbox")
+        if bndbox is None:
+            return None
+
+    xmin = int(float(bndbox.findtext("xmin"))) - 1
+    ymin = int(float(bndbox.findtext("ymin"))) - 1
+    xmax = int(float(bndbox.findtext("xmax")))
+    ymax = int(float(bndbox.findtext("ymax")))
+    assert xmax > xmin and ymax > ymin, f"Box size error !: (xmin, ymin, xmax, ymax): {xmin, ymin, xmax, ymax}"
+    o_width = xmax - xmin
+    o_height = ymax - ymin
+    return {
+        "area": o_width * o_height,
+        "iscrowd": 0,
+        "bbox": [xmin, ymin, o_width, o_height],
+        "category_id": category_id,
+        "ignore": 0,
+        "segmentation": [],  # This script is not for segmentation
+    }
+
+
+def convert_bbox_to_coco(
+    annotation: List[str],
+    label2id: Dict[str, int],
+    output_jsonpath: str,
+    output_imgpath: str,
+    general_info,
+    image_id_list=None,
+    bnd_id_list=None,
+    get_label_from_folder=False,
+    resize=1.0,
+    add_foldername=False,
+    extract_num_from_imgid=False,
+):
+    """
+    Annotation Format
+    -----------------
+    "image name" "the number of bounding boxes(bb)" "x1" "y1" "x2" "y2" "label" "score" "x1" "y1" "x2" "y2" ...
+
+    For example, the following line:
+
+    TRAIN_RGB/n12710693_12225.png 5 515 68 759 285 2 1.000 624 347 868 582 2 1.000 480 488 693 712 2 1.000 44 433 268 657 2 1.000 112 198 342 401 2 1.000
+
+    will become:
+
+    image name=TRAIN_RGB/n12710693_12225.png
+    the number of bb=5
+    x1=515
+    y1=68
+    x2=759
+    y2=285
+    label=2 "0=background, 1=capsicum, 2=rockmelon..."
+    score=1.000
+
+    Reference: https://drive.google.com/drive/folders/1CmsZb1caggLRN7ANfika8WuPiywo4mBb
+    """
+    output_json_dict = {"images": [], "type": "instances", "annotations": [], "categories": [], "info": general_info}
+
+    # TODO: Use multi thread to boost up the speed
+    print("Converting annotations into COCO JSON and process the images")
+    for img_idx, anno_line in enumerate(tqdm(annotation)):
+        if image_id_list:
+            img_unique_id = image_id_list[img_idx]
+        else:
+            if extract_num_from_imgid:
+                filename = anno_line[0].split("/")[-1]
+                img_unique_id = int("".join(filter(str.isdigit, filename)))
+            else:
+                img_unique_id = img_idx + 1
+
+        img_info, img = get_image_info_from_annoline(
+            annotation_root=anno_line, idx=img_unique_id, resize=resize, add_foldername=add_foldername
+        )
+
+        if img_info:
+            output_json_dict["images"].append(img_info)
+
+            bbox_cnt = int(anno_line[1])
+            if bbox_cnt > 0:
+                ann_reshape = np.reshape(anno_line[2:], (bbox_cnt, -1))
+                for bnd_idx, obj in enumerate(ann_reshape):
+                    if get_label_from_folder:
+                        # Change label based on folder
+                        try:
+                            category_name = anno_line[0].split("/")[-3]
+                            if category_name not in label2id:
+                                raise
+                        except:
+                            try:
+                                category_name = anno_line[0].split("/")[-2]
+                                if category_name not in label2id:
+                                    raise
+                            except Exception as e:
+                                raise e
+
+                        if len(obj) < 5:
+                            obj = np.append(obj, label2id[category_name])
+                        else:
+                            obj[4] = label2id[category_name]
+                    else:
+                        pass
+
+                    try:
+                        ann = get_coco_annotation_from_annoline(obj=obj, resize=resize)
+                    except:
+                        ann = None
+
+                    if ann:
+                        if bnd_id_list:
+                            bnd_idx = bnd_id_list[img_idx][bnd_idx]
+                        else:
+                            bnd_idx + 1  # noqa
+                        ann.update({"image_id": img_info["id"], "id": bnd_idx})
+                        output_json_dict["annotations"].append(ann)
+
+            img_name = img_info["file_name"]
+            dest_path = os.path.join(output_imgpath, img_name)
+            try:
+                if resize == 1.0:
+                    shutil.copyfile(anno_line[0], dest_path)
+                else:
+                    cv2.imwrite(dest_path, img)
+            except:  # Cannot copy the image file
+                pass
+
+        else:  # Not valid image => Delete from annotation
+            pass
+
+    for label, label_id in label2id.items():
+        category_info = {"supercategory": "none", "id": label_id, "name": label}
+        output_json_dict["categories"].append(category_info)
+
+    with open(output_jsonpath, "w") as f:
+        output_json = json.dumps(output_json_dict)
+        f.write(output_json)
+
+    return output_json_dict
+
+
+def convert_xmls_to_cocojson(
+    general_info,
+    annotation_paths: List[str],
+    img_paths: List[str],
+    label2id: Dict[str, int],
+    name_converter,
+    output_jsonpath: str,
+    output_imgpath: str,
+    extract_num_from_imgid: bool = True,
+):
+    """Converts the XML annotation format into COCO JSON."""
+    output_json_dict = {"images": [], "type": "instances", "annotations": [], "categories": [], "info": general_info}
+    bnd_id = 1  # START_BOUNDING_BOX_ID, TODO input as args ?
+    print("Start converting !")
+    for img_idx, a_path in enumerate(tqdm(annotation_paths)):
+        # Read annotation xml
+        ann_tree = ET.parse(a_path)
+        ann_root = ann_tree.getroot()
+
+        if extract_num_from_imgid:
+            filename = a_path.split("/")[-1]
+            img_unique_id = int("".join(filter(str.isdigit, filename)))
+        else:
+            img_unique_id = img_idx + 1
+
+        if len(img_paths) == len(annotation_paths):
+            ann_root.find("filename").text = img_paths[img_idx]
+
+        img_info, img = get_image_info(annotation_root=ann_root, idx=img_unique_id, resize=1.0, add_foldername=False)
+        output_json_dict["images"].append(img_info)
+
+        for obj in ann_root.findall("object"):
+            ann = get_coco_annotation_from_obj(obj=obj, label2id=label2id, name_converter=name_converter)
+            if ann:
+                ann.update({"image_id": img_info["id"], "id": bnd_id})
+                output_json_dict["annotations"].append(ann)
+                bnd_id = bnd_id + 1
+
+        # Process images
+        img_name = img_info["file_name"]
+        dest_path = os.path.join(output_imgpath, img_name)
+        try:
+            cv2.imwrite(dest_path, img)
+        except:  # Cannot copy the image file
+            pass
+
+    for label, label_id in label2id.items():
+        category_info = {"supercategory": "none", "id": label_id, "name": label}
+        output_json_dict["categories"].append(category_info)
+
+    with open(output_jsonpath, "w") as f:
+        output_json = json.dumps(output_json_dict)
+        f.write(output_json)
+
+
+# Reference: https://www.immersivelimit.com/create-coco-annotations-from-scratch
+def create_sub_masks(mask_image):
+    width, height = mask_image.size
+
+    # Initialize a dictionary of sub-masks indexed by RGB colors
+    sub_masks = {}
+    for x in range(width):
+        for y in range(height):
+            # Get the RGB values of the pixel
+            pixel = mask_image.getpixel((x, y))[:3]
+
+            # If the pixel is not black...
+            if pixel != (0, 0, 0):
+                # Check to see if we've created a sub-mask...
+                pixel_str = str(pixel)
+                sub_mask = sub_masks.get(pixel_str)
+                if sub_mask is None:
+                    # Create a sub-mask (one bit per pixel) and add to the dictionary
+                    # Note: we add 1 pixel of padding in each direction
+                    # because the contours module doesn't handle cases
+                    # where pixels bleed to the edge of the image
+                    sub_masks[pixel_str] = Image.new("1", (width + 2, height + 2))
+
+                # Set the pixel value to 1 (default is 0), accounting for padding
+                sub_masks[pixel_str].putpixel((x + 1, y + 1), 1)
+
+    return sub_masks
+
+
+# Reference: https://www.immersivelimit.com/create-coco-annotations-from-scratch
+def create_sub_mask_annotation(sub_mask, image_id, category_id, annotation_id, is_crowd):
+    # Find contours (boundary lines) around each sub-mask
+    # Note: there could be multiple contours if the object
+    # is partially occluded. (E.g. an elephant behind a tree)
+    contours = measure.find_contours(sub_mask, 0.5, positive_orientation="low")
+
+    segmentations = []
+    polygons = []
+    for contour in contours:
+        # Flip from (row, col) representation to (x, y)
+        # and subtract the padding pixel
+        for i in range(len(contour)):
+            row, col = contour[i]
+            contour[i] = (col - 1, row - 1)
+
+        # Make a polygon and simplify it
+        poly = Polygon(contour)
+        poly = poly.simplify(1.0, preserve_topology=False)
+        polygons.append(poly)
+        segmentation = np.array(poly.exterior.coords).ravel().tolist()
+        segmentations.append(segmentation)
+
+    # Combine the polygons to calculate the bounding box and area
+    multi_poly = MultiPolygon(polygons)
+    x, y, max_x, max_y = multi_poly.bounds
+    width = max_x - x
+    height = max_y - y
+    bbox = (x, y, width, height)
+    area = multi_poly.area
+
+    annotation = {
+        "segmentation": segmentations,
+        "iscrowd": is_crowd,
+        "image_id": image_id,
+        "category_id": category_id,
+        "id": annotation_id,
+        "bbox": bbox,
+        "area": area,
+    }
+
+    return annotation
+
+
+def create_sub_mask_annotation_per_bbox(sub_mask, image_id, category_id, annotation_id, is_crowd):
+    # Find contours (boundary lines) around each sub-mask
+    # Note: there could be multiple contours if the object
+    # is partially occluded. (E.g. an elephant behind a tree)
+    sub_mask_np = np.array(sub_mask)
+    contours = measure.find_contours(sub_mask_np, 0.5, positive_orientation="low")
+
+    segmentations = []
+    polygons = []
+    annotations = []
+    for idx, contour in enumerate(contours):
+        # Flip from (row, col) representation to (x, y)
+        # and subtract the padding pixel
+        for i in range(len(contour)):
+            row, col = contour[i]
+            contour[i] = (col - 1, row - 1)
+
+        # Make a polygon and simplify it
+        poly = Polygon(contour)
+        poly = poly.simplify(1.0, preserve_topology=False)
+        polygons.append(poly)
+        segmentation = np.array(poly.exterior.coords).ravel().tolist()
+        segmentations.append(segmentation)
+
+        if poly.area > 0:
+            # Combine the polygons to calculate the bounding box and area
+            multi_poly = MultiPolygon([poly])
+            x, y, max_x, max_y = multi_poly.bounds
+            width = max_x - x
+            height = max_y - y
+            bbox = (x, y, width, height)
+            area = multi_poly.area
+
+            annotations.append(
+                {
+                    "segmentation": [segmentation],
+                    "iscrowd": is_crowd,
+                    "image_id": image_id,
+                    "category_id": category_id,
+                    "id": annotation_id + idx,
+                    "bbox": bbox,
+                    "area": area,
+                }
+            )
+
+    return annotations
+
+
+def mask_annotation_per_bbox(anno_line, image_id, category_id, annotation_id, is_crowd):
+    # Find contours (boundary lines) around each sub-mask
+    # Note: there could be multiple contours if the object
+    # is partially occluded. (E.g. an elephant behind a tree)
+    segmentations = []
+    polygons = []
+    annotations = []
+    mask_data = json.loads(anno_line[5])
+
+    if len(mask_data["all_points_x"]) > 3:
+        # Flip from (row, col) representation to (x, y)
+        # and subtract the padding pixel
+        contour = []
+        for i in range(len(mask_data["all_points_x"])):
+            contour.append([int(mask_data["all_points_x"][i]), int(mask_data["all_points_y"][i])])
+
+        contour = np.array(contour)
+        # Make a polygon and simplify it
+        poly = Polygon(contour)
+        poly = poly.simplify(1.0, preserve_topology=False)
+        polygons.append(poly)
+        # Only need the contours, not the polygon itself:
+        # segmentation = np.array(poly.exterior.coords).ravel().tolist()
+        segmentation = contour.ravel().tolist()
+
+        segmentations.append(segmentation)
+
+        if poly.area > 0:
+            # Combine the polygons to calculate the bounding box and area
+            x, y, max_x, max_y = poly.bounds
+            width = max_x - x
+            height = max_y - y
+            bbox = (x, y, width, height)
+            area = poly.area
+
+            annotation = {
+                "segmentation": [segmentation],
+                "iscrowd": is_crowd,
+                "image_id": image_id,
+                "category_id": category_id,
+                "id": annotation_id,
+                "bbox": bbox,
+                "area": area,
+            }
+            annotations.append(annotation)
+
+    return annotations
+
+
+def move_segmentation_dataset(
+    out_dir,
+    dataset_name,
+    train_images,
+    annotation_images,
+    train_dir,
+    annotation_dir,
+    train_preprocess_fn=None,
+    annotation_preprocess_fn=None,
+):
+    """Moves segmentation images and annotations to a new location."""
+    processed_dir = os.path.join(out_dir, dataset_name)
+    os.makedirs(processed_dir, exist_ok=True)
+    processed_image_dir = os.path.join(processed_dir, "images")
+    os.makedirs(processed_image_dir, exist_ok=True)
+    processed_annotation_dir = os.path.join(processed_dir, "annotations")
+    os.makedirs(processed_annotation_dir, exist_ok=True)
+    for image_path, annotation_path in zip(
+        tqdm(train_images, desc="Processing Images", file=sys.stdout), annotation_images
+    ):
+        orig_image_path = os.path.join(train_dir, image_path)
+        orig_annotation_path = os.path.join(annotation_dir, annotation_path)
+        out_image_path = os.path.join(processed_image_dir, image_path)
+        out_label_path = os.path.join(processed_annotation_dir, annotation_path)
+        if train_preprocess_fn is None:
+            shutil.copyfile(orig_image_path, out_image_path)
+        else:
+            train_preprocess_fn(orig_image_path, out_image_path)
+        if annotation_preprocess_fn is None:
+            shutil.copyfile(orig_annotation_path, out_label_path)
+        else:
+            annotation_preprocess_fn(orig_annotation_path, out_label_path)

AgML/source/agml/_internal/s3internal.py

@@ -0,0 +1,191 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Internal class for interacting with the AgML public dataset S3 bucket.
+"""
+
+import json
+import os
+import sys
+import warnings
+import zipfile
+
+import boto3
+import botocore.exceptions
+
+from agml.utils.logging import tqdm
+
+
+class InternalAgMLS3API(object):
+    """
+    Internal API for interaction with the dataset S3 bucket.
+
+    *Note*: This class should only be used by the AgML developers.
+    """
+
+    def __init__(self):
+        # Read in metadata for data sources file
+        self.data_srcs_path = os.path.join(
+            os.path.dirname(os.path.dirname(__file__)),
+            "_assets",
+            "public_datasources.json",
+        )
+        with open(self.data_srcs_path) as f:
+            self.data_srcs = json.load(f)
+
+        # Define s3 bucket URI
+        self.agdata_s3_uri = "https://s3.us-west-1.amazonaws.com/agdata-data/"
+
+        # Initialize attribute for storing dataset download path
+        self.dataset_download_path = None
+
+    @property
+    def data_sources(self):
+        """Returns a list of AgML public data sources."""
+        return list(self.data_srcs.keys())
+
+    def upload_dataset(self, dataset_name, dataset_dir):
+        """
+        Uploads dataset to agdata-data s3 file storage.
+
+        Parameters
+        ----------
+        dataset_name : str
+            name of dataset (without '.zip') -- for list of datasets run self.data_srcs.keys()
+        dataset_dir : str
+            path to directory where dataset is stored
+        """
+        # Establish connection with s3 via boto
+        self.s3 = boto3.client("s3")
+
+        # Setup progress bar
+        self.pg = tqdm(
+            total=os.stat(os.path.abspath(os.path.join(dataset_dir, dataset_name + ".zip"))).st_size,
+            file=sys.stdout,
+            desc=f"Uploading {dataset_name}",
+        )
+
+        # Upload data to agdata-data bucket
+        try:
+            with open(os.path.join(dataset_dir, dataset_name + ".zip"), "rb") as data:
+                self.s3.upload_fileobj(
+                    Fileobj=data,
+                    Bucket="agdata-data",
+                    Key="datasets/" + dataset_name + ".zip",
+                    Callback=lambda x: self.pg.update(x),
+                )
+        except:
+            warnings.warn(
+                f"Upload of {dataset_name} unsuccessful. You may not have permission "
+                f"to upload to the agdata-data s3 bucket.",
+                category=UserWarning,
+            )
+        finally:
+            self.pg.close()
+
+    def upload_model(self, model_name, model_dir):
+        """Uploads model to agdata-data s3 file storage.
+
+        Parameters
+        ----------
+        model_name : str
+            name of model (without '.pth')
+        model_dir : str
+            path to directory where model is stored
+        """
+        # Establish connection with s3 via boto
+        self.s3 = boto3.client("s3")
+
+        # Setup progress bar
+        self.pg = tqdm(
+            total=os.stat(os.path.abspath(os.path.join(model_dir, model_name + ".pth"))).st_size,
+            file=sys.stdout,
+            desc=f"Uploading {model_name}",
+        )
+
+        # Upload data to agdata-data bucket
+        try:
+            with open(os.path.join(model_dir, model_name + ".pth"), "rb") as data:
+                self.s3.upload_fileobj(
+                    Fileobj=data,
+                    Bucket="agdata-data",
+                    Key="models/" + model_name + ".pth",
+                    Callback=lambda x: self.pg.update(x),
+                )
+        except:
+            warnings.warn(
+                f"Upload of {model_name} unsuccessful. You may not have permission "
+                f"to upload to the agdata-data s3 bucket.",
+                category=UserWarning,
+            )
+        finally:
+            self.pg.close()
+
+    def download_dataset(self, dataset_name, dest_dir):
+        """
+        Downloads dataset from agdata-data s3 file storage.
+
+        Parameters
+        ----------
+        dataset_name : str
+            name of dataset to download
+        dest_dir : str
+            path for saving downloaded dataset
+        """
+        # Validate the dataset name.
+        if dataset_name not in self.data_sources:
+            raise ValueError(f"Invalid dataset '{dataset_name}.'")
+
+        # Establish connection with s3 via boto
+        self.s3 = boto3.client("s3")
+        self.s3_resource = boto3.resource("s3")
+
+        # Setup progress bar
+        try:
+            self.pg = tqdm(
+                total=float(self.s3_resource.ObjectSummary(bucket_name="agdata-data", key=dataset_name + ".zip").size),
+                file=sys.stdout,
+                desc=f"Downloading {dataset_name}",
+            )
+        except botocore.exceptions.ClientError as ce:
+            if "Not Found" in str(ce):
+                raise ValueError(
+                    f"The dataset '{dataset_name}' could not be found in "
+                    f"the bucket, perhaps it has not been uploaded yet."
+                )
+            raise ce
+
+        # File path of zipped dataset
+        self.dataset_download_path = os.path.join(dest_dir, dataset_name + ".zip")
+
+        # Upload data to agdata-data bucket
+        with open(self.dataset_download_path, "wb") as data:
+            self.s3.download_fileobj(
+                Bucket="agdata-data",
+                Key=dataset_name + ".zip",
+                Fileobj=data,
+                Callback=lambda x: self.pg.update(x),
+            )
+        self.pg.close()
+
+        # Unzip downloaded dataset
+        with zipfile.ZipFile(self.dataset_download_path, "r") as z:
+            z.printdir()
+            print("Extracting files...")
+            z.extractall(path=dest_dir)
+            print("Done!")
+
+        # Delete zipped file
+        os.remove(self.dataset_download_path)

AgML/source/agml/_internal/syntheticdata.py

@@ -0,0 +1,756 @@
+import json
+import csv
+import copy
+import os
+import numpy as np
+from dict2xml import dict2xml
+import platform
+import subprocess
+import sys
+
+from skimage.io import imread, imshow
+import skimage
+from skimage.morphology import closing
+import imantics
+import pandas as pd
+import matplotlib.pyplot as plt
+import xml.etree.ElementTree as ET
+
+HELIOS_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "_helios/Helios")
+
+
+class HeliosDataGenerator(object):
+    def __init__(self, path_helios_dir=HELIOS_PATH):
+        self.path_canopygen_header = os.path.join(path_helios_dir, "plugins/canopygenerator/include/CanopyGenerator.h")
+        self.path_canopygen_cpp = os.path.join(path_helios_dir, "plugins/canopygenerator/src/CanopyGenerator.cpp")
+        self.path_cmakelists = os.path.join(
+            os.path.dirname(os.path.dirname(__file__)),
+            "_helios/Helios/projects/SyntheticImageAnnotation/CMakeLists.txt",
+        )
+        self.path_main_cpp = os.path.join(
+            os.path.dirname(os.path.dirname(__file__)), "_helios/Helios/projects/SyntheticImageAnnotation/main.cpp"
+        )
+
+        self.path_lidar_cpp = os.path.join(path_helios_dir, "plugins/lidar/src/LiDAR.cpp")
+        self.canopy_types = self.get_canopy_types()
+        self.canopy_params = self.get_canopy_params()
+        self.canopy_param_ranges = self.set_initial_canopy_param_ranges()
+        self.lidar_params = self.get_lidar_params()
+        self.lidar_param_ranges = self.set_initial_lidar_param_ranges()
+        self.camera_params = self.get_camera_params()
+        self.camera_param_ranges = self.set_initial_camera_param_ranges()
+
+    def get_canopy_types(self):
+        """
+        Find all occurrences of 'struct' in canopygen_header_txt.
+        Parse canopy type from struct occurrences.
+        Generate a list of canopy types.
+        """
+
+        # Read CanopyGenerator.h to define potential canopy types
+        with open(self.path_canopygen_header) as f:
+            canopygen_header_txt = f.readlines()
+
+        # Generate list of canopy types
+        canopy_types = []
+        search_term = "struct "
+        length_search_term = len(search_term)
+        for i, string in enumerate(canopygen_header_txt):
+            if search_term in string:
+                canopy_types.append(string[length_search_term:].split("Parameters{")[0])
+
+        return canopy_types
+
+    def set_seed(self, seed):
+        np.random.seed(seed)
+
+    def get_canopy_params(self):
+        # Flag for parsing
+        param_flag = 0
+
+        # Initialize canopy parameters dictionary
+        canopy_params = {}
+
+        # Read CanopyGenerator.cpp to define potential canopy types
+        with open(self.path_canopygen_cpp) as f:
+            canopygen_header_txt = f.readlines()
+
+            # Find parameters for each canopy type
+        for canopy_type in self.canopy_types:
+            # Find first line of parameter definition in cpp file
+            search_term = canopy_type + "Parameters::"
+
+            for i, string in enumerate(canopygen_header_txt):
+                if param_flag == 1:
+                    if string != "\n" and string != "}\n" and string != "  \n":
+                        line = string.split(";\n")[0]
+                        key, value = line.split(" = ")
+
+                        if "(" in value:
+                            value = value.split("(")[1].split(")")[0].replace(",", " ")
+
+                        canopy_params[canopy_type][key.strip()] = (
+                            value.strip().strip('"').replace("\n", "").replace(";", "").replace(".f", "0")
+                        )
+
+                        if len(canopy_params[canopy_type][key.strip()]) == 1:
+                            canopy_params[canopy_type][key.strip()] = canopy_params[canopy_type][key.strip()][0]
+
+                        if "M_PI" in value:
+                            new_val = float(value.split("*")[0]) * 3.14159
+                            canopy_params[canopy_type][key.strip()] = str(new_val)
+
+                if search_term in string:
+                    canopy_params[canopy_type] = {}
+                    param_flag = 1
+
+                if "}" in string and param_flag == 1:
+                    param_flag = 0
+
+        return canopy_params
+
+    def get_lidar_params(self):
+        # Flag for parsing
+        param_flag = 0
+
+        # Initialize canopy parameters dictionary
+        lidar_params = {}
+
+        # Read LiDAR.cpp to find parameters
+        with open(self.path_lidar_cpp) as f:
+            LiDAR_txt = f.readlines()
+
+            # Find Metadata line of parameter definition in cpp file
+            search_term = "ScanMetadata::ScanMetadata"
+
+            for i, string in enumerate(LiDAR_txt):
+                if param_flag == 1:
+                    if (
+                        string != "\n"
+                        and string != "}\n"
+                        and string != "  \n"
+                        and string != "  //Copy arguments into structure variables\n"
+                    ):
+                        line = string.split(";\n")[0]
+                        key = line.split(" = ")
+                        # Initialization of values from Helios -- Need to read this values from c++
+                        if key[0].strip() == "origin":
+                            lidar_params[key[0].strip()] = "0 0 0"
+                        elif key[0].strip() == "Ntheta":
+                            lidar_params["size"] = "250"
+                        elif key[0].strip() == "thetaMin":
+                            lidar_params[key[0].strip()] = "0"
+                        elif key[0].strip() == "thetaMax":
+                            lidar_params[key[0].strip()] = "180"
+                        elif key[0].strip() == "Nphi":
+                            lidar_params["size"] = lidar_params["size"] + " 450"
+                        elif key[0].strip() == "phiMin":
+                            lidar_params[key[0].strip()] = "0"
+                        elif key[0].strip() == "phiMax":
+                            lidar_params[key[0].strip()] = "360"
+                        elif key[0].strip() == "exitDiameter":
+                            lidar_params[key[0].strip()] = "0"
+                        elif key[0].strip() == "beamDivergence":
+                            lidar_params[key[0].strip()] = "0"
+                        elif key[0].strip() == "columnFormat":
+                            lidar_params["ASCII_format"] = "x y z"
+
+                if "ScanMetadata::ScanMetadata" in string:
+                    param_flag = 1
+
+                if "}" in string and param_flag == 1:
+                    param_flag = 0
+        return lidar_params
+
+    def get_camera_params(self):
+        # Initialize canopy parameters dictionary
+        camera_params = {}
+
+        # Initialization of image resolution and camera position
+        camera_params["image_resolution"] = "600 400"
+        camera_params["camera_lookat"] = "0 0 1"
+        camera_params["camera_position"] = "0 -2 1"
+
+        return camera_params
+
+    def set_initial_lidar_param_ranges(self):
+        lidar_param_ranges = copy.deepcopy(self.lidar_params)
+
+        # Check if parameter is a path or number; this assumes that all strings will be paths
+        for i in lidar_param_ranges.keys():
+            if i == "ASCII_format":
+                val = lidar_param_ranges[i]
+                lidar_param_ranges[i] = [(val.split(" ")[j].replace("f", "0")) for j in range(len(val.split()))]
+            else:
+                val = lidar_param_ranges[i]
+                lidar_param_ranges[i] = [float(val.split(" ")[j].replace("f", "0")) for j in range(len(val.split()))]
+
+        return lidar_param_ranges
+
+    def set_initial_canopy_param_ranges(self):
+        canopy_param_ranges = copy.deepcopy(self.canopy_params)
+
+        # Check if parameter is a path or number; this assumes that all strings will be paths
+        for i in canopy_param_ranges.keys():
+            for key in list(canopy_param_ranges[i]):
+                val = canopy_param_ranges[i][key]
+                if val.isalpha() or "/" in val or ":" in val:
+                    canopy_param_ranges[i].pop(key)
+                else:
+                    canopy_param_ranges[i][key] = [
+                        [float(val.split(" ")[j].replace("f", "0")), float(val.split(" ")[j].replace("f", "0"))]
+                        for j in range(len(val.split()))
+                    ]
+
+        return canopy_param_ranges
+
+    def set_initial_camera_param_ranges(self):
+        camera_param_ranges = copy.deepcopy(self.camera_params)
+
+        # Check if parameter is a path or number; this assumes that all strings will be paths
+        for i in camera_param_ranges.keys():
+            val = camera_param_ranges[i]
+            camera_param_ranges[i] = [float(val.split(" ")[j]) for j in range(len(val.split()))]
+
+        return camera_param_ranges
+
+    def generate_one_datapair(self, canopy_type, simulation_type, export_format="xml"):
+        """
+        Find all occurrences of 'struct' in canopygen_header_txt.
+        Parse canopy type from struct occurrences.
+        Generate a list of canopy types.
+
+        Args:
+        canopy_type (string): the selected canopy type for the synthetic images
+            It should contain a out_channels attribute, which indicates the number of output
+            channels that each feature map has (and it should be the same for all feature maps).
+            The backbone should return a single Tensor or and OrderedDict[Tensor].
+        simulation_type (string): choose between RGB only and Lidar mode using 'rgb' or 'lidar'
+        export_format (string): default is xml for Helios
+        """
+
+        assert canopy_type in self.canopy_types, "Canopy type not available."
+
+        assert export_format in ["xml", "csv", "json"], "Only xml, csv and json export formats are possible."
+
+        canopy_params_filtered = {k: v for k, v in self.canopy_params.items() if k.startswith(canopy_type)}
+
+        canopy_params_filtered[canopy_type + "Parameters"] = canopy_params_filtered.pop(canopy_type)
+
+        canopy_params_filtered["Ground"] = {
+            "origin": "0 0 0",
+            "extent": "10 10",
+            "texture_subtiles": "10 10",
+            "texture_subpatches": "1 1",
+            "ground_texture_file": "plugins/canopygenerator/textures/dirt.jpg",
+            "rotation": "0",
+        }
+
+        canopy_params_filtered = {"canopygenerator": canopy_params_filtered}
+
+        if simulation_type == "lidar":
+            canopy_params_filtered["scan"] = self.lidar_params
+
+        if simulation_type == "rgb":
+            canopy_params_filtered[""] = self.camera_params
+
+        canopy_params_filtered = {"helios": canopy_params_filtered}
+        from pprint import pprint
+        # print(canopy_params_filtered)
+
+        if not os.path.exists(
+            os.path.join(
+                os.path.dirname(os.path.dirname(__file__)), "_helios/Helios/projects/SyntheticImageAnnotation/xml"
+            )
+        ):
+            os.makedirs(
+                os.path.join(
+                    os.path.dirname(os.path.dirname(__file__)), "_helios/Helios/projects/SyntheticImageAnnotation/xml"
+                )
+            )
+
+        if export_format == "xml":
+            with open(
+                os.path.join(
+                    os.path.dirname(os.path.dirname(__file__)),
+                    "_helios/Helios/projects/SyntheticImageAnnotation/xml/tmp_canopy_params_image.xml",
+                ),
+                "w",
+            ) as f:
+                f.write(dict2xml(canopy_params_filtered))
+
+    def generate_data(
+        self, n_imgs, canopy_type, simulation_type, annotation_type, label_elements="none", output_directory=".."
+    ):
+        """
+        Given the path to the output of Helios, this method can be used to convert the data to a more standard format such as COCO JSON
+
+        Args:
+        n_imgs (int): The number of images that to be generated
+        canopy_type (string): the selected canopy type for the synthetic images
+        simulation_type (string): choose between RGB only and Lidar mode using 'rgb' or 'lidar'
+        annotation_type (string): choose between semantic segmentation, object detection, and instance segmentation as 'semantic', 'object', and 'instance'.
+        output_directory (string) (optional): optionally you may pass in a custom path to save the Helios output to the custom path
+
+        """
+        output_directory = os.path.abspath(output_directory)
+
+        assert simulation_type in ["rgb", "lidar"], "Simulation type unavailable."
+
+        assert annotation_type in ["semantic", "object", "instance", "none"], "Annotation type unavailable."
+
+        Labels = label_elements.replace(" ", "").split(",")
+
+        for j in range(len(Labels)):
+            assert Labels[j] in ["trunk", "leaves", "fruits", "branches", "ground", "none"], "Label type unavailable"
+
+        param_ranges = self.canopy_param_ranges[canopy_type]
+
+        camera_ranges = self.camera_param_ranges
+
+        if simulation_type == "lidar":
+            lidar_ranges = copy.deepcopy(self.lidar_param_ranges)
+
+            # LiDAR parameters
+            for key in lidar_ranges:
+                # param_vals=lidar_params[key].split(' ')
+                arr = [lidar_ranges[key][i] for i in range(len(lidar_ranges[key]))]
+                string_arr = [str(a) for a in arr]
+                self.lidar_params[key] = " ".join(string_arr)
+            # Mutiple LiDAR
+            LiDARs = []
+            for i in range(len(lidar_ranges["origin"])):
+                for key in lidar_ranges:
+                    if key == "origin":
+                        arr = [lidar_ranges[key][i] for i in range(len(lidar_ranges[key]))]
+                        arr = arr[i]
+                        string_arr = [str(a) for a in arr]
+                        self.lidar_params[key] = " ".join(string_arr)
+                    A = copy.deepcopy(self.lidar_params)
+                LiDARs.append(A)
+            self.lidar_params = LiDARs
+
+        if simulation_type == "rgb":
+            # Camera parameters
+            for key in camera_ranges:
+                arr = [camera_ranges[key][i] for i in range(len(camera_ranges[key]))]
+                string_arr = [str(a).replace(",", "").replace("[", " ").replace("]", " ") for a in arr]
+                self.camera_params[key] = " ".join(string_arr)
+
+        for n in range(n_imgs):
+            params = self.canopy_params[canopy_type]
+            # lidar_params = self.lidar_params
+
+            # Context parameters
+            for key in param_ranges:
+                print(param_ranges[key])
+                arr = [
+                    np.random.uniform(param_ranges[key][i][0], param_ranges[key][i][1])
+                    for i in range(len(param_ranges[key]))
+                ]
+                string_arr = [str(a) for a in arr]
+                params[key] = " ".join(string_arr)
+            self.canopy_params[canopy_type] = params
+            print(self.canopy_params[canopy_type])
+
+            self.generate_one_datapair(canopy_type, simulation_type)
+
+            # Re-write tags of XML to have the expected Helios input
+            tree = ET.parse(
+                os.path.join(
+                    os.path.dirname(os.path.dirname(__file__)),
+                    "_helios/Helios/projects/SyntheticImageAnnotation/xml",
+                    "tmp_canopy_params_image.xml",
+                )
+            )
+            root = tree.getroot()
+            for child in root:
+                if child.tag == "camera_position":
+                    child.tag = "globaldata_vec3"
+                    child.set("label", "camera_position")
+                if child.tag == "camera_lookat":
+                    child.tag = "globaldata_vec3"
+                    child.set("label", "camera_lookat")
+                if child.tag == "image_resolution":
+                    child.tag = "globaldata_int2"
+                    child.set("label", "image_resolution")
+
+            tree.write(
+                os.path.join(
+                    os.path.dirname(os.path.dirname(__file__)),
+                    "_helios/Helios/projects/SyntheticImageAnnotation/xml",
+                    "tmp_canopy_params_image.xml",
+                )
+            )
+
+            # Modify cmake file for rgb versus lidar simulation
+            with open(self.path_cmakelists) as f:
+                cmakelists_txt = f.readlines()
+
+            for i, string in enumerate(cmakelists_txt):
+                if "set( PLUGINS " in string and simulation_type == "lidar":
+                    cmakelists_txt[i] = 'set( PLUGINS "lidar;visualizer;canopygenerator;syntheticannotation" )\n'
+
+                if "set( PLUGINS " in string and simulation_type == "rgb":
+                    cmakelists_txt[i] = 'set( PLUGINS "visualizer;canopygenerator;syntheticannotation" )\n'
+
+            # and write everything back
+            with open(self.path_cmakelists, "w") as f:
+                f.writelines(cmakelists_txt)
+
+            # Modify maincpp file for rgb versus lidar simulation
+            # with open(self.path_main_cpp) as f:
+            #     main_cpp = f.readlines()
+
+            # Define paths for CMAKE compilation and output files
+            current_directory = os.getcwd()
+            build_dir = os.path.join(current_directory, "Helios/projects/SyntheticImageAnnotation/build")
+            if output_directory == "":
+                output_dir = os.path.join(current_directory, "Helios/projects/SyntheticImageAnnotation/output")
+            else:
+                assert os.path.isdir(output_directory), "Please, introduce a valid directory"
+                output_dir = output_directory + "/output"
+            point_cloud_dir = os.path.join(current_directory, output_dir + "/point_cloud/")
+            images_dir = os.path.join(current_directory, output_dir + "/images/")
+
+            if not os.path.exists(build_dir):
+                os.makedirs(build_dir)
+
+            if not os.path.exists(output_dir):
+                os.makedirs(output_dir)
+
+            if simulation_type == "lidar":
+                if not os.path.exists(point_cloud_dir):
+                    os.makedirs(point_cloud_dir)
+
+            if simulation_type == "rgb":
+                if not os.path.exists(images_dir):
+                    os.makedirs(images_dir)
+
+            exe = os.path.join(build_dir, "SyntheticImageAnnotation")
+
+            # Modify main.cpp file for compilation in LIDAR and RGB case
+            """
+            print('Generation synthetic data: #' + str(n))
+            for i, string in enumerate(main_cpp):
+                
+                # Managing annotation types requested by user
+                if 'annotation.disableSemanticSegmentation();' in string and 'semantic' not in annotation_type:
+                    main_cpp[i] = '    annotation.disableSemanticSegmentation();\n'
+
+                if 'annotation.disableSemanticSegmentation();' in string and 'semantic' in annotation_type:
+                    main_cpp[i] = '    //annotation.disableSemanticSegmentation();\n'
+
+                if 'annotation.disableObjectDetection();' in string and 'object' not in annotation_type:
+                    main_cpp[i] = '    annotation.disableObjectDetection();\n'
+
+                if 'annotation.disableObjectDetection();' in string and 'object' in annotation_type:
+                    main_cpp[i] = '    //annotation.disableObjectDetection();\n'
+
+                if 'annotation.disableInstanceSegmentation();' in string and 'instance' not in annotation_type:
+                    main_cpp[i] = '    annotation.disableInstanceSegmentation();\n'
+
+                if 'annotation.disableInstanceSegmentation();' in string and 'instance' in annotation_type:
+                    main_cpp[i] = '    //annotation.disableInstanceSegmentation();\n'
+                
+                # Managing LiDAR versus rgb request by user
+                if '#include "L' in string and simulation_type == 'lidar':
+                    main_cpp[i] = '#include "LiDAR.h"\n'
+
+                if '#include "L' in string and simulation_type == 'rgb':
+                    main_cpp[i] = '//#include "LiDAR.h"\n'
+
+                if 'flag=' in string and simulation_type == 'lidar':
+                    main_cpp[i] = 'bool flag=true;\n'
+
+                if 'flag=' in string and simulation_type == 'rgb':
+                    main_cpp[i] = 'bool flag=false;\n'
+
+                if 'LiDARcloud lidarcloud' in string and simulation_type == 'lidar':
+                    main_cpp[i] = ' LiDARcloud lidarcloud;\n'
+
+                if 'LiDARcloud lidarcloud' in string and simulation_type == 'rgb':
+                    main_cpp[i] = ' //LiDARcloud lidarcloud;\n'
+
+                if 'lidarcloud.loadXML' in string and simulation_type == 'lidar':
+                    main_cpp[i] = ' lidarcloud.loadXML("../xml/tmp_canopy_params_image.xml");\n'
+
+                if 'lidarcloud.loadXML' in string and simulation_type == 'rgb':
+                    main_cpp[i] = ' //lidarcloud.loadXML("../xml/tmp_canopy_params_image.xml");\n'
+
+                if 'lidarcloud.syntheticScan' in string and simulation_type == 'lidar':
+                    main_cpp[i] = ' lidarcloud.syntheticScan( &context);\n'
+
+                if 'lidarcloud.syntheticScan' in string and simulation_type == 'rgb':
+                    main_cpp[i] = ' //lidarcloud.syntheticScan( &context);\n'
+
+                #
+                if 'lidarcloud.exportPointCloud' in string and simulation_type == 'lidar':
+                    main_cpp[i] = ' lidarcloud.exportPointCloud( "' + output_dir + '/point_cloud/synthetic_scan_' + str(
+                        n) + '.xyz" );\n'
+
+                if 'lidarcloud.exportPointCloud' in string and simulation_type == 'rgb':
+                    main_cpp[
+                        i] = ' //lidarcloud.exportPointCloud( "' + output_dir + '/point_cloud/synthetic_scan_' + str(
+                        n) + '.xyz" );\n'
+
+                # Change output  directory -- if requested
+
+                if 'annotation.render(' in string and simulation_type == 'rgb':
+                    main_cpp[i] = ' annotation.render( "' + images_dir + '" );\n'
+
+                # Choose elements to label
+                for k in Labels:
+                        
+                    if 'annotation.labelPrimitives(cgen.getTrunkUUIDs(p)' in string and simulation_type == 'rgb' and 'trunk' in Labels:
+                        main_cpp[i] = ' annotation.labelPrimitives(cgen.getTrunkUUIDs(p), "trunks");\n'
+                    elif  'annotation.labelPrimitives(cgen.getTrunkUUIDs(p)' in string and simulation_type == 'rgb' and 'trunk' not in Labels:
+                        main_cpp[i] = ' //annotation.labelPrimitives(cgen.getTrunkUUIDs(p), "trunks");\n'
+                    
+                    if 'annotation.labelPrimitives(cgen.getBranchUUIDs(p)' in string and simulation_type == 'rgb' and 'branches' in Labels:
+                        main_cpp[i] = ' annotation.labelPrimitives(cgen.getBranchUUIDs(p), "branches");\n'
+                    elif 'annotation.labelPrimitives(cgen.getBranchUUIDs(p)' in string and simulation_type == 'rgb' and 'branches' not in Labels:
+                        main_cpp[i] = ' //annotation.labelPrimitives(cgen.getBranchUUIDs(p), "branches");\n'
+                    
+                    if 'annotation.labelPrimitives(cgen.getLeafUUIDs(p)' in string and simulation_type == 'rgb' and 'leaves' in Labels:
+                        main_cpp[i] = ' annotation.labelPrimitives(cgen.getLeafUUIDs(p), "leaves");\n'
+                    elif 'annotation.labelPrimitives(cgen.getLeafUUIDs(p)' in string and simulation_type == 'rgb' and 'leaves' not in Labels:
+                        main_cpp[i] = ' //annotation.labelPrimitives(cgen.getLeafUUIDs(p), "leaves");\n'
+
+                    if 'annotation.labelPrimitives( flatten(fruitUUIDs.at(c))' in string and simulation_type == 'rgb' and 'fruits' in Labels:
+                        main_cpp[i] = ' annotation.labelPrimitives( flatten(fruitUUIDs.at(c)), "clusters" );\n'
+                    elif 'annotation.labelPrimitives( flatten(fruitUUIDs.at(c))' in string and simulation_type == 'rgb' and 'fruits' not in Labels:
+                        main_cpp[i] = ' //annotation.labelPrimitives( flatten(fruitUUIDs.at(c)), "clusters" );\n'
+
+
+
+            # and write everything back
+            with open(self.path_main_cpp, 'w') as f:
+                f.writelines(main_cpp)
+            """
+
+            default_xml_path = os.path.join(
+                os.path.dirname(os.path.dirname(__file__)),
+                "_helios/Helios/projects/SyntheticImageAnnotation/xml",
+                "tmp_canopy_params_image.xml",
+            )
+            str_to_write = f"{annotation_type}\n{simulation_type}\n{' '.join(Labels)}\n{default_xml_path}\n{images_dir}"
+            with open(
+                os.path.join(
+                    os.path.dirname(os.path.dirname(__file__)),
+                    "_helios/Helios/projects/SyntheticImageAnnotation/config.txt",
+                ),
+                "w",
+            ) as f:
+                f.write(str_to_write)
+
+            # System call to helios @DARIO
+            # current_directory = os.getcwd()
+            helios_directory = os.path.join(os.path.dirname(os.path.dirname(__file__)), "_helios")
+            build_dir = os.path.join(helios_directory, "Helios/projects/SyntheticImageAnnotation/build")
+            output_dir = os.path.join(helios_directory, "Helios/projects/SyntheticImageAnnotation/output")
+            point_cloud_dir = os.path.join(
+                helios_directory, "Helios/projects/SyntheticImageAnnotation/output/point_cloud/"
+            )
+            images_dir = os.path.join(helios_directory, "Helios/projects/SyntheticImageAnnotation/output/images/")
+
+            if not os.path.exists(build_dir):
+                os.makedirs(build_dir)
+
+            if not os.path.exists(output_dir):
+                os.makedirs(output_dir)
+
+            if simulation_type == "lidar":
+                if not os.path.exists(point_cloud_dir):
+                    os.makedirs(point_cloud_dir)
+
+            if simulation_type == "rgb":
+                if not os.path.exists(images_dir):
+                    os.makedirs(images_dir)
+
+            exe = os.path.join(build_dir, "SyntheticImageAnnotation")
+
+            cmake_args = ["-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=" + build_dir]
+            cmake_args += ["-G", "Unix Makefiles"]
+            # current_directory = os.getcwd()
+            helios_directory = os.path.join(os.path.dirname(os.path.dirname(__file__)), "_helios")
+            build_dir = os.path.join(helios_directory, "Helios/projects/SyntheticImageAnnotation/build")
+            output_dir = os.path.join(helios_directory, "Helios/projects/SyntheticImageAnnotation/output")
+            point_cloud_dir = os.path.join(
+                helios_directory, "Helios/projects/SyntheticImageAnnotation/output/point_cloud/"
+            )
+            images_dir = os.path.join(helios_directory, "Helios/projects/SyntheticImageAnnotation/output/images/")
+
+            if not os.path.exists(build_dir):
+                os.makedirs(build_dir)
+
+            if not os.path.exists(output_dir):
+                os.makedirs(output_dir)
+
+            if simulation_type == "lidar":
+                if not os.path.exists(point_cloud_dir):
+                    os.makedirs(point_cloud_dir)
+
+            if n == 0:
+                subprocess.run(["cmake", ".."] + cmake_args, cwd=build_dir, check=True)
+            subprocess.run(["cmake", "--build", "."], cwd=build_dir, check=True)
+
+            subprocess.run(
+                [
+                    exe,
+                    "/Users/amoghjoshi/Documents/Amogh/Programs/Python/AgML/agml/_helios/Helios/projects/SyntheticImageAnnotation/config.txt",
+                    output_dir,
+                ],
+                cwd=build_dir,
+            )
+
+        # self.convert_data() # Conversion to standard formats @PRANAV
+
+    def convert_data(self, Frames_path, annotation_format="instance_segmentation"):
+        """
+        Given the path to the output of Helios, this method can be used to convert the data to a more standard format such as COCO JSON
+
+        Args:
+        annotation_format (string): choose between 'instance_segmentation', 'object_detection', or 'panoptic_segmentation'
+        Frames_path (string): Specify the path to the output of Helios
+
+        """
+
+        if annotation_format == "object_detection":
+            pass  # YOLO @PRANAV
+        if annotation_format == "semantic_segmentation":
+            pass  # single channel + mapping @PRANAV
+        if annotation_format == "instance_segmentation" or annotation_format == "object_detection":
+            # data_path = sorted(os.listdir(data_path))
+            frames_view = sorted(os.listdir(Frames_path))
+            # Initialize a list of label numpy arrays
+            # imgs = []
+            # npy_arrs = []
+            images = []
+            if not os.path.exists("./train_images/"):
+                os.mkdir("./train_images/")
+            for i, frame in enumerate(frames_view):
+                if frame == ".DS_Store":
+                    continue
+
+                else:
+                    # Generate labels and append to list
+                    image, mask, pixel_ID = self.generate_npy_arr(Frames_path, Frames_path + frame + "/")
+                    plt.imsave("./train_images/" + frame + ".jpeg", image)
+
+                    strel = skimage.morphology.selem.disk(2)
+
+                    # %% Convert masks to COCO for detectron implementation.
+
+                    mapping = pd.read_csv(
+                        Frames_path + frame + "/" + "ID_mapping.txt", delim_whitespace=True, names=["ID", "Class"]
+                    )
+
+                    category = imantics.Category(
+                        mapping["Class"].iloc[pixel_ID], color=imantics.Color([255, 0, 0])
+                    )  # color for debug only
+                    c = 0
+
+                    im2 = imantics.Image(
+                        image_array=image, path="data/train_images/" + frame + ".jpeg", id=i
+                    )  # imantics image object
+
+                    for i in range(mask.shape[2]):  # create the polygons for each slice
+                        if annotation_format == "instance_segmentation":
+                            poly = imantics.Mask(closing(mask[:, :, i], strel)).polygons()
+                            ann = imantics.Annotation(image=im2, category=category, polygons=poly, id=c)
+                            box = imantics.Mask(closing(mask[:, :, i], strel)).bbox()
+                            # print(box)
+
+                        if annotation_format == "object_detection":
+                            box = imantics.Mask(closing(mask[:, :, i], strel)).bbox()
+                            ann = imantics.Annotation(image=im2, category=category, bbox=box, id=c)
+                            # print(box)
+                        c += 1
+
+                        if np.sum(mask[:, :, i]) > 1300:
+                            im2.add(ann)  # add the polygon to the image object
+                            print(box)
+
+                    images.append(im2)  # collect the image objects after they get polygons
+                # %% Create Imantics dataset and export
+            ds = imantics.Dataset(name="coco", images=images)
+            obj = ds.coco()
+            with open("trainval.json", "w") as json_file:
+                json.dump(obj, json_file)
+            #     json_file.close()
+            # obj=obj.clear()
+            # del obj, ds, json_file, images
+
+            # COCO json @PRANAV
+        if annotation_format == "panoptic_segmentation":
+            pass
+        if annotation_format == "regression":
+            pass
+
+    def generate_npy_arr(self, Frames_path, frames_view_path):
+        Frame_paths = frames_view_path
+
+        # Grab dimensions
+        render = imread(frames_view_path + "/RGB_rendering" + ".jpeg")
+        render_xy_shape = (render.shape[0], render.shape[1])
+        # print(render_xy_shape)
+
+        # Number of instances
+        frames_list = []
+        exclude_files = ["RGB_rendering.jpeg", "pixelID_combined.txt", "ID_mapping.txt"]
+        frames_files = [x for x in os.listdir(frames_view_path) if x not in exclude_files]
+        frames_list.append(frames_files)
+        n_instances = [len(frame) for frame in frames_list]
+
+        # Initialize numpy array of shape (x, y, n_instances)
+        npy_arr = np.zeros(
+            shape=(render_xy_shape[0], render_xy_shape[1], n_instances[0]), dtype=np.bool_
+        )  # Make boolean dtype
+
+        # Read frames as a list
+        bboxes = []
+        frames = []
+        i = 0
+        for file_grape_arr in os.listdir(frames_view_path):
+            if file_grape_arr == ".DS_Store":
+                continue
+
+            if file_grape_arr == "pixelID_combined.txt":
+                continue
+
+            if file_grape_arr == "ID_mapping.txt":
+                continue
+
+            if file_grape_arr == "RGB_rendering.jpeg":
+                continue
+
+            if file_grape_arr == "rectangular_labels.txt":
+                continue
+
+            else:
+                # Extract pixel id based off name of the file. We want the number between _ and .
+
+                pixel_id_tmp = int(file_grape_arr.split("_")[1].split(".")[0])
+                # print(pixel_id_tmp)
+
+                # Load bounding box coordinates from file
+                bbox_tmp = np.loadtxt(frames_view_path + file_grape_arr, max_rows=1)
+                bbox_tmp = [int(bbox_tmp[i]) for i in range(0, 4)]
+                bbox = np.copy(bbox_tmp)
+                bbox[3] = render_xy_shape[0] - bbox_tmp[2]
+                bbox[2] = render_xy_shape[0] - bbox_tmp[3]
+
+                # print(bbox_tmp)
+
+                # Load pixel positions from file
+                grape_arr_tmp = np.loadtxt(frames_view_path + file_grape_arr, skiprows=1, ndmin=2)
+
+                #
+                grape_arr_tmp = grape_arr_tmp == pixel_id_tmp
+
+                npy_arr[bbox[2] : (bbox[3] + 1), bbox[0] : (bbox[1] + 1), i] = grape_arr_tmp
+
+                i += 1
+
+        return render, npy_arr, pixel_id_tmp

AgML/source/agml/_internal/utils.py

@@ -0,0 +1,42 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Methods for checking the dataset information in AgML.
+"""
+
+import os
+import pickle
+
+from agml.utils.io import recursive_dirname
+
+_PERSONAL_ACCESS_TOKEN = None
+
+
+def shape_info_file_contents():
+    with open(
+        os.path.join(recursive_dirname(__file__, 2), "_assets", "shape_info.pickle"),
+        "rb",
+    ) as f:
+        return pickle.load(f)
+
+
+def set_git_personal_access_token(token):
+    global _PERSONAL_ACCESS_TOKEN
+    _PERSONAL_ACCESS_TOKEN = token
+
+
+def get_personal_access_token():
+    global _PERSONAL_ACCESS_TOKEN
+    return _PERSONAL_ACCESS_TOKEN

AgML/source/agml/backend/__init__.py

@@ -0,0 +1,39 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Controls the AgML backend system.
+
+The first part of the backend is the backend deep learning library.
+The backend, one of {TensorFlow, PyTorch}, primarily exists for internal
+purposes, e.g. figuring out which methods to use in the data module or
+the actual model configuring/training modules.
+
+Secondly, the backend controls the loading/saving procedure for files within
+AgML, specifically data loaded or generated from the data module.
+"""
+
+from . import experimental
+from .config import (
+    clear_all_datasets,
+    data_save_path,
+    downloaded_datasets,
+    model_save_path,
+    set_data_save_path,
+    set_model_save_path,
+    set_synthetic_save_path,
+    synthetic_data_save_path,
+)
+from .random import set_seed
+from .tftorch import get_backend, set_backend

AgML/source/agml/backend/config.py

@@ -0,0 +1,206 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import os
+import shutil
+from pathlib import Path
+
+from agml.utils.logging import log
+
+# The super base save directory for AgML. This is the overriding base
+# save directory and default original save directory. This is saved here
+# because the default path to save datasets to can be overridden by
+# `set_dataset_save_path()`, but we need to keep the super directory to
+# access the config file which stores this information in the first place.
+SUPER_BASE_DIR = Path(os.getenv("AGML_CONFIG_DIR", os.path.join(os.path.expanduser("~"), ".agml")))
+
+
+# This is the path to the saved datasets. By default, this saves to
+# SUPER_BASE_DIR/datasets, but can be overridden. The value of this
+# is set upon the instantiation of the module (see below code).
+DATASET_SAVE_DIR: str
+
+
+# This is similar to `DATASET_SAVE_DIR`, but is for synthetically generated
+# datasets using Helios. By default, this will be SUPER_BASE_DIR/synthetic,
+# but it can be overridden. The value is set upon instantiation of the module.
+SYNTHETIC_SAVE_DIR: str
+
+
+# This is the path to any downloaded models. By default, this saves to
+# SUPER_BASE_DIR/models, but can be overridden. The value is set upon
+# instantiation of the module (see the below code).
+MODEL_SAVE_DIR: str
+
+
+# Loads the configuration info. We don't cache this since it may
+# change if the user decides to change the path. This method
+# also runs upon the first import of AgML to set it properly.
+def _load_config_info():
+    global DATASET_SAVE_DIR, SYNTHETIC_SAVE_DIR, MODEL_SAVE_DIR
+    try:
+        with open(os.path.join(SUPER_BASE_DIR, "config.json"), "r") as f:
+            contents = json.load(f)
+            DATASET_SAVE_DIR = contents["data_path"]
+            SYNTHETIC_SAVE_DIR = contents["synthetic_data_path"]
+            MODEL_SAVE_DIR = contents["model_path"]
+    except (OSError, KeyError):
+        with open(os.path.join(SUPER_BASE_DIR, "config.json"), "w") as f:
+            json.dump(
+                {
+                    "data_path": os.path.join(SUPER_BASE_DIR, "datasets"),
+                    "synthetic_data_path": os.path.join(SUPER_BASE_DIR, "synthetic"),
+                    "model_path": os.path.join(SUPER_BASE_DIR, "models"),
+                },
+                f,
+            )
+        _load_config_info()
+
+
+_load_config_info()
+
+
+def data_save_path():
+    """Returns the default dataset save path for AgML."""
+    global DATASET_SAVE_DIR
+    return DATASET_SAVE_DIR
+
+
+def set_data_save_path(location=None):
+    """Sets the default dataset save path for AgML.
+    Changing the data save path using this method permanently changes
+    the data save path for all future sessions, until it is changed
+    or switched back to the original. If you just want to download one
+    dataset to a different path, use the `dataset_path` argument.
+    Parameters
+    ----------
+    location : str
+        The location to save the data to.
+    Returns
+    -------
+    The fully expanded location.
+    """
+    global SUPER_BASE_DIR
+    if location is None or location == "reset":
+        location = os.path.join(SUPER_BASE_DIR, "datasets")
+    location = os.path.expanduser(location)
+    if not os.path.exists(location) and not os.path.isdir(location):
+        raise NotADirectoryError(f"The provided destination {location} does " f"not exist, or is not a directory.")
+    _update_config("data_path", os.path.realpath(os.path.abspath(location)))
+    return
+
+
+def synthetic_data_save_path():
+    """Returns the default synthetic data save path for AgML."""
+    global SYNTHETIC_SAVE_DIR
+    return SYNTHETIC_SAVE_DIR
+
+
+def set_synthetic_save_path(location=None):
+    """Sets the default synthetic data save path for AgML.
+    Changing the data save path using this method permanently changes
+    the data save path for all future sessions, until it is changed
+    or switched back to the original. If you just want to download one
+    dataset to a different path, use the `dataset_path` argument.
+    Parameters
+    ----------
+    location : str
+        The location to save the data to.
+    Returns
+    -------
+    The fully expanded location.
+    """
+    global SUPER_BASE_DIR
+    if location is None or location == "reset":
+        location = os.path.join(SUPER_BASE_DIR, "synthetic")
+    location = os.path.expanduser(location)
+    if not os.path.exists(location) and not os.path.isdir(location):
+        raise NotADirectoryError(f"The provided destination {location} does " f"not exist, or is not a directory.")
+    _update_config("synthetic_data_path", os.path.realpath(os.path.abspath(location)))
+    return
+
+
+def model_save_path():
+    """Returns the default model save path for AgML."""
+    global MODEL_SAVE_DIR
+    return MODEL_SAVE_DIR
+
+
+def set_model_save_path(location=None):
+    """Sets the default model save path for AgML.
+    Changing the data save path using this method permanently changes
+    the data save path for all future sessions, until it is changed
+    or switched back to the original.
+    Parameters
+    ----------
+    location : str
+        The location to save the model to.
+    Returns
+    -------
+    The fully expanded location.
+    """
+    global SUPER_BASE_DIR
+    if location is None or location == "reset":
+        location = os.path.join(SUPER_BASE_DIR, "models")
+    location = os.path.expanduser(location)
+    if not os.path.exists(location) and not os.path.isdir(location):
+        raise NotADirectoryError(f"The provided destination {location} does " f"not exist, or is not a directory.")
+    _update_config("model_path", os.path.realpath(os.path.abspath(location)))
+    return
+
+
+def _get_config(param):
+    """Update the configuration file with a new parameter."""
+    global SUPER_BASE_DIR
+    with open(os.path.join(SUPER_BASE_DIR, "config.json"), "r") as f:
+        contents = json.load(f)
+    if param not in contents.keys():
+        return None
+    return contents[param]
+
+
+def _update_config(param, value):
+    """Update the configuration file with a new parameter."""
+    global SUPER_BASE_DIR
+    with open(os.path.join(SUPER_BASE_DIR, "config.json"), "r") as f:
+        contents = json.load(f)
+    contents[param] = value
+    with open(os.path.join(SUPER_BASE_DIR, "config.json"), "w") as f:
+        json.dump(contents, f)
+
+
+def clear_all_datasets():
+    """Deletes all of the datasets within the AgML local storage."""
+    log("Entering AgML interactive dataset deletion mode.", logging.WARNING)
+    msg_format = "\033[91m{0}\033[0m"
+    if not input(msg_format.format("Please confirm that you want to delete datasets: [y|n] ")) == "y":
+        print("Aborting dataset deletion.")
+    local_datasets = os.listdir(data_save_path())
+    deleted_datasets = []
+    for dataset in local_datasets:
+        if not os.path.isdir(os.path.join(data_save_path(), dataset)):
+            continue
+        if not input(msg_format.format(f"Delete dataset '{dataset}'? [y|n] ")) == "y":
+            continue
+        shutil.rmtree(os.path.join(data_save_path(), dataset))
+        deleted_datasets.append(dataset)
+    print(f"Deleted datasets {deleted_datasets}.")
+    log("Exiting AgML interactive dataset deletion mode.", logging.WARNING)
+
+
+def downloaded_datasets():
+    """Lists downloaded datasets in ~/.agml/datasets"""
+    return [d for d in os.listdir(data_save_path()) if os.path.isdir(os.path.join(data_save_path(), d))]

AgML/source/agml/backend/experimental.py

@@ -0,0 +1,60 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Experimental features in AgML, these may be added or removed permanently."""
+
+from agml.framework import AgMLSerializable
+
+__all__ = ["allow_nested_data_splitting"]
+
+
+class AgMLExperimentalFeatureWrapper(AgMLSerializable):
+    """Stores all of the feature values."""
+
+    _NESTED_SPLITTING = False
+
+    def __init__(self):
+        raise ValueError("This class should not be instantiated!")
+
+    @classmethod
+    def allow_nested_data_splitting(cls, value: bool) -> None:
+        """Enables/disables nested splitting of `AgMLDataLoader`s.
+
+        This method can be used to either enable or disable a feature which
+        allows an sub-AgMLDataLoader which has been split to a custom data
+        split to be split again. This is an experimental feature, which
+        would allow for multiple levels of nested splits (e.g., you could
+        have a `loader.train_data.val_data.test_data`).
+
+        Parameters
+        ----------
+        value : bool
+            Whether to enable or disable the feature.
+
+        Notes
+        -----
+        This method must be called at the start of each script in order
+        to function, otherwise it will default to `False`.
+        """
+        if not isinstance(value, bool):
+            raise TypeError("Expected either True or False.")
+        cls._NESTED_SPLITTING = value
+
+    @classmethod
+    def nested_splitting(cls):
+        return cls._NESTED_SPLITTING
+
+
+# While the `AgMLExperimentalFeatureWrapper` class controls the actual values,
+# we expose each of its toggle methods as part of the `agml.backend` API.
+allow_nested_data_splitting = AgMLExperimentalFeatureWrapper.allow_nested_data_splitting

AgML/source/agml/backend/random.py

@@ -0,0 +1,29 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from agml.backend.tftorch import get_backend, tf, torch
+
+
+def set_seed(seed=None):
+    """Sets a new random seed. If None, uses a random seed."""
+    import random
+
+    random.seed(seed)
+    import numpy as np
+
+    np.random.seed(seed)
+    if get_backend() == "torch":
+        torch.random.manual_seed(seed)
+    elif get_backend() == "tensorflow":
+        tf.random.set_seed(seed)

AgML/source/agml/backend/tftorch.py

@@ -0,0 +1,334 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This part of the backend controls the AgML methods where either
+TensorFlow or PyTorch methods can be used, and prevents unnecessary
+importing of either library (which takes a significant amount of time).
+"""
+
+import functools
+import importlib
+import inspect
+import logging
+import types
+
+# Suppress any irrelevant warnings which will pop up from either backend.
+import warnings
+
+import numpy as np
+
+from agml.utils.image import consistent_shapes
+from agml.utils.logging import log
+
+warnings.filterwarnings("ignore", category=UserWarning, message=".*Named tensors.*Triggered internally.*")
+
+
+class StrictBackendError(ValueError):
+    def __init__(self, message=None, change=None, obj=None):
+        if message is None:
+            message = (
+                f"Backend was manually set to "
+                f"'{get_backend()}', but got an object "
+                f"from backend '{change}': {obj}."
+            )
+        super(StrictBackendError, self).__init__(message)
+
+
+# Check if TensorFlow and PyTorch exist in the environment.
+_HAS_TENSORFLOW: bool
+_HAS_TORCH: bool
+
+
+@functools.lru_cache(maxsize=None)
+def _check_tf_torch():
+    global _HAS_TENSORFLOW, _HAS_TORCH
+    try:
+        import tensorflow
+    except ImportError:
+        _HAS_TENSORFlOW = False
+    else:
+        _HAS_TENSORFLOW = True
+    try:
+        import torch
+    except ImportError:
+        _HAS_TORCH = False
+    else:
+        _HAS_TORCH = True
+
+
+# Default backend is PyTorch.
+_BACKEND = None
+_USER_SET_BACKEND = False
+
+
+def get_backend():
+    """Returns the current AgML backend."""
+    return _BACKEND
+
+
+def set_backend(backend):
+    """Change the AgML backend for the current session.
+
+    By default, AgML uses PyTorch as a backend, but it is
+    compatible with both TensorFlow and PyTorch. AgML can
+    also automatically inference the backend from the
+    different parameters passed into `AgMLDataLoader` and
+    other internal library methods.
+
+    This method allows a user to automatically set the backend.
+    """
+    global _USER_SET_BACKEND, _BACKEND
+    # Check whether the user has modified the backend.
+    mod = inspect.getmodule(inspect.stack()[1][0])
+    if mod is None:  # IPython shell
+        _USER_SET_BACKEND = True
+    elif "agml." not in mod.__name__:
+        _USER_SET_BACKEND = True
+
+    # If the backend is the same, don't do anything.
+    if backend == _BACKEND:
+        return
+
+    _check_tf_torch()
+    if backend not in ["tensorflow", "tf", "torch", "pytorch"]:
+        raise ValueError(f"Invalid backend: {backend}.")
+    if backend in ["tensorflow", "tf"] and _BACKEND != "tensorflow":
+        if not _HAS_TENSORFLOW:
+            raise ImportError(
+                "TensorFlow not found on system, cannot be used as " "backend. Try running `pip install tensorflow`."
+            )
+        _BACKEND = "tf"
+        log("Switched backend to TensorFlow.", level=logging.INFO)
+    elif backend in ["torch", "pytorch"] and _BACKEND != "torch":
+        if not _HAS_TORCH:
+            raise ImportError(
+                "PyTorch not found on system, cannot be used as " "backend. Try running `pip install torch`."
+            )
+        _BACKEND = "torch"
+        log("Switched backend to PyTorch.", level=logging.INFO)
+
+
+def user_changed_backend():
+    """Returns whether the backend has been manually changed."""
+    return _USER_SET_BACKEND
+
+
+# Ported from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/util/lazy_loader.py
+class LazyLoader(types.ModuleType):
+    """Lazily import a module, mainly to avoid pulling in large dependencies."""
+
+    def __init__(self, local_name, parent_module_globals, name):
+        self._local_name = local_name
+        self._parent_module_globals = parent_module_globals
+        super(LazyLoader, self).__init__(name)
+
+    def _load(self):
+        """Load the module and insert it into the parent's globals."""
+        # Import the target module and insert it into the parent's namespace.
+        module = importlib.import_module(self.__name__)
+        self._parent_module_globals[self._local_name] = module
+
+        # Update this object's dict so that if someone keeps a reference to the
+        #   LazyLoader, lookups are efficient (__getattr__ is only called on lookups
+        #   that fail).
+        self.__dict__.update(module.__dict__)
+        return module
+
+    def __getattr__(self, item):
+        module = self._load()
+        return getattr(module, item)
+
+    def __dir__(self):
+        module = self._load()
+        return dir(module)
+
+
+# Load TensorFlow and PyTorch lazily to prevent pulling them in when unnecessary.
+torch = LazyLoader("torch", globals(), "torch")
+torch_data = LazyLoader("torch_data", globals(), "torch.utils.data")
+torchvision = LazyLoader("torchvision", globals(), "torchvision")
+tf = LazyLoader("tensorflow", globals(), "tensorflow")
+
+
+######### GENERAL METHODS #########
+
+
+def _convert_image_to_torch(image):
+    """Converts an image (np.ndarray) to a torch Tensor."""
+    if isinstance(image, (list, tuple)):
+        return torch.tensor(image)
+    if isinstance(image, torch.Tensor) or image.ndim == 4:
+        if image.shape[0] == 1 and image.shape[-1] <= 3 and image.ndim == 4:
+            return torch.from_numpy(image).permute(0, 3, 1, 2).float()
+        return image
+    if image.ndim == 3:
+        if image.shape[0] > image.shape[-1]:
+            return torch.from_numpy(image).permute(2, 0, 1).float()
+    elif image.ndim == 2:
+        return torch.from_numpy(image)
+    return torch.from_numpy(image).float()
+
+
+def _postprocess_torch_annotation(image):
+    """Post-processes a spatially augmented torch annotation."""
+    try:
+        if image.dtype.is_floating_point:
+            image = (image * 255).int()
+    except AttributeError:
+        pass
+    return image
+
+
+def as_scalar(inp):
+    """Converts an input value to a scalar."""
+    if isinstance(inp, (int, float)):
+        return inp
+    if np.isscalar(inp):
+        return inp.item()
+    if isinstance(inp, np.ndarray):
+        return inp.item()
+    if isinstance(inp, torch.Tensor):
+        return inp.item()
+    if isinstance(inp, tf.Tensor):
+        return inp.numpy()
+    raise TypeError(f"Unsupported variable type {type(inp)}.")
+
+
+def scalar_unpack(inp):
+    """Unpacks a 1-d array into a list of scalars."""
+    return [as_scalar(item) for item in inp]
+
+
+def is_array_like(inp, no_list=False):
+    """Determines if an input is a np.ndarray, torch.Tensor, or tf.Tensor."""
+    if isinstance(inp, (list, tuple)):  # no need to import tensorflow for this
+        if no_list:
+            return False
+        return True
+    if isinstance(inp, np.ndarray):
+        return True
+    if isinstance(inp, torch.Tensor):
+        return True
+    if isinstance(inp, tf.Tensor):
+        return True
+    return False
+
+
+def convert_to_batch(images):
+    """Converts a set of images to a batch."""
+    # If `images` is already an array type, nothing to do.
+    if is_array_like(images):
+        return images
+
+    # NumPy Arrays.
+    if isinstance(images[0], np.ndarray):
+        if not consistent_shapes(images):
+            images = np.array(images, dtype=object)
+            log(
+                "Created a batch of images with different "
+                "shapes. If you want the shapes to be consistent, "
+                "run `loader.resize_images('auto')`."
+            )
+        else:
+            images = np.array(images)
+        return images
+
+    # Torch Tensors.
+    if isinstance(images[0], torch.Tensor):
+        if not consistent_shapes(images):
+            images = [image.numpy() for image in images]
+            images = np.array(images, dtype=object)
+            log(
+                "Created a batch of images with different "
+                "shapes. If you want the shapes to be consistent, "
+                "run `loader.resize_images('auto')`."
+            )
+        else:
+            images = torch.stack(images)
+        return images
+
+    # TensorFlow Tensors.
+    if isinstance(images[0], tf.Tensor):
+        if not consistent_shapes(images):
+            images = tf.ragged.stack(images)
+            log(
+                "Created a batch of images with different "
+                "shapes. If you want the shapes to be consistent, "
+                "run `loader.resize_images('auto')`."
+            )
+        else:
+            images = tf.stack(images)
+        return images
+
+
+######### AGMLDATALOADER METHODS #########
+
+
+class AgMLObject(object):
+    """Base class for the `AgMLDataLoader` to enable inheritance.
+
+    This class solves a bug which arises when trying to dynamically
+    inherit from `tf.keras.utils.Sequence` and/or `torch.utils.data.Dataset`.
+    The fact that the `AgMLDataLoader` has this `AgMLObject` as a subclass
+    enables it to be able to handle dynamic inheritance. This is the sole
+    purpose of this subclass, it does not have any features.
+    """
+
+
+def _add_dataset_to_mro(inst, mode):
+    """Adds the relevant backend class to the `AgMLDataLoader` MRO.
+
+    This allows for the loader to dynamically inherent from the
+    `tf.keras.utils.Sequence` and `torch.utils.data.Dataset`.
+    """
+    if mode == "tf":
+        if not get_backend() == "tf":
+            if user_changed_backend():
+                raise StrictBackendError(change="tf", obj=inst)
+            set_backend("tf")
+        if tf.keras.utils.Sequence not in inst.__class__.__bases__:
+            inst.__class__.__bases__ += (tf.keras.utils.Sequence,)
+    if mode == "torch":
+        if not get_backend() == "torch":
+            if user_changed_backend():
+                raise StrictBackendError(change="torch", obj=inst)
+        if torch_data.Dataset not in inst.__class__.__bases__:
+            inst.__class__.__bases__ += (torch_data.Dataset,)
+
+
+def collate_fn_basic(batch):
+    images = torch.stack([i[0] for i in batch], dim=0)
+    coco = tuple(zip(*[i[1] for i in batch]))
+    return images, coco
+
+
+def collate_fn_efficientdet(batch):
+    """Collates items together into a batch."""
+    images, targets = tuple(zip(*batch))
+    images = torch.stack(images)
+    images = images.float()
+
+    boxes = [target["bboxes"].float() for target in targets]
+    labels = [target["labels"].float() for target in targets]
+    img_size = torch.stack([target["img_size"] for target in targets]).float()
+    img_scale = torch.stack([target["img_scale"] for target in targets]).float()
+
+    annotations = {
+        "bbox": boxes,
+        "cls": labels,
+        "img_size": img_size,
+        "img_scale": img_scale,
+    }
+    return images, annotations, targets

AgML/source/agml/data/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import experimental, exporters, extensions
+from .image_loader import ImageLoader
+from .loader import AgMLDataLoader
+from .point_cloud import PointCloud
+from .public import download_public_dataset, public_data_sources, source
+from .tools import coco_to_bboxes, convert_bbox_format

AgML/source/agml/data/builder.py

@@ -0,0 +1,326 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import re
+import sys
+
+from agml.backend.config import data_save_path
+from agml.framework import AgMLSerializable
+from agml.utils.downloads import download_dataset
+from agml.utils.io import get_dir_list, get_file_list, is_image_file
+
+
+class DataBuilder(AgMLSerializable):
+    """Builds an internal representation format of AgML data.
+
+    This class doesn't affect the state of the `AgMLDataLoader`, but
+    instead generates an internal representation of the data content
+    of a dataset. This allows for all data to be loaded in a standard
+    format and streamlined into the `DataManager`.
+
+    Primarily, this class attempts to locate the dataset and if unable
+    to do so, it downloads the dataset from the public bucket. Then,
+    it creates a mapping between the images and annotations which is
+    used by the `DataManager` inside the `AgMLDataLoader`.
+    """
+
+    serializable = frozenset(("name", "labels_for_image", "data", "info", "dataset_root", "data_length"))
+
+    def __init__(self, info, dataset_path, overwrite):
+        # Attempt to locate or download the dataset.
+        self._info = info
+        self._name = info.name
+        self._configure_dataset(dataset_path=dataset_path, overwrite=overwrite)
+        self._data = None
+        self._data_length = None
+        self._labels_for_image = None
+
+    @classmethod
+    def from_data(cls, contents, info, root, builder=None):
+        """Initializes the `DataBuilder` from a pre-built set of data.
+
+        This is mainly used when running `AgMLDataLoader.split`, to create
+        a `DataManager` with a split of data. Functionally, it disables
+        all of the actual generation protocols of the `DataBuilder` and
+        just pre-assigns all of the values to it.
+        """
+        obj = super(DataBuilder, cls).__new__(cls)
+        obj._name = info.name
+        obj._info = info
+        obj._dataset_root = root
+        obj._data = contents[0]
+        obj._data_length = len(obj._data)
+        obj._labels_for_image = contents[1]
+        if builder is not None:
+            if hasattr(builder, "_default_coco_annotations"):
+                obj._default_coco_annotations = DataBuilder._regenerate_coco_annotations(
+                    builder._default_coco_annotations, obj._data
+                )
+        return obj
+
+    @staticmethod
+    def _regenerate_coco_annotations(coco, data):
+        """Regenerates COCO annotations from a reduced set."""
+        image_coco = []
+        annotation_coco = []
+        image_id_tracker = 0
+
+        # Update the list of image paths.
+        for c in coco["images"]:
+            if c["file_name"] in list(data.keys()):
+                c["image_id"] = image_id_tracker
+                image_coco.append(c)
+                image_id_tracker += 1
+
+        # Update the image IDs for the annotations
+        for image_id, (_, annotation) in enumerate(data.items()):
+            for a in annotation:
+                a["id"] = image_id
+            annotation_coco.extend(annotation)
+
+        # Update the complete COCO dictionary.
+        ret_coco = coco.copy()
+        ret_coco["images"] = image_coco
+        ret_coco["annotations"] = annotation_coco
+        return ret_coco
+
+    @property
+    def dataset_root(self):
+        return self._dataset_root
+
+    def _configure_dataset(self, **kwargs):
+        """Finds and configures the dataset into the loader."""
+        # Check if the user wants to overwrite the existing dataset,
+        # and resolve the potentially provided custom dataset path.
+        overwrite = kwargs.get("overwrite", False)
+        if kwargs.get("dataset_path", None):
+            kwargs["dataset_path"] = os.path.realpath(os.path.expanduser(kwargs["dataset_path"]))
+
+        # If a custom dataset has been provided, then we assume the directory
+        # provided is the root of the dataset, e.g., if /root/datasets/dir
+        # is provided, then dir doesn't necessarily need to be the name of
+        # the dataset, but it is assumed to be its root. Note that if no path
+        # is given, then we check similarly to the default path for datasets,
+        # e.g., if there is a directory with the provided name in the custom path.
+        #
+        # No downloading phase for custom datasets, as there is nothing to do.
+        if self._info.is_custom_dataset:
+            if kwargs.get("dataset_path", False):
+                path = kwargs.get("dataset_path")
+                if os.path.exists(path):
+                    self._dataset_root = path
+                    return
+
+            elif os.path.exists(os.path.join(data_save_path(), self._name)):
+                self._dataset_root = os.path.join(data_save_path(), self._name)
+                return
+
+            # Otherwise, we don't know what to do.
+            raise OSError(
+                f"Could not find a directory for the dataset {self._name}. "
+                f"Dataset path given: {kwargs.get('dataset_path', None)}"
+            )
+
+        # If the user doesn't want to overwrite the existing contents,
+        # first check whether the dataset already exists. If so, then set
+        # the dataset root and return without doing any downloading.
+        #
+        # Note that the `dataset_path` is resolved as follows: If the path
+        # ends with the name of the dataset, e.g., `/root/datasets/<name>`,
+        # then we pop '<name>' from the end of the path in order to prevent
+        # the dataset from being downloaded to `/root/datasets/<name>/<name>`.
+        if not overwrite:
+            if kwargs.get("dataset_path", False):
+                path = kwargs.get("dataset_path")
+                if (os.path.basename(path) == self._name and os.path.exists(path)) or os.path.exists(
+                    os.path.join(path, self._name)
+                ):
+                    if os.path.basename(path) != self._name:
+                        path = os.path.join(path, self._name)
+                    self._dataset_root = path
+                    return
+
+            elif os.path.exists(os.path.join(data_save_path(), self._name)):
+                self._dataset_root = os.path.join(data_save_path(), self._name)
+                return
+
+        # If the user wants to overwrite, or the dataset doesn't exist
+        # at the path, then we create the root in the same way, except
+        # we just also download the dataset to the path.
+        else:
+            if kwargs.get("dataset_path", False):
+                path = kwargs.get("dataset_path")
+                if (os.path.basename(path) == self._name and os.path.exists(path)) or os.path.exists(
+                    os.path.join(path, self._name)
+                ):
+                    if os.path.basename(path) != self._name:
+                        path = os.path.join(path, self._name)
+                    print(f"[AgML Download]: Overwriting dataset at " f"{os.path.join(path)}")
+                    if os.path.basename(path) != self._name:
+                        path = os.path.join(path, self._name)
+                    self._dataset_root = path
+
+            elif os.path.exists(os.path.join(data_save_path(), self._name)):
+                self._dataset_root = os.path.join(data_save_path(), self._name)
+                sys.stderr.write(f"[AgML Download]: Overwriting dataset at " f"{os.path.join(self._dataset_root)}")
+
+        # Performs the actual downloading of the dataset.
+        if kwargs.get("dataset_path", False):
+            download_path = kwargs["dataset_path"]
+            if os.path.basename(download_path) != self._name:
+                download_path = os.path.join(download_path, self._name)
+        else:
+            download_path = os.path.join(data_save_path(), self._name)
+        sys.stderr.write(f"[AgML Download]: Downloading dataset " f"`{self._name}` to {download_path}.")
+        download_dataset(self._name, download_path)
+        self._dataset_root = download_path
+
+    def _generate_contents(self, task):
+        """Dispatches to a content generation method for the provided task."""
+        if self._data is not None:
+            return
+        if task == "image_classification":
+            self._generate_image_classification_data()
+        elif task == "image_regression":
+            self._generate_image_regression_data()
+        elif task == "object_detection":
+            self._generate_object_detection_data()
+        else:
+            self._generate_semantic_segmentation_data()
+
+    def get_contents(self):
+        """Extracts the internal representation of the data content."""
+        # Create the internal content representation of the dataset.
+        self._generate_contents(self._info.tasks.ml)
+
+        # Update the length of the data in the metadata dictionary for
+        # custom datasets, since they may not be providing the total number.
+        self._data_length = len(self._data)
+        if self._info._metadata.get("n_images", None) is None:
+            self._info._metadata["n_images"] = str(self._data_length)
+        return self._data
+
+    def export_contents(self, export_format):
+        """Returns the raw contents of the loader."""
+        # We start by constructing a default mapping.
+        contents = self.get_contents()
+
+        # For a COCO JSON dictionary, we have to make the full paths.
+        if self._info.tasks.ml == "object_detection":
+            paths, coco = contents.keys(), contents.values()
+            paths = [os.path.join(self._dataset_root, "images", i) for i in paths]
+            contents = dict({k: v for k, v in zip(paths, coco)})
+
+        # If the export format is `None`, we return the default mapping.
+        if export_format is None:
+            return contents
+
+        # If the export format is `arrays`, return the keys and
+        # the values of the mapping as two independent arrays.
+        if export_format == "arrays":
+            return list(contents.keys()), list(contents.values())
+
+        # A special case for COCO JSON dictionaries.
+        if export_format == "coco":
+            if self._info.tasks.ml != "object_detection":
+                raise ValueError("The `coco` export format is " "only for object detection tasks.")
+            return self._default_coco_annotations
+
+    # The following methods actually generate the content mappings for
+    # the different tasks. In essence, for each image path, `image`, a
+    # mapping is generated with a corresponding annotation, such as a
+    # label for image classification, mask for semantic segmentation, or
+    # a COCO JSON dictionary for object detection.
+
+    def _generate_image_classification_data(self):
+        """Loads image classification data for the `directory_names` format.
+
+        In this format, images are organized by class where the directory
+        they are placed in corresponds to their label in the dataset.
+        """
+        image_label_mapping = {}
+        candidate_dirs = get_dir_list(self._dataset_root)
+        for dir_ in candidate_dirs:
+            if dir_.startswith("."):
+                continue
+            dir_path = os.path.join(self._dataset_root, dir_)
+            if len(get_file_list(dir_path)) == 0:
+                continue
+            for file_ in get_file_list(dir_path):
+                file_ = os.path.join(dir_path, file_)
+                image_label_mapping[file_] = self._info.class_to_num[dir_]
+        self._data = image_label_mapping
+
+    def _generate_image_regression_data(self):
+        """Loads image regression data for the loader.
+
+        In this format, there are input images in an `images` folder as
+        well as other image formats in other various `*_images` folders,
+        and an `annotations.json` file containing the regression outputs.
+        """
+        with open(os.path.join(self._dataset_root, "annotations.json"), "r") as f:
+            annotations = json.load(f)
+        content_mapping = {"inputs": [], "outputs": []}
+        annotation_types = set(list(self._info.class_to_num.keys()))
+        annotation_types.remove("regression")
+        for sample in annotations:
+            for k, v in sample.items():
+                if is_image_file(v):
+                    sample[k] = os.path.join(self._dataset_root, f"{k}s", v)
+            content_mapping["inputs"].append({k: v for k, v in sample.items() if re.match("(.*?)image", k)})
+            out = {"regression": list(sample["outputs"]["regression"].values())}
+            out.update(
+                {k: self._info.class_to_num[k][v] for k, v in sample["outputs"].items() if k in annotation_types}
+            )
+            content_mapping["outputs"].append(out)
+        self._data = content_mapping
+
+    def _generate_semantic_segmentation_data(self):
+        """Loads semantic segmentation data for the loader.
+
+        Image data is loaded from an `images` directory, and pixel-wise
+        annotated images are loaded from an `annotations` directory.
+        """
+        image_dir = os.path.join(self._dataset_root, "images")
+        annotation_dir = os.path.join(self._dataset_root, "annotations")
+        images, annotations = (
+            sorted(get_file_list(image_dir)),
+            sorted(get_file_list(annotation_dir)),
+        )
+        image_annotation_map = {}
+        for image_path, annotation_path in zip(images, annotations):
+            image_annotation_map[os.path.join(image_dir, image_path)] = os.path.join(annotation_dir, annotation_path)
+        self._data = image_annotation_map
+
+    def _generate_object_detection_data(self):
+        """Generates object detection data for the loader.
+
+        Image data is loaded from an `images` directory, and the COCO
+        JSON annotations are loaded from an `annotations.json` file.
+        """
+        with open(os.path.join(self._dataset_root, "annotations.json")) as f:
+            self._default_coco_annotations = json.load(f)
+        coco_annotations = self._default_coco_annotations
+        image_id_mapping = {}
+        for img_meta in coco_annotations["images"]:
+            image_id_mapping[img_meta["id"]] = img_meta["file_name"]
+        coco_map = {fname: [] for fname in image_id_mapping.values()}
+        image_category_map = {k: [] for k in coco_map.keys()}
+        for a_meta in coco_annotations["annotations"]:
+            image_category_map[image_id_mapping[a_meta["image_id"]]].append(a_meta["category_id"])
+            coco_map[image_id_mapping[a_meta["image_id"]]].append(a_meta)
+        self._labels_for_image = image_category_map
+        self._data = coco_map

AgML/source/agml/data/experimental.py

@@ -0,0 +1,100 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Experimental data utilities that are in development.
+"""
+
+import functools
+
+import numpy as np
+
+__all__ = ["generate_keras_segmentation_dual_transform"]
+
+
+def generate_keras_segmentation_dual_transform(*layers):
+    """Generates a `dual_transform` pipeline from Keras preprocessing layers.
+
+    This method takes in Keras preprocessing layers and generates a
+    transformation pipeline for the `dual_transform` argument in
+    *semantic segmentation* loaders, which applies the transform in the
+    same fashion to both the image and annotation.
+
+    This is due to the fact that TensorFlow has its operation-level
+    random states different than its module-level random state, so
+    the layers need to have their seeds manually set in order to work.
+
+    In essence, for each of the preprocessing layers passed, this
+    method conducts the following operations:
+
+    > def preprocessing_transform(image, annotation):
+    >    layer = functools.partial(KerasPreprocessingLayer, **kwargs)
+    >    seed = np.random.randint(BUFFER_SIZE) # up to sys.maxsize
+    >    image = layer(image, seed = seed)
+    >    annotation = layer(annotation, seed = seed)
+    >    return image, annotation
+
+    It then repeats this transform for all of the preprocessing layers
+    passed, and returns a method which has this behavior wrapped into
+    it and can perform it when the preprocessing is actually conducted.
+
+    Parameters
+    ----------
+    layers : Any
+       Either a Sequential model with preprocessing layers, or a
+       set of instantiated preprocessing layers.
+
+    Returns
+    -------
+    """
+    import tensorflow as tf
+
+    if len(layers) == 1:
+        if isinstance(layers[0], tf.keras.Sequential):
+            layers = layers[0].layers
+
+    # These methods perform the behavior indicated in the
+    # code snippet above (for each of the layers given).
+    def _single_preprocessing_layer_base(layer_, build_dict):
+        def _internal(image, annotation, seed):
+            instantiated_layer = functools.partial(layer_, **build_dict)
+            seed_update = {}
+            if seed is not None:
+                seed_update["seed"] = seed
+            image = instantiated_layer(**seed_update)(image)
+            annotation = instantiated_layer(**seed_update)(annotation)
+            return image, annotation
+
+        return _internal
+
+    preprocessing_methods, use_seeds = [], []
+    for layer in layers:
+        config = layer.get_config()
+        if "seed" in config:
+            config.pop("seed")
+            use_seeds.append(True)
+        else:
+            use_seeds.append(False)
+        preprocessing_methods.append(_single_preprocessing_layer_base(layer.__class__, config))
+
+    def _execute_preprocessing(layers_, use_seeds_):
+        def _execute(image, annotation):
+            for p_layer, seed_ in zip(layers_, use_seeds_):
+                seed = np.random.randint(2147483647) if seed_ else None
+                image, annotation = p_layer(image, annotation, seed=seed)
+            return image, annotation
+
+        return _execute
+
+    return _execute_preprocessing(preprocessing_methods, use_seeds)

AgML/source/agml/data/exporters/__init__.py

@@ -0,0 +1 @@
+from .yolo import export_yolo

AgML/source/agml/data/exporters/tensorflow.py

@@ -0,0 +1,245 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from agml.backend.tftorch import tf
+from agml.data.managers.training import TrainState
+from agml.data.object import DataObject
+from agml.utils.logging import log
+
+
+class TFExporter(object):
+    """Exports an `AgMLDataLoader` as a `tf.data.Dataset`.
+
+    This class manages the methods and conversions between `AgMLDataLoader`
+    parameters, managers, and methods, into TensorFlow native methods.
+    """
+
+    def __init__(self, task, builder):
+        self._task = task
+        self._builder = builder
+
+    def digest_transforms(self, transforms, resizing):
+        """Parses the transforms for the `AgMLDataLoader`."""
+        self._transforms = {k: state for k, state in transforms.items()}
+        self._size = resizing if resizing is not None else (512, 512)
+
+    def assign_state(self, state):
+        """Updates the training state for the `tf.data.Dataset`."""
+        if state in [
+            TrainState.EVAL,
+            TrainState.EVAL_TF,
+            TrainState.EVAL_TORCH,
+            TrainState.FALSE,
+        ]:
+            self._state = "eval"
+        else:
+            self._state = "train"
+
+    def _build_from_contents_by_type(self, builder):
+        """Parses the provided mapping into a valid set of contents."""
+        if self._task == "image_classification":
+            ds = self._build_image_classification(builder)
+            return ds.map(self._image_classification_load)
+        elif self._task == "semantic_segmentation":
+            ds = self._build_semantic_segmentation(builder)
+            return ds.map(self._semantic_segmentation_load)
+        else:
+            ds = self._build_object_detection(builder)
+            return ds.map(self._object_detection_load)
+
+    def _apply_resizing_by_type(self, ds):
+        """Applies resizing based on the task."""
+        if self._task == "image_classification":
+            return ds.map(self._image_classification_resize)
+        elif self._task == "semantic_segmentation":
+            return ds.map(self._semantic_segmentation_resize)
+        else:
+            return ds.map(self._object_detection_resize)
+
+    def build(self, batch_size=None):
+        """Builds the `tf.data.Dataset` using the provided parameters."""
+        # Construct the dataset from the contents.
+        ds = self._build_from_contents_by_type(self._builder)
+
+        # Apply the digested transforms and resizing.
+        ds = self._apply_resizing_by_type(ds)
+        if self._state != "eval":
+            # No transforms for object detection, since it is near impossible
+            # for TensorFlow's graph mode to use COCO JSON dictionaries.
+            if self._task == "object_detection":
+                if len(self._transforms) != 0:
+                    log(
+                        "Got transforms when exporting an `AgMLDataLoader`"
+                        "to a `tf.data.Dataset`. These transforms will not be "
+                        "applied. To use transforms in TensorFlow, use the "
+                        "`as_keras_sequence()` method instead."
+                    )
+            else:
+                tfm = self._apply_transforms
+                if len(self._transforms) != 0:
+                    ds = ds.map(tfm)
+
+        # Apply batching and prefetching, then return the dataset.
+        if batch_size is not None:
+            ds = ds.batch(batch_size)
+            ds = ds.prefetch(batch_size)
+        return ds
+
+    # The following methods are used to parse the input
+    # contents into valid methods for the loaders.
+
+    @staticmethod
+    def _build_image_classification(builder):
+        images, labels = builder.export_contents(export_format="arrays")
+        images, labels = tf.constant(images), tf.constant(labels)
+        ds = tf.data.Dataset.from_tensor_slices((images, labels))
+        return ds.shuffle(len(images))
+
+    @staticmethod
+    def _build_semantic_segmentation(builder):
+        images, masks = builder.export_contents(export_format="arrays")
+        images, masks = tf.constant(images), tf.constant(masks)
+        ds = tf.data.Dataset.from_tensor_slices((images, masks))
+        return ds.shuffle(len(images))
+
+    @staticmethod
+    def _build_object_detection(builder):
+        images, annotations = builder.export_contents(export_format="arrays")
+        images = tf.constant(images)
+        processed_annotations = [DataObject._parse_coco(a) for a in annotations]
+        features = {
+            "bbox": [],
+            "category_id": [],
+            "area": [],
+            "image_id": [],
+            "iscrowd": [],
+            "segmentation": [],
+        }
+        for a_set in processed_annotations:
+            for feature in features.keys():
+                features[feature].append(a_set[feature])  # noqa
+        for feature in features.keys():
+            features[feature] = tf.ragged.constant(features[feature])
+        feature_ds = tf.data.Dataset.from_tensor_slices(features)
+        ds = tf.data.Dataset.zip((tf.data.Dataset.from_tensor_slices(images), feature_ds))
+        return ds.shuffle(len(images))
+
+    # The following methods are used for loading images and
+    # annotations for each of the different types of tasks.
+
+    @staticmethod
+    @tf.function
+    def _image_classification_load(image, label):
+        image = tf.cast(tf.image.decode_jpeg(tf.io.read_file(image)), tf.float32) / 255.0
+        return image, tf.convert_to_tensor(label)
+
+    @staticmethod
+    @tf.function
+    def _semantic_segmentation_load(image, mask):
+        image = tf.cast(tf.image.decode_jpeg(tf.io.read_file(image)), tf.float32) / 255.0
+        mask = tf.image.decode_jpeg(tf.io.read_file(mask))
+        return image, mask
+
+    @staticmethod
+    @tf.function
+    def _object_detection_load(image, coco):
+        image = tf.cast(tf.image.decode_jpeg(tf.io.read_file(image)), tf.float32) / 255.0
+        ret_coco = coco.copy()
+        for key in coco.keys():
+            try:
+                ret_coco[key] = coco[key].to_tensor()
+            except:
+                pass
+        return image, ret_coco
+
+    # The following methods apply resizing to the data.
+
+    def _image_classification_resize(self, image, label):
+        image = (tf.image.resize(image, self._size, method="nearest"), tf.float32)
+        return image, label
+
+    def _semantic_segmentation_resize(self, image, mask):
+        image = tf.cast(tf.image.resize(image, self._size, method="nearest"), tf.float32)
+        mask = tf.cast(tf.image.resize(mask, self._size, method="nearest"), tf.float32)
+        return image, mask
+
+    def _object_detection_resize(self, image, coco):
+        # Helper for the `tf.py_function` for object detection.
+        def _resize_image_and_bboxes(image, coco_boxes):
+            nonlocal size
+            y_scale, x_scale = image.shape[0:2]
+            stack_boxes = tf.stack(
+                [
+                    coco_boxes[:, 0] / x_scale,
+                    coco_boxes[:, 1] / y_scale,
+                    coco_boxes[:, 2] / x_scale,
+                    coco_boxes[:, 3] / y_scale,
+                ],
+                axis=-1,
+            )
+            image = tf.cast(tf.image.resize(image, size), tf.float32)
+            y_new, x_new = image.shape[0:2]
+            new_stack = tf.cast(
+                tf.stack(
+                    [
+                        stack_boxes[:, 0] * x_new,
+                        stack_boxes[:, 1] * y_new,
+                        stack_boxes[:, 2] * x_new,
+                        stack_boxes[:, 3] * y_new,
+                    ],
+                    axis=-1,
+                ),
+                tf.int32,
+            )
+            areas = new_stack[:, 2] * new_stack[:, 3]
+            return image, new_stack, areas
+
+        # The actual resizing can't take place in graph mode, so we
+        # dispatch to a `tf.py_function` to do the resizing, then
+        # re-assign the values back to the COCO JSON dictionary.
+        size = self._size
+        image, ret_coco_boxes, ret_areas = tf.py_function(
+            _resize_image_and_bboxes,
+            [image, coco["bbox"]],
+            [tf.float32, tf.int32, tf.int32],
+        )
+        coco["bbox"] = ret_coco_boxes
+        coco["area"] = ret_areas
+        return image, coco
+
+    # The following method manages the application of transforms.
+
+    def _apply_transforms(self, image, annotation):
+        # Helper for the `tf.py_function` for most transforms.
+        def _py_apply(img, ann):
+            nonlocal transforms
+            img, ann = img.numpy(), ann.numpy()
+            for key, state in transforms.items():
+                if key == "transform":
+                    for t in state:
+                        img = t(img)
+                elif key == "target_transform":
+                    for t in state:
+                        ann = t(ann)
+                else:
+                    for t in state:
+                        img, ann = t(img, ann)
+
+            return img, ann
+
+        # The actual transforming can't take place in graph mode
+        # (in most cases), so we dispatch and reassign.
+        transforms = self._transforms
+        image, annotation = tf.py_function(_py_apply, [image, annotation], [tf.float32, tf.int32])
+        return image, annotation

AgML/source/agml/data/exporters/yolo.py

@@ -0,0 +1,172 @@
+import os
+import shutil
+
+import yaml
+
+import agml
+from agml.utils.general import flatten
+from agml.utils.logging import log
+
+
+def convert_annotations_to_yolo_list(annotation_dict, image_shape, annotation_remapper):
+    yolo_list = []
+    height, width = image_shape
+    for annotation in annotation_dict:
+        x, y, w, h = annotation["bbox"]
+        x_center = x + w / 2
+        y_center = y + h / 2
+
+        # scale XYWH to [0, 1]
+        yolo_list.append(
+            [
+                annotation_remapper[annotation["category_id"]],
+                x_center / width,
+                y_center / height,
+                w / width,
+                h / width,
+            ]
+        )
+    return yolo_list
+
+
+def export_yolo(dataset, yolo_path=None):
+    """Exports an object detection dataset to YOLO format, ready-to-use for YOLO training.
+
+    This method will export an AgML dataset to the YOLO format, given its name and the
+    desired output location. This is so that the data is prepared within the YOLO format -
+    however, this does not translate to *integration* with the YOLO training pipeline.
+    Instead, this will simply enable you to add the dataset path to the YOLO training
+    configuration (and all other preprocessing steps are abstracted away in that same
+    pipeline, not requiring AgML).
+
+    Note that you can also use this function to export a custom dataset - in this case,
+    rather than passing in the name of a dataset simply pass in an AgMLDataLoader with
+    the dataset (and this will work for multi-dataset datasets, as well as dataset splits).
+
+    If you provide a path to a YOLO implementation, this function will automatically write
+    the corresponding files (including the `dataset.yaml` file). However, if an empty path,
+    no path, or a non-YOLO path is provided, the function will simply create a new directory
+    and write the files within a `metadata` directory there.
+
+    Additionally, if the data is already split, then `train.txt`, `val.txt`, and `test.txt`
+    files will be written to the data directory, which contain the split paths.
+
+    Parameters
+    ----------
+    dataset : {str, AgMLDataLoader}
+        The name of the dataset to export to YOLO format, or an AgMLDataLoader object.
+    yolo_path : str
+        The path to the directory where the YOLO-formatted dataset will be saved.
+
+    """
+    # if a name is provided, load the dataset
+    if isinstance(dataset, str):
+        loader = agml.data.AgMLDataLoader(dataset)
+    else:
+        loader = dataset
+
+    if yolo_path is None:
+        yolo_path = os.path.join(os.getcwd(), f"{loader.name}_yolo_export")
+    if not os.path.exists(os.path.join(yolo_path, "data")):
+        log("YOLO Export Tool did not receive a valid YOLO path. Creating a new directory for the export.")
+
+    yolo_data_path = os.path.join(yolo_path, "datasets", loader.name)
+    output_image_dir = os.path.join(yolo_data_path, "images")
+    output_annotation_dir = os.path.join(yolo_data_path, "labels")
+    os.makedirs(output_image_dir, exist_ok=True)
+    os.makedirs(output_annotation_dir, exist_ok=True)
+
+    # create the dataset YAML
+    yaml_dump = {"path": yolo_data_path, "names": loader.num_to_class}
+
+    # AgML indexes object detection datasets from 1 -> N, so reindex from 0 -> N-1
+    class_mapper = {i: i - 1 for i in yaml_dump["names"]}
+    yaml_dump["names"] = {i - 1: name for i, name in yaml_dump["names"].items()}
+
+    # get the height/width for all the images for normalization purposes
+    if loader.IS_MULTI_DATASET:
+        image_info = flatten(
+            [sub_loader._builder._default_coco_annotations["images"] for sub_loader in loader._loaders]
+        )  # noqa
+    else:
+        image_info = loader._builder._default_coco_annotations["images"]
+    image_info = {image["file_name"]: (image["height"], image["width"]) for image in image_info}
+
+    # compatibility for content format for multi-dataset loaders
+    if not loader.IS_MULTI_DATASET:
+        loader_contents = {loader.name: loader.export_contents(export_format=None)}
+    else:
+        loader_contents = loader.export_contents(export_format=None)
+
+    # check for data splits and get a list of images in each split
+    all_split_images = {}
+    if loader._is_split_generated():
+        for split_name in ["train", "val", "test"]:
+            split_content = getattr(loader, f"_{split_name}_content")
+            if not loader.IS_MULTI_DATASET:
+                split_content = {loader.name: split_content}
+
+            # skip empty splits by checking the contents
+            if all(i is None for i in split_content.values()):
+                continue
+
+            split_images = flatten(
+                [
+                    [f"{curr_name}_{os.path.basename(image)}" for image in curr_split]
+                    for curr_name, curr_split in split_content.items()
+                ]
+            )
+            all_split_images[split_name] = split_images
+    else:
+        # if no split generated, put all in `train`
+        all_split_images["train"] = flatten(
+            [
+                [f"{loader.name}_{os.path.basename(image)}" for image in curr_split]
+                for curr_split in loader_contents.values()
+            ]
+        )
+
+    # write the text files containing the split contents
+    for split_name, split_images in all_split_images.items():
+        split_images = [os.path.join(output_image_dir, image) for image in split_images]
+        with open(os.path.join(yolo_data_path, f"{split_name}.txt"), "w") as f:
+            f.write("\n".join(split_images))
+
+        # update the YAML file with the corresponding locs of train/val/test
+        yaml_dump[split_name] = f"{split_name}.txt"
+
+    for loader_name, loader_content in loader_contents.items():
+        for image, annotation_set in loader_content.items():
+            # convert from the default AgML format (COCO: [x, y, w, h])
+            # to YOLO format: (normalized [x_center, y_center, w, h])
+            image_shape = image_info[os.path.basename(image)]
+            new_annotation_set = convert_annotations_to_yolo_list(annotation_set, image_shape, class_mapper)
+            text_content = "\n".join([" ".join(map(str, annotation)) for annotation in new_annotation_set])
+
+            # the name of the new image is {dataset_name}_{old_name}.{ext}
+            image_name = f"{loader_name}_{os.path.basename(image)}"
+            txt_name = os.path.splitext(image_name)[0] + ".txt"
+
+            # save the image and the text file
+            shutil.copy(image, os.path.join(output_image_dir, image_name))
+            with open(os.path.join(output_annotation_dir, txt_name), "w") as f:
+                f.write(text_content)
+
+    # write the dataset.yaml file
+    os.makedirs(os.path.join(yolo_path, "data"), exist_ok=True)
+    with open(os.path.join(yolo_path, "data", "dataset.yaml"), "w") as f:
+        yaml.safe_dump(yaml_dump, f)
+
+    # log information about the dataset location
+    information = """
+    Dataset successfully exported to YOLO format. 
+
+    You can find the dataset at the following location: {0}
+    The dataset metadata is stored at {1}
+    """.format(yolo_data_path, os.path.join(yolo_path, "data", "dataset.yaml"))
+    log(information)
+
+    return {
+        "dataset_path": yolo_data_path,
+        "metadata_path": os.path.join(yolo_path, "data", "dataset.yaml"),
+    }

AgML/source/agml/data/extensions/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .cvat import restructure_cvat_annotations

AgML/source/agml/data/extensions/cvat.py

@@ -0,0 +1,88 @@
+# Copyright 2021 UC Davis Plant AI and Biophysics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import shutil
+
+
+def restructure_cvat_annotations(image_dir, cvat_dir, dataset_name, output_dir=None):
+    """
+    Restructure CVAT annotations and images into the AgML dataset format.
+
+    Parameters
+    ----------
+    image_dir : str
+        Path to the directory containing the images
+    cvat_dir : str
+        Path to the directory containing the CVAT annotations
+    dataset_name : str
+        Name of the new dataset
+    output_dir : str
+        Directory where the dataset will be saved. By default, saves to `~/.agml/datasets`
+
+    Returns
+    -------
+    The path to the dataset
+    """
+    image_dir = os.path.expanduser(image_dir)
+    cvat_dir = os.path.expanduser(cvat_dir)
+
+    # Determine the dataset path
+    if output_dir is None:
+        import agml.backend  # Ensure that agml is imported
+
+        dataset_path = os.path.join(agml.backend.data_save_path(), dataset_name)
+    else:
+        dataset_path = os.path.join(output_dir, dataset_name)
+
+    images_output_dir = os.path.join(dataset_path, "images")
+    os.makedirs(images_output_dir, exist_ok=True)
+
+    # Copy images to the new dataset directory
+    for image_file in os.listdir(image_dir):
+        src_image_path = os.path.join(image_dir, image_file)
+        dst_image_path = os.path.join(images_output_dir, image_file)
+        shutil.copy2(src_image_path, dst_image_path)
+
+    # Load the CVAT annotations
+    cvat_annotations_path = os.path.join(cvat_dir, "instances_default.json")
+    with open(cvat_annotations_path, "r") as f:
+        coco = json.load(f)
+
+    # Build a mapping from base image names to actual filenames (with extensions)
+    image_files = os.listdir(image_dir)
+    base_name_to_file_name = {}
+    for file_name in image_files:
+        base_name, ext = os.path.splitext(file_name)
+        base_name_to_file_name[base_name] = file_name
+
+    # Update the 'file_name' field in each image entry to match the actual filenames
+    for image in coco.get("images", []):
+        original_file_name = image["file_name"]
+        base_name, _ = os.path.splitext(original_file_name)
+        actual_file_name = base_name_to_file_name.get(base_name)
+
+        if actual_file_name:
+            image["file_name"] = actual_file_name
+        else:
+            print(f"Warning: No matching image file found for '{original_file_name}'.")
+            # Optionally, handle this case as needed (e.g., remove the image from the list)
+
+    # Save the updated annotations to 'annotations.json' in the new dataset directory
+    annotations_output_path = os.path.join(dataset_path, "annotations.json")
+    with open(annotations_output_path, "w") as f:
+        json.dump(coco, f)
+
+    print(f"Dataset '{dataset_name}' has been created successfully at '{dataset_path}'.")