fixed count of species

README.md

@@ -14,7 +14,7 @@ license: apache-2.0
 
 ### Environment
 ```bash
-source ~/miniconda3/etc/profile.d/conda.sh && conda activate agllm-env1-updates-1
+source ~/miniconda3/etc/profile.d/conda.sh && conda activate agllm-june-15
 ```
 
 ### Key Commands
@@ -57,9 +57,14 @@ source ~/miniconda3/etc/profile.d/conda.sh && conda activate agllm-env1-updates-
 ```
 
 ### Database Build Flow
-1. PDFs loaded from `agllm-data/` (insects + weeds)
-2. Metadata read from `matched_species_results_v2.csv` files
-3. Excel sheets (India, Africa) processed from `PestID Species.xlsx`
+**US Data (80 species):**
+1. PDFs loaded from `agllm-data/.../raw-pdfs/` (content source)
+2. `matched_species_results_v2.csv` maps PDF filename → species name (metadata)
+
+**Africa/India Data (35 + 11 species):**
+3. Excel `species-organized/PestID Species - Organized.xlsx` provides both content (IPM Info) and metadata
+
+**All Data:**
 4. Documents chunked (512 tokens, 10 overlap)
 5. Tagged with `matched_specie_X` + `region` metadata
 6. Stored in ChromaDB at `vector-databases-deployed/db5-*/`

app_database_prep.py

@@ -120,8 +120,8 @@ documents = insects_documents + weeds_documents
 metadata_raw = pd.concat([insects_metadata_raw, weeds_metadata_raw], ignore_index=True)
 
 
-## Load Excel File Path (Define once)
-excel_file_path = "agllm-data/PestID Species.xlsx"
+## Load Excel File Path (Define once) - Using Organized file as single source of truth
+excel_file_path = "species-organized/PestID Species - Organized.xlsx"
 
 
 ## Process PDF documents using CSV → PDF approach

retrieval_evaluation_results.json

@@ -1,109 +1,109 @@
 {
   "no_filter": {
     "precision@1": {
-      "mean": 0.59,
-      "std": 0.49183330509431744,
+      "mean": 0.55,
+      "std": 0.49749371855331,
       "count": 100
     },
     "precision@3": {
-      "mean": 0.76,
-      "std": 0.42708313008125254,
+      "mean": 0.74,
+      "std": 0.4386342439892262,
       "count": 100
     },
     "precision@5": {
-      "mean": 0.82,
-      "std": 0.38418745424597095,
+      "mean": 0.8,
+      "std": 0.4,
       "count": 100
     },
     "ndcg@1": {
-      "mean": 0.59,
-      "std": 0.49183330509431744,
+      "mean": 0.55,
+      "std": 0.49749371855331,
       "count": 100
     },
     "ndcg@3": {
-      "mean": 0.6946394630357184,
-      "std": 0.41495405707705707,
+      "mean": 0.6620208679642894,
+      "std": 0.4224663020789173,
       "count": 100
     },
     "ndcg@5": {
-      "mean": 0.7182888689781796,
-      "std": 0.3848500116841757,
+      "mean": 0.6865467489235274,
+      "std": 0.39428047037500696,
       "count": 100
     }
   },
   "species_only": {
     "precision@1": {
-      "mean": 0.73,
-      "std": 0.4439594576084623,
+      "mean": 0.72,
+      "std": 0.4489988864128729,
       "count": 100
     },
     "precision@3": {
-      "mean": 0.98,
-      "std": 0.13999999999999999,
+      "mean": 1.0,
+      "std": 0.0,
       "count": 100
     },
     "precision@5": {
-      "mean": 0.99,
-      "std": 0.09949874371066199,
+      "mean": 1.0,
+      "std": 0.0,
       "count": 100
     },
     "ndcg@1": {
-      "mean": 0.73,
-      "std": 0.4439594576084623,
+      "mean": 0.72,
+      "std": 0.4489988864128729,
       "count": 100
     },
     "ndcg@3": {
-      "mean": 0.8811859507142915,
-      "std": 0.2136019453378135,
+      "mean": 0.8861859507142915,
+      "std": 0.18517270734359137,
       "count": 100
     },
     "ndcg@5": {
-      "mean": 0.8850544787866369,
-      "std": 0.2007226726424171,
+      "mean": 0.8861859507142915,
+      "std": 0.18517270734359137,
       "count": 100
     }
   },
   "region_only": {
     "precision@1": {
-      "mean": 0.61,
-      "std": 0.4877499359302879,
+      "mean": 0.56,
+      "std": 0.4963869458396343,
       "count": 100
     },
     "precision@3": {
-      "mean": 0.77,
-      "std": 0.4208325082500163,
+      "mean": 0.74,
+      "std": 0.4386342439892262,
       "count": 100
     },
     "precision@5": {
-      "mean": 0.83,
-      "std": 0.375632799419859,
+      "mean": 0.82,
+      "std": 0.38418745424597095,
       "count": 100
     },
     "ndcg@1": {
-      "mean": 0.61,
-      "std": 0.4877499359302879,
+      "mean": 0.56,
+      "std": 0.4963869458396343,
       "count": 100
     },
     "ndcg@3": {
-      "mean": 0.7083301655000039,
-      "std": 0.4110942789611411,
+      "mean": 0.6683301655000039,
+      "std": 0.423160630771083,
       "count": 100
     },
     "ndcg@5": {
-      "mean": 0.7319795714424648,
-      "std": 0.37983366654728035,
+      "mean": 0.7010313401123213,
+      "std": 0.38430545848027436,
       "count": 100
     }
   },
   "species_and_region": {
     "precision@1": {
-      "mean": 0.75,
-      "std": 0.4330127018922193,
+      "mean": 0.74,
+      "std": 0.4386342439892262,
       "count": 100
     },
     "precision@3": {
-      "mean": 0.98,
-      "std": 0.13999999999999999,
+      "mean": 1.0,
+      "std": 0.0,
       "count": 100
     },
     "precision@5": {
@@ -112,18 +112,18 @@
       "count": 100
     },
     "ndcg@1": {
-      "mean": 0.75,
-      "std": 0.4330127018922193,
+      "mean": 0.74,
+      "std": 0.4386342439892262,
       "count": 100
     },
     "ndcg@3": {
-      "mean": 0.8898766531785769,
-      "std": 0.2091728695998248,
+      "mean": 0.8961859507142915,
+      "std": 0.17738436382801476,
       "count": 100
     },
     "ndcg@5": {
-      "mean": 0.8980519468316562,
-      "std": 0.18024378480878206,
+      "mean": 0.8961859507142915,
+      "std": 0.17738436382801476,
       "count": 100
     }
   }

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/{0bdb47f3-00af-43ed-a2af-ae5a3eee5f98 → 8577d142-4583-4e84-bf32-088c2e98b3b6}/length.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:11491cf0eac47e805aa1b059bb8d72b895d20b41d24581b6a4383eff57db12f5
+oid sha256:1fa22d78fd3b20a685b167ad464489ed4470048542d3ee1c780b316fdcb38211
 size 40000

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:12653e79b55a19108699f56736a4d97a4ad00f3627d6504348862d911eaa1688
-size 5410816
+oid sha256:97bfefa450a9f891951d6d1f06ec715f9e3b072afd569ba010315ee9034d2ec6
+size 5423104