matched_species_i fixed but only less number of tags issue remainign while db creation

README.md

@@ -47,7 +47,8 @@ This repository encountered several Git LFS issues during setup. Here's a summar
 
 
 while running in claude code : 
-source ~/miniconda3/etc/profile.d/conda.sh && conda activate agthinker
+source ~/miniconda3/etc/profile.d/conda.sh && conda install -c conda-forge numpy
+ate agthinker
 
 run command like example:  source ~/miniconda3/etc/profile.d/conda.sh && conda activate agllm-env1-updates-1 &&         │
 │   python whatebverscriptis.py

retrieval_evaluation.py

@@ -42,8 +42,9 @@ class QuestionGenerator:
 
         # Build context from metadata
         context_parts = []
-        if 'species' in metadata:
-            context_parts.append(f"Species: {metadata['species']}")
+        species_name = metadata.get('matched_specie_0')
+        if species_name:
+            context_parts.append(f"Species: {species_name}")
         if 'common_name' in metadata:
             context_parts.append(f"Common Name: {metadata['common_name']}")
         if 'region' in metadata:
@@ -79,7 +80,7 @@ Generate a single, clear question (no explanations, just the question):"""
         except Exception as e:
             print(f"Error generating question: {e}")
             # Fallback question
-            species = metadata.get('species', 'this species')
+            species = metadata.get('matched_specie_0', 'this species')
             return f"What IPM information is available for {species}?"
 
 class RetrievalEvaluator:
@@ -140,16 +141,31 @@ class RetrievalEvaluator:
             metadata = sample.metadata
 
             # Define filter strategies (using ChromaDB filter format)
+            species_value = metadata.get('matched_specie_0')
+            region_value = metadata.get('region')
+
             filters = {
                 'no_filter': None,
-                'species_only': {'species': {'$eq': metadata['species']}} if 'species' in metadata else None,
-                'region_only': {'region': {'$eq': metadata['region']}} if 'region' in metadata else None,
+                'species_only': {
+                    '$or': [
+                        {'matched_specie_0': {'$eq': species_value}},
+                        {'matched_specie_1': {'$eq': species_value}},
+                        {'matched_specie_2': {'$eq': species_value}}
+                    ]
+                } if species_value else None,
+                'region_only': {'region': {'$eq': region_value}} if region_value else None,
                 'species_and_region': {
                     '$and': [
-                        {'species': {'$eq': metadata['species']}},
-                        {'region': {'$eq': metadata['region']}}
+                        {
+                            '$or': [
+                                {'matched_specie_0': {'$eq': species_value}},
+                                {'matched_specie_1': {'$eq': species_value}},
+                                {'matched_specie_2': {'$eq': species_value}}
+                            ]
+                        },
+                        {'region': {'$eq': region_value}}
                     ]
-                } if 'species' in metadata and 'region' in metadata else None
+                } if species_value and region_value else None
             }
 
             for filter_name, filter_dict in filters.items():
@@ -233,6 +249,25 @@ def main():
     chunks = load_chunks_from_vectordb(VECTOR_DB_PATH, sample_size=SAMPLE_SIZE)
     print(f"   Loaded {len(chunks)} chunks")
 
+    # Step 1.5: Analyze metadata availability
+    print("\n   Metadata Analysis:")
+    matched_specie_count = sum(1 for chunk in chunks if chunk['metadata'].get('matched_specie_0'))
+    region_count = sum(1 for chunk in chunks if chunk['metadata'].get('region'))
+    both_species_region = sum(1 for chunk in chunks if chunk['metadata'].get('matched_specie_0') and chunk['metadata'].get('region'))
+
+    print(f"   - Chunks with 'matched_specie_0' field: {matched_specie_count}")
+    print(f"   - Chunks with 'region' field: {region_count}")
+    print(f"   - Chunks with species and region: {both_species_region}")
+
+    # Show sample metadata
+    if chunks:
+        sample_metadata = chunks[0]['metadata']
+        print(f"   - Sample metadata keys: {list(sample_metadata.keys())}")
+        species_field = sample_metadata.get('matched_specie_0')
+        region_field = sample_metadata.get('region')
+        print(f"   - Sample species: {species_field}")
+        print(f"   - Sample region: {region_field}")
+
     # Step 2: Generate questions for chunks
     print("\n2. Generating questions from chunks...")
     question_generator = QuestionGenerator()
@@ -273,13 +308,35 @@ def main():
     print("\n4. Evaluation Results:")
     print("=" * 50)
 
-    for pipeline_name, metrics in results.items():
-        print(f"\n{pipeline_name.upper()} Pipeline:")
-        for metric_name, values in metrics.items():
-            if isinstance(values, dict):
-                mean = values['mean']
-                std = values['std']
-                print(f"   {metric_name}: {mean:.3f} ± {std:.3f}")
+    # Create results table
+    pipelines = ['no_filter', 'species_only', 'region_only', 'species_and_region']
+    pipeline_names = ['No Filter', 'Species Only', 'Region Only', 'Species + Region']
+
+    # Precision table
+    print("\nPRECISION RESULTS:")
+    print("-" * 70)
+    print(f"{'Pipeline':<15} {'P@1':<8} {'P@3':<8} {'P@5':<8}")
+    print("-" * 70)
+
+    for i, pipeline in enumerate(pipelines):
+        if pipeline in results:
+            p1 = results[pipeline].get('precision@1', {}).get('mean', 0.0)
+            p3 = results[pipeline].get('precision@3', {}).get('mean', 0.0)
+            p5 = results[pipeline].get('precision@5', {}).get('mean', 0.0)
+            print(f"{pipeline_names[i]:<15} {p1:<8.3f} {p3:<8.3f} {p5:<8.3f}")
+
+    # nDCG table
+    print("\nnDCG RESULTS:")
+    print("-" * 70)
+    print(f"{'Pipeline':<15} {'nDCG@1':<8} {'nDCG@3':<8} {'nDCG@5':<8}")
+    print("-" * 70)
+
+    for i, pipeline in enumerate(pipelines):
+        if pipeline in results:
+            n1 = results[pipeline].get('ndcg@1', {}).get('mean', 0.0)
+            n3 = results[pipeline].get('ndcg@3', {}).get('mean', 0.0)
+            n5 = results[pipeline].get('ndcg@5', {}).get('mean', 0.0)
+            print(f"{pipeline_names[i]:<15} {n1:<8.3f} {n3:<8.3f} {n5:<8.3f}")
 
     # Save detailed results
     with open(OUTPUT_FILE, 'w') as f:
@@ -287,27 +344,5 @@ def main():
 
     print(f"\nDetailed results saved to {OUTPUT_FILE}")
 
-    # Generate comparison statement for paper
-    print("\n" + "=" * 50)
-    print("RESULTS SUMMARY FOR PAPER:")
-    print("=" * 50)
-
-    baseline = results.get('no_filter', {})
-    species_region = results.get('species_and_region', {})
-
-    if baseline and species_region:
-        for k in K_VALUES:
-            precision_baseline = baseline.get(f'precision@{k}', {}).get('mean', 0)
-            precision_filtered = species_region.get(f'precision@{k}', {}).get('mean', 0)
-            ndcg_baseline = baseline.get(f'ndcg@{k}', {}).get('mean', 0)
-            ndcg_filtered = species_region.get(f'ndcg@{k}', {}).get('mean', 0)
-
-            precision_improvement = ((precision_filtered - precision_baseline) / precision_baseline * 100) if precision_baseline > 0 else 0
-            ndcg_improvement = ((ndcg_filtered - ndcg_baseline) / ndcg_baseline * 100) if ndcg_baseline > 0 else 0
-
-            print(f"\nCompared to a region-agnostic baseline, precision@{k} improves from {precision_baseline:.3f} "
-                  f"to {precision_filtered:.3f} ({precision_improvement:+.1f}%) and nDCG@{k} from {ndcg_baseline:.3f} "
-                  f"to {ndcg_filtered:.3f} ({ndcg_improvement:+.1f}%) when using species and region filters.")
-
 if __name__ == "__main__":
     main()

retrieval_evaluation_results.json

@@ -1,130 +1,130 @@
 {
   "no_filter": {
     "precision@1": {
-      "mean": 0.75,
-      "std": 0.4330127018922193,
+      "mean": 0.55,
+      "std": 0.49749371855331,
       "count": 20
     },
     "precision@3": {
-      "mean": 0.95,
-      "std": 0.21794494717703372,
+      "mean": 0.85,
+      "std": 0.3570714214271425,
       "count": 20
     },
     "precision@5": {
-      "mean": 1.0,
-      "std": 0.0,
+      "mean": 0.9,
+      "std": 0.30000000000000004,
       "count": 20
     },
     "ndcg@1": {
-      "mean": 0.75,
-      "std": 0.4330127018922193,
+      "mean": 0.55,
+      "std": 0.49749371855331,
       "count": 20
     },
     "ndcg@3": {
-      "mean": 0.8696394630357187,
-      "std": 0.2567840676954238,
+      "mean": 0.7327324383928644,
+      "std": 0.353724839687973,
       "count": 20
     },
     "ndcg@5": {
-      "mean": 0.8911732909393884,
-      "std": 0.19311947983364772,
+      "mean": 0.7542662662965341,
+      "std": 0.319960314564507,
       "count": 20
     }
   },
   "species_only": {
     "precision@1": {
-      "mean": 1.0,
-      "std": 0.0,
-      "count": 5
+      "mean": 0.7692307692307693,
+      "std": 0.4213250442347432,
+      "count": 13
     },
     "precision@3": {
-      "mean": 1.0,
-      "std": 0.0,
-      "count": 5
+      "mean": 0.9230769230769231,
+      "std": 0.26646935501059654,
+      "count": 13
     },
     "precision@5": {
       "mean": 1.0,
       "std": 0.0,
-      "count": 5
+      "count": 13
     },
     "ndcg@1": {
-      "mean": 1.0,
-      "std": 0.0,
-      "count": 5
+      "mean": 0.7692307692307693,
+      "std": 0.4213250442347432,
+      "count": 13
     },
     "ndcg@3": {
-      "mean": 1.0,
-      "std": 0.0,
-      "count": 5
+      "mean": 0.8662968851648396,
+      "std": 0.28284691370224896,
+      "count": 13
     },
     "ndcg@5": {
-      "mean": 1.0,
-      "std": 0.0,
-      "count": 5
+      "mean": 0.8960547934136506,
+      "std": 0.19766235701592574,
+      "count": 13
     }
   },
   "region_only": {
     "precision@1": {
-      "mean": 0.75,
-      "std": 0.4330127018922193,
+      "mean": 0.6,
+      "std": 0.48989794855663565,
       "count": 20
     },
     "precision@3": {
-      "mean": 0.95,
-      "std": 0.21794494717703372,
+      "mean": 0.85,
+      "std": 0.3570714214271425,
       "count": 20
     },
     "precision@5": {
-      "mean": 1.0,
-      "std": 0.0,
+      "mean": 0.9,
+      "std": 0.30000000000000004,
       "count": 20
     },
     "ndcg@1": {
-      "mean": 0.75,
-      "std": 0.4330127018922193,
+      "mean": 0.6,
+      "std": 0.48989794855663565,
       "count": 20
     },
     "ndcg@3": {
-      "mean": 0.8696394630357187,
-      "std": 0.2567840676954238,
+      "mean": 0.7511859507142915,
+      "std": 0.3575390024008766,
       "count": 20
     },
     "ndcg@5": {
-      "mean": 0.8911732909393884,
-      "std": 0.19311947983364772,
+      "mean": 0.7727197786179613,
+      "std": 0.32294384868681797,
       "count": 20
     }
   },
   "species_and_region": {
     "precision@1": {
-      "mean": 1.0,
-      "std": 0.0,
-      "count": 5
+      "mean": 0.8461538461538461,
+      "std": 0.36080121229410994,
+      "count": 13
     },
     "precision@3": {
-      "mean": 1.0,
-      "std": 0.0,
-      "count": 5
+      "mean": 0.9230769230769231,
+      "std": 0.26646935501059654,
+      "count": 13
     },
     "precision@5": {
       "mean": 1.0,
       "std": 0.0,
-      "count": 5
+      "count": 13
     },
     "ndcg@1": {
-      "mean": 1.0,
-      "std": 0.0,
-      "count": 5
+      "mean": 0.8461538461538461,
+      "std": 0.36080121229410994,
+      "count": 13
     },
     "ndcg@3": {
-      "mean": 1.0,
-      "std": 0.0,
-      "count": 5
+      "mean": 0.8946869041208814,
+      "std": 0.27624290045474437,
+      "count": 13
     },
     "ndcg@5": {
-      "mean": 1.0,
-      "std": 0.0,
-      "count": 5
+      "mean": 0.9244448123696922,
+      "std": 0.18354431531186644,
+      "count": 13
     }
   }
 }

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0323fbf65a7d0d8cfbad75ed514829fc5d979a0d89603c61f511ed46c87dd69e
+oid sha256:4942e0dbb09693a3162b420dd2471ef8fcfaa541f479979627fa6125d12f2af6
 size 9072640

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/e82d58e5-16f1-41a6-9289-211464329861/length.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0eca7ce2600dfc137188f7b969056d2155f188796a248ab9b3b78f60431df7e
+oid sha256:e632323b84e2258a31c2401bbb859c7fc59cd994aa4f6b2217651488f3cf3be3
 size 40000