Add SLF reporting sources to knowledge base, increase retriever k to 8

Added Iowa Dept of Agriculture and ISU Extension PDFs for spotted
lanternfly reporting guidance. Rebuilt vector DB with clean sources.
Increased retriever k from 3 to 8 to match paper's evaluation range
and capture both identification and reporting chunks.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

app.py

@@ -98,7 +98,7 @@ langchain.debug=False # TODO: DOUBLE CHECK
 from langchain import globals
 globals.set_debug(False)
 
-retriever_k_value=3
+retriever_k_value=8
 embedding = OpenAIEmbeddings()
 print("Started....")
 class ChatOpenRouter(ChatOpenAI):
@@ -347,7 +347,7 @@ def initialize_qa_chain(specie_selector, application_mode, model_name, region, d
     
     # --- Create multi-region retrieval chain --- #
     class MultiRegionRetriever:
-        def __init__(self, vectordb, species_filter, selected_region, other_regions, k=3):
+        def __init__(self, vectordb, species_filter, selected_region, other_regions, k=8):
             self.vectordb = vectordb
             self.species_filter = species_filter
             self.selected_region = selected_region

examples-work-out/example5_slf_rerun_20260331_121826.json

@@ -0,0 +1,9 @@
+{
+  "species": "Lycorma delicatula",
+  "region": "Midwest USA",
+  "model": "Llama-3 70B",
+  "mode": "Researcher",
+  "question": "How can I identify and report spotted lanternfly in Iowa?",
+  "availability": "**Common Name:** Spotted Lanternfly\n\nInformation for **Lycorma delicatula** is available in region(s): **Midwest USA, USA**.",
+  "answer": "According to Midwest USA guidance:\n\n* The Iowa Department of Agriculture and Land Stewardship (IDALS) urges Iowans to be on the lookout for spotted lanternfly (SLF) insects.\n* For identification and reporting, refer to the Iowa Department of Natural Resources (DNR) resource: https://www.iowadnr.gov/Conservation/Forestry/Forest-Health/Spotted-Lanternfly\n* Additionally, Iowa State University Extension provides information on spotted lanternfly identification and reporting: https://www.extension.iastate.edu/dallas/spotted-lanternfly\n\nNo further information is available in the provided context."
+}

examples-work-out/example5_slf_rerun_20260331_122154.json

@@ -0,0 +1,9 @@
+{
+  "species": "Lycorma delicatula",
+  "region": "Midwest USA",
+  "model": "Llama-3 70B",
+  "mode": "Researcher",
+  "question": "How can I identify and report spotted lanternfly in Iowa?",
+  "availability": "**Common Name:** Spotted Lanternfly\n\nInformation for **Lycorma delicatula** is available in region(s): **Midwest USA, USA**.",
+  "answer": "According to Midwest USA guidance:\n\n* The Iowa Department of Agriculture and Land Stewardship (IDALS) urges Iowans to be on the lookout for spotted lanternfly (SLF) insects.\n* For identification and reporting, refer to the Iowa Department of Natural Resources (DNR) resource: https://www.iowadnr.gov/Conservation/Forestry/Forest-Health/Spotted-Lanternfly\n* Additionally, Iowa State University Extension provides information on spotted lanternfly identification and management: https://www.extension.iastate.edu/dallas/spotted-lanternfly\n\nNo further information is available in the provided context regarding specific identification and reporting procedures in Iowa."
+}

examples-work-out/example5_slf_rerun_20260331_122315.json

@@ -0,0 +1,9 @@
+{
+  "species": "Lycorma delicatula",
+  "region": "Midwest USA",
+  "model": "Llama-3 70B",
+  "mode": "Researcher",
+  "question": "How can I identify and report spotted lanternfly in Iowa?",
+  "availability": "**Common Name:** Spotted Lanternfly\n\nInformation for **Lycorma delicatula** is available in region(s): **Midwest USA, USA**.",
+  "answer": "According to Midwest USA guidance:\n\n* The Iowa Department of Agriculture and Land Stewardship (IDALS) urges Iowans to be on the lookout for spotted lanternfly (SLF) insects.\n* For identification and reporting, refer to the Iowa Department of Natural Resources (DNR) resource: https://www.iowadnr.gov/Conservation/Forestry/Forest-Health/Spotted-Lanternfly\n* Iowa State University Extension also provides information on identifying and managing spotted lanternfly: https://www.extension.iastate.edu/dallas/spotted-lanternfly\n\nNo additional information is available in the provided context regarding specific identification and reporting procedures in Iowa."
+}

examples-work-out/example5_slf_rerun_20260331_122623.json

@@ -0,0 +1,9 @@
+{
+  "species": "Lycorma delicatula",
+  "region": "Midwest USA",
+  "model": "Llama-3 70B",
+  "mode": "Researcher",
+  "question": "How can I identify and report spotted lanternfly in Iowa?",
+  "availability": "**Common Name:** Spotted Lanternfly\n\nInformation for **Lycorma delicatula** is available in region(s): **Midwest USA, USA**.",
+  "answer": "According to Midwest USA guidance, to identify and report spotted lanternfly in Iowa:\n\n* Be on the lookout for the eye-catching, destructive insect, which is native to Asia and was unintentionally introduced into the region.\n* Refer to the Iowa State University Extension resources, such as the Yard and Garden Encyclopedia, for identification and reporting guidelines.\n* Report any spotted lanternfly sightings to the Iowa Department of Agriculture and Land Stewardship (IDALS).\n\nNo specific identification characteristics or reporting procedures are mentioned in the provided context. If you need more information, please consult the Iowa State University Extension resources or contact IDALS directly."
+}

examples-work-out/example5_slf_rerun_20260331_122846.json

@@ -0,0 +1,9 @@
+{
+  "species": "Lycorma delicatula",
+  "region": "Midwest USA",
+  "model": "Llama-3 70B",
+  "mode": "Researcher",
+  "question": "How can I identify and report spotted lanternfly in Iowa?",
+  "availability": "**Common Name:** Spotted Lanternfly\n\nInformation for **Lycorma delicatula** is available in region(s): **Midwest USA, USA**.",
+  "answer": "**Identification and Reporting in Iowa:**\n\n* According to Midwest USA guidance, spotted lanternfly (SLF) can be identified in its different life stages:\n\t+ Egg masses: present in Iowa from late July to the following spring\n\t+ Nymphs: black with white spots (young) or red and black with white spots (older), fond of host trees, usually on branches\n\t+ Adults: pinkish-gray cicada-like with black spots, long grayish-black legs, and a second pair of red, white, and black wings beneath\n* If you suspect you've spotted a SLF in Iowa:\n\t+ Take a picture\n\t+ Collect the insect (it can be preserved in hand sanitizer)\n\t+ Note the location\n\t+ Submit photos through the Iowa Invasive Species Tracking website (https://www.eddmaps.org/reportform/test.cfm?formid=206) for confirmation\n\nNo additional information is available in the provided context from other regions."
+}

generate_usa_ipm_info.py

@@ -41,8 +41,8 @@ def get_all_unique_species() -> list[dict]:
     for df, source in [(df_midwest, "Midwest USA"), (df_africa, "Africa"), (df_india, "India")]:
         for _, row in df.iterrows():
             species = row.get("Species")
-            if pd.notna(species) and species not in seen_species:
-                seen_species.add(species)
+            if pd.notna(species) and str(species).strip().lower() not in seen_species:
+                seen_species.add(str(species).strip().lower())
                 species_list.append({
                     "species": species,
                     "common_name": row.get("Common Name", ""),
@@ -217,8 +217,15 @@ def parse_results_and_create_excel(results_path: str, output_excel_path: str):
                 print(f"Error parsing result: {e}")
                 continue
 
-    # Create DataFrame
+    # Create DataFrame — deduplicate by species name, case-insensitively
+    # (results JSONL may accumulate across runs AND across capitalization variants)
     df = pd.DataFrame(results)
+    before = len(df)
+    df['_species_lower'] = df['Species'].str.strip().str.lower()
+    df = df.drop_duplicates(subset=['_species_lower'], keep='last')
+    df = df.drop(columns=['_species_lower'])
+    if before != len(df):
+        print(f"Deduplicated {before - len(df)} entries (multiple runs + capitalization variants)")
     print(f"\nCreated DataFrame with {len(df)} US species")
     print(df.head())
 
@@ -287,6 +294,10 @@ def main():
 
     if args.step in ["process", "all"]:
         print("\n=== Step 2: Processing API requests ===")
+        # Clear results file before a new run to prevent duplicate accumulation
+        if os.path.exists(RESULTS_FILEPATH) and args.force:
+            os.remove(RESULTS_FILEPATH)
+            print(f"Cleared {RESULTS_FILEPATH} (--force run)")
         if os.path.exists(REQUESTS_FILEPATH):
             # Check if there are requests to process
             with open(REQUESTS_FILEPATH, "r") as f:

runtime.yml

@@ -1 +0,0 @@
-python_version: "3.9"

species-organized/PestID Species - Organized.xlsx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:965f330dfbc74381e712a8ea4e65633ed5a07fcae68c69dae797c1b533fdcc6b
-size 98454
+oid sha256:529c38f5fe3530c1b78256ad464ca51e53cff761fff08726d1a1ce2bddf7340a
+size 62345

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/ba4b6a9d-5012-4456-8635-7ea7d6cb7a71/index_metadata.pickle

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dd94800fffc4e1b3aca003062b6cf6e8eb3c4683b77ed4eae5f2d378cd41d183
-size 129852

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/ba4b6a9d-5012-4456-8635-7ea7d6cb7a71/link_lists.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:38d6d79326a1813390ce0d391cfa648cb550eba165548e079e7dd1f6ea240ef6
-size 12508

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/{ba4b6a9d-5012-4456-8635-7ea7d6cb7a71 → c91cad1d-966e-4715-84ef-dcf238664025}/data_level0.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0df39a7224e80c717058f5c4e95a21faf300fe9102de146ce32a9f3e2e7d5a76
+oid sha256:bb76ae0f6ca830a9a048b0cf53962a78c88c5c7fcda63fc846077d3456eb3890
 size 62840000

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/{ba4b6a9d-5012-4456-8635-7ea7d6cb7a71 → c91cad1d-966e-4715-84ef-dcf238664025}/header.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f23645792d8664b1d497ca854c4459f7c29d757472fdb6591d327f0303495f93
+oid sha256:ec666c9828420c69fc6b597461d8c18487becec1527c7d1cff9b898cbb393c2d
 size 100

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/{ba4b6a9d-5012-4456-8635-7ea7d6cb7a71 → c91cad1d-966e-4715-84ef-dcf238664025}/length.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e7e2dcff542de95352682dc186432e98f0188084896773f1973276b0577d5305
+oid sha256:23cba9764296419b54a051be0156c6daa10cd2b84f1561bcfdd19719dbb86f7f
 size 40000

vector-databases-deployed/db5-agllm-data-isu-field-insects-all-species/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8ba30aefc5e3dbf2fee25c2fe8f49eda6fe5396d31f8d8dbcc34660ed926392
-size 17203200
+oid sha256:fa868d69f3fc0a6acd546c9c8f32c4fb45980dc7d4803736e584491e78b437c6
+size 12128256