... initial commit

.gitattributes

@@ -1,35 +1,3 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.gguf filter=lfs diff=lfs merge=lfs -text
+cover[[:space:]]photo.png filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.env

Modelfile

@@ -0,0 +1,61 @@
+FROM ./agri_llama.Q4_K_M.gguf
+
+TEMPLATE """
+{{ if .Messages }}
+{{- if or .System .Tools }}<|start_header_id|>system<|end_header_id|>
+{{- if .System }}
+
+{{ .System }}
+{{- end }}
+{{- if .Tools }}
+
+You are a helpful assistant specialized in providing information about plant diseases to Indian farmers. When you receive a tool call response, use the output to format an answer to the original user question.
+{{- end }}
+{{- end }}<|eot_id|>
+{{- range $i, $_ := .Messages }}
+{{- $last := eq (len (slice $.Messages $i)) 1 }}
+{{- if eq .Role "user" }}<|start_header_id|>user<|end_header_id|>
+{{- if and $.Tools $last }}
+
+Given the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt.
+
+Respond in the format {"name": "function_name", "parameters": {"argument_name": "argument_value"}}. Do not use variables.
+
+{{ $.Tools }}
+{{- end }}
+
+{{ .Content }}<|eot_id|>{{ if $last }}<|start_header_id|>assistant<|end_header_id|>
+
+{{ end }}
+{{- else if eq .Role "assistant" }}<|start_header_id|>assistant<|end_header_id|>
+{{- if .ToolCalls }}
+
+{{- range .ToolCalls }}{"name": "{{ .Function.Name }}", "parameters": {{ .Function.Arguments }}}{{ end }}
+{{- else }}
+
+{{ .Content }}{{ if not $last }}<|eot_id|>{{ end }}
+{{- end }}
+{{- else if eq .Role "tool" }}<|start_header_id|>ipython<|end_header_id|>
+
+{{ .Content }}<|eot_id|>{{ if $last }}<|start_header_id|>assistant<|end_header_id|>
+
+{{ end }}
+{{- end }}
+{{- end }}
+{{- else }}
+{{- if .System }}<|start_header_id|>system<|end_header_id|>
+
+{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>
+
+{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>
+
+{{ end }}{{ .Response }}{{ if .Response }}<|eot_id|>{{ end }}
+"""
+PARAMETER stop "<|start_header_id|>"
+PARAMETER stop "<|end_header_id|>"
+PARAMETER stop "<|eot_id|>"
+PARAMETER stop "<|eom_id|>"
+PARAMETER temperature 0.6
+PARAMETER top_k 50
+PARAMETER top_p 0.8
+PARAMETER min_p 0.3

README.md

@@ -1,3 +1,147 @@
 ---
-license: apache-2.0
+title: "AgriLlama: Plant Disease Information Assistant"
+description: "AgriLlama is a fine-tuned large language model based on Llama3.2:1B, designed to assist Indian farmers with plant disease identification and management."
+version: "1.0"
+author: "Sike Aditya"
+repository: "https://huggingface.co/sikeaditya/agri_assist_llm"
+tags:
+  - Agriculture
+  - Plant Disease
+  - LLM
+  - AI
+  - India
+model:
+  base: "meta/Llama3.2:1B"
+  fine_tuning_dataset: "1000 samples focusing on plant diseases in Indian agriculture"
+  format: "Alpaca Instruct Format"
+usage:
+  - "Farmers"
+  - "Agronomists"
+  - "Agricultural extension workers"
+installation:
+  ollama: "curl -fsSL https://ollama.ai/install.sh | sh"
+usage_examples:
+  - command: "ollama run AgriLlama 'Explain Red Rot in sugarcane in simple terms for Indian farmers.'"
+    description: "Provides an easy-to-understand explanation of Red Rot disease in sugarcane."
+dataset:
+  crops:
+    Sugarcane:
+      - "Bacterial Blight"
+      - "Healthy"
+      - "Red Rot"
+    Maize:
+      - "Blight"
+      - "Common Rust"
+      - "Gray Leaf Spot"
+      - "Healthy"
+    Cotton:
+      - "Bacterial Blight"
+      - "Curl Virus"
+      - "Fusarium Wilt"
+      - "Healthy"
+    Rice:
+      - "Bacterial Blight"
+      - "Blast"
+      - "Brownspot"
+      - "Tungro"
+    Wheat:
+      - "Healthy"
+      - "Septoria"
+      - "Strip Rust"
+contact:
+  email: "sikeaditya@example.com"
+  issues: "https://github.com/sikeaditya/agri_assist_llm/issues"
 ---
+# AgriLlama: Plant Disease Information Assistant
+
+AgriLlama is a fine-tuned large language model based on Llama3.2:1B, specifically designed to provide detailed, actionable information about plant diseases to Indian farmers. It offers clear, concise, and locally relevant guidance on disease identification, symptoms, causes, severity, and treatment measures across major crops such as Sugarcane, Maize, Cotton, Rice, and Wheat.
+
+## Features
+
+- **Tailored Guidance:** Provides comprehensive details on various plant diseases affecting Indian crops.
+- **Practical Recommendations:** Offers clear instructions on treatment and prevention, helping farmers manage crop health.
+- **User-Friendly:** Utilizes the Alpaca Instruct Format to generate responses in simple, accessible language.
+- **Versatile Applications:** Suitable for use by farmers, agronomists, and agricultural extension workers.
+
+## Model Details
+
+- **Base Model:** Llama3.2:1B
+- **Fine-Tuning Dataset:** Custom dataset of 200 samples focusing on plant diseases in Indian agriculture.
+- **Intended Use:** Assisting in the identification, explanation, and management of plant diseases.
+
+## Installation
+
+To use AgriLlama, install the required libraries:
+
+```bash
+pip install transformers torch
+```
+
+## Usage
+
+### Using Hugging Face Transformers
+
+Here’s an example of how to use AgriLlama with the Hugging Face Transformers library:
+
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+# Load the tokenizer and model from the Hugging Face Hub
+tokenizer = AutoTokenizer.from_pretrained("your-username/AgriLlama")
+model = AutoModelForCausalLM.from_pretrained("your-username/AgriLlama")
+# Define a prompt
+prompt = "Explain Red Rot in sugarcane in simple terms for Indian farmers."
+# Tokenize and generate a response
+inputs = tokenizer(prompt, return_tensors="pt")
+outputs = model.generate(**inputs, max_new_tokens=256)
+# Decode and print the generated response
+print(tokenizer.decode(outputs[0], skip_special_tokens=True))
+```
+
+*Note:* Replace `your-username/AgriLlama` with the actual path of your repository.
+
+### Using Ollama
+
+You can also use AgriLlama with [Ollama](https://ollama.ai), a simple way to run large language models locally.
+
+1. Install Ollama if you haven't already:
+
+```bash
+curl -fsSL https://ollama.ai/install.sh | sh
+```
+
+2. Pull the model from Ollama [Library](https://ollama.com/sike_aditya/AgriLlama)
+```bash
+ollama pull sike_aditya/AgriLlama
+```
+
+3. Run the model using Ollama:
+
+```bash
+ollama run AgriLlama "Explain Red Rot in sugarcane in simple terms for Indian farmers."
+```
+
+This will generate a response based on the model’s fine-tuned dataset.
+
+## Fine-Tuning and Training
+
+AgriLlama was fine-tuned using a custom dataset created in the Alpaca Instruct Format. The dataset covers detailed plant disease information tailored to the Indian context and includes samples for:
+
+- **Sugarcane:** Bacterial Blight, Healthy, Red Rot
+- **Maize:** Blight, Common Rust, Gray Leaf Spot, Healthy
+- **Cotton:** Bacterial Blight, Curl Virus, Fusarium Wilt, Healthy
+- **Rice:** Bacterial Blight, Blast, Brownspot, Tungro
+- **Wheat:** Healthy, Septoria, Strip Rust
+
+## Dataset
+
+The fine-tuning dataset consists of carefully curated samples that provide comprehensive, accurate information designed to help users manage crop diseases effectively.
+
+
+## Contact
+
+For questions or suggestions, please open an issue in the repository or contact the authors directly.
+
+---
+
+Happy farming with AgriLlama!
+

README_hf.md

@@ -0,0 +1,147 @@
+---
+title: "AgriAssist-LLM: Plant Disease Information Assistant"
+description: "AgriAssist-LLM is a fine-tuned large language model based on Llama3.2:1B, designed to assist Indian farmers with plant disease identification and management."
+version: "1.0"
+author: "Sike Aditya"
+repository: "https://huggingface.co/sikeaditya/agri_assist_llm"
+tags:
+  - Agriculture
+  - Plant Disease
+  - LLM
+  - AI
+  - India
+model:
+  base: "meta/Llama3.2:1B"
+  fine_tuning_dataset: "1000 samples focusing on plant diseases in Indian agriculture"
+  format: "Alpaca Instruct Format"
+usage:
+  - "Farmers"
+  - "Agronomists"
+  - "Agricultural extension workers"
+installation:
+  ollama: "curl -fsSL https://ollama.ai/install.sh | sh"
+usage_examples:
+  - command: "ollama run AgriAssist-LLM 'Explain Red Rot in sugarcane in simple terms for Indian farmers.'"
+    description: "Provides an easy-to-understand explanation of Red Rot disease in sugarcane."
+dataset:
+  crops:
+    Sugarcane:
+      - "Bacterial Blight"
+      - "Healthy"
+      - "Red Rot"
+    Maize:
+      - "Blight"
+      - "Common Rust"
+      - "Gray Leaf Spot"
+      - "Healthy"
+    Cotton:
+      - "Bacterial Blight"
+      - "Curl Virus"
+      - "Fusarium Wilt"
+      - "Healthy"
+    Rice:
+      - "Bacterial Blight"
+      - "Blast"
+      - "Brownspot"
+      - "Tungro"
+    Wheat:
+      - "Healthy"
+      - "Septoria"
+      - "Strip Rust"
+contact:
+  email: "sikeaditya@example.com"
+  issues: "https://github.com/sikeaditya/agri_assist_llm/issues"
+---
+# AgriAssist-LLM: Plant Disease Information Assistant
+
+AgriAssist-LLM is a fine-tuned large language model based on Llama3.2:1B, specifically designed to provide detailed, actionable information about plant diseases to Indian farmers. It offers clear, concise, and locally relevant guidance on disease identification, symptoms, causes, severity, and treatment measures across major crops such as Sugarcane, Maize, Cotton, Rice, and Wheat.
+
+## Features
+
+- **Tailored Guidance:** Provides comprehensive details on various plant diseases affecting Indian crops.
+- **Practical Recommendations:** Offers clear instructions on treatment and prevention, helping farmers manage crop health.
+- **User-Friendly:** Utilizes the Alpaca Instruct Format to generate responses in simple, accessible language.
+- **Versatile Applications:** Suitable for use by farmers, agronomists, and agricultural extension workers.
+
+## Model Details
+
+- **Base Model:** Llama3.2:1B
+- **Fine-Tuning Dataset:** Custom dataset of 200 samples focusing on plant diseases in Indian agriculture.
+- **Intended Use:** Assisting in the identification, explanation, and management of plant diseases.
+
+## Installation
+
+To use AgriAssist-LLM, install the required libraries:
+
+```bash
+pip install transformers torch
+```
+
+## Usage
+
+### Using Hugging Face Transformers
+
+Here’s an example of how to use AgriAssist-LLM with the Hugging Face Transformers library:
+
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+# Load the tokenizer and model from the Hugging Face Hub
+tokenizer = AutoTokenizer.from_pretrained("your-username/AgriAssist-LLM")
+model = AutoModelForCausalLM.from_pretrained("your-username/AgriAssist-LLM")
+# Define a prompt
+prompt = "Explain Red Rot in sugarcane in simple terms for Indian farmers."
+# Tokenize and generate a response
+inputs = tokenizer(prompt, return_tensors="pt")
+outputs = model.generate(**inputs, max_new_tokens=256)
+# Decode and print the generated response
+print(tokenizer.decode(outputs[0], skip_special_tokens=True))
+```
+
+*Note:* Replace `your-username/AgriAssist-LLM` with the actual path of your repository.
+
+### Using Ollama
+
+You can also use AgriAssist-LLM with [Ollama](https://ollama.ai), a simple way to run large language models locally.
+
+1. Install Ollama if you haven't already:
+
+```bash
+curl -fsSL https://ollama.ai/install.sh | sh
+```
+
+2. Pull the model from Hugging Face and convert it to an Ollama-compatible format.
+```bash
+ollama run hf.co/sikeaditya/agri_assist_llm
+```
+
+3. Run the model using Ollama:
+
+```bash
+ollama run AgriAssist-LLM "Explain Red Rot in sugarcane in simple terms for Indian farmers."
+```
+
+This will generate a response based on the model’s fine-tuned dataset.
+
+## Fine-Tuning and Training
+
+AgriAssist-LLM was fine-tuned using a custom dataset created in the Alpaca Instruct Format. The dataset covers detailed plant disease information tailored to the Indian context and includes samples for:
+
+- **Sugarcane:** Bacterial Blight, Healthy, Red Rot
+- **Maize:** Blight, Common Rust, Gray Leaf Spot, Healthy
+- **Cotton:** Bacterial Blight, Curl Virus, Fusarium Wilt, Healthy
+- **Rice:** Bacterial Blight, Blast, Brownspot, Tungro
+- **Wheat:** Healthy, Septoria, Strip Rust
+
+## Dataset
+
+The fine-tuning dataset consists of carefully curated samples that provide comprehensive, accurate information designed to help users manage crop diseases effectively.
+
+
+## Contact
+
+For questions or suggestions, please open an issue in the repository or contact the authors directly.
+
+---
+
+Happy farming with AgriAssist-LLM!
+

agri_llama.Q4_K_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fc600b570a00d3eacc8a51e1b6e4ec3737285e5ad116b548cd095cb9e3d8e09
+size 807694784

agri_synthetic_data_generated_openrouter.jsonl

@@ -0,0 +1,2 @@
+{"text": "<bos><start_of_turn>user\nAnalyze the provided description of plant symptoms and provide a detailed technical diagnosis and recommendations.\nRice - Blast: The image shows multiple leaves of a rice plant during the tillering stage. Many leaves exhibit diamond-shaped lesions, approximately 1-2 cm long, with grayish-white centers and distinct dark brown borders. Some lesions are merging, causing larger blighted areas. No insects are visible.<end_of_turn>\n<start_of_turn>model\nThe described symptoms are characteristic of Rice Blast disease, caused by the fungus *Magnaporthe oryzae*. The diamond-shaped lesions with grayish-white centers and dark brown borders are typical of this disease. The presence of these lesions, especially during the tillering stage, can significantly impact the plant's ability to produce healthy tillers and subsequently affect grain yield.\n\nDetailed Technical Diagnosis:\n1. **Disease Identification**: The symptoms described are classic for Rice Blast, a major fungal disease affecting rice crops worldwide.\n2. **Causal Pathogen**: The disease is caused by *Magnaporthe oryzae*, a fungus that thrives in humid environments with moderate temperatures.\n3. **Impact**: The disease can cause significant yield loss by damaging leaves and potentially infecting neck nodes, leading to neck blast, which directly affects grain filling.\n\nRecommendations:\n1. **Cultural Practices**: Implement cultural practices that reduce disease pressure, such as using resistant varieties, adjusting planting dates, and ensuring balanced nitrogen fertilization.\n2. **Chemical Control**: Apply fungicides that are effective against *Magnaporthe oryzae*. Systemic fungicides, such as those from the triazole group, can be effective when applied at the onset of disease symptoms or as a preventive measure during high disease pressure periods.\n3. **Monitoring**: Regularly monitor the field for early detection of disease symptoms to enable timely intervention.\n4. **Sanitation**: Remove infected crop residues to reduce the source of inoculum for subsequent crops.\n\nBy adopting these strategies, the impact of Rice Blast can be minimized, and yield losses can be reduced. Regular monitoring and a combination of cultural and chemical controls are key to managing this disease effectively.<end_of_turn>"}
+{"text": "<bos><start_of_turn>user\nBriefly identify the problem shown in the description and give the main point.\nRice - Blast: The image shows multiple leaves of a rice plant during the tillering stage. Many leaves exhibit diamond-shaped lesions, approximately 1-2 cm long, with grayish-white centers and distinct dark brown borders. Some lesions are merging, causing larger blighted areas. No insects are visible.<end_of_turn>\n<start_of_turn>model\nThe problem shown is Rice Blast disease, characterized by diamond-shaped lesions on the leaves with grayish-white centers and dark brown borders. The main point is that the disease is causing significant damage to the rice leaves during the tillering stage.<end_of_turn>"}

data.py

@@ -0,0 +1,327 @@
+import google.generativeai as genai
+import os
+import json
+import time
+import re
+import math
+import sys # To check for file existence and handle errors
+
+# --- Configuration ---
+API_KEY = "AIzaSyChwTleVWKrti4nQmFJAA5aFLuZPXYf7ds"
+if not API_KEY:
+    raise ValueError("Please set the GOOGLE_API_KEY environment variable.")
+
+MODEL_NAME = "gemini-1.5-flash-latest" # Or "gemini-1.5-pro-latest"
+
+# Your list of detailed input descriptions (Keep the full list here)
+input_descriptions = [
+    # === PASTE THE 20 DESCRIPTIONS FROM THE PREVIOUS ANSWER HERE ===
+    """Rice - Blast: The image shows multiple leaves of a rice plant during the tillering stage. Many leaves exhibit diamond-shaped lesions, approximately 1-2 cm long, with grayish-white centers and distinct dark brown borders. Some lesions are merging, causing larger blighted areas. No insects are visible.
+Cotton - Bollworm: Close-up view of a developing cotton boll. Several small (<1cm), pinkish-white larvae are seen boring into the boll surface near the bracts. Small entry holes with some dark, granular frass (excrement) are visible. The surrounding leaves appear relatively healthy.
+Maize - Nitrogen Deficiency: Image displays the lower leaves of a young maize plant. The leaves show distinct yellowing starting at the leaf tip and progressing down the midrib in a characteristic V-shape pattern. The leaf margins remain relatively green initially. Upper leaves appear greener.
+Tomato - Early Blight: Several lower leaves of a mature tomato plant are shown. They exhibit circular to irregular dark brown lesions, 0.5-1.5 cm in diameter. Many lesions display characteristic concentric rings, giving a 'target board' appearance. Significant yellowing (chlorosis) surrounds the lesions, and some affected leaves are starting to wither.
+Wheat - Stripe Rust: Picture shows the upper leaves of a wheat plant nearing the flag leaf stage. Numerous small, bright yellow-orange pustules are arranged in distinct, long stripes running parallel to the leaf veins. When touched, a yellowish powder (spores) rubs off.
+Chili - Leaf Curl Virus: The image focuses on the growing tip of a chili plant. The young leaves are severely deformed, showing upward curling, puckering, and twisting. The leaves appear thickened and brittle, and veins are sometimes noticeably thicker. Plant growth appears stunted overall. Tiny whitefly insects may be subtly present but are not the primary focus.
+Groundnut - Tikka Disease (Leaf Spot): Several leaves of a groundnut plant are displayed. They are covered with numerous circular, dark brown to black spots, about 2-8 mm wide. Each spot is surrounded by a distinct yellow halo. Some older spots on lower leaves might lack the clear halo.
+Soybean - Healthy: Image shows a section of a healthy, vigorously growing soybean plant canopy during the vegetative stage. The leaves are uniformly deep green, fully expanded, and show no signs of spotting, yellowing, mosaic patterns, insect feeding, or wilting. Stems appear sturdy.
+Mango - Anthracnose on Fruit: A near-ripe mango fruit is shown. It displays several sunken, irregular-shaped black spots on the peel. Some spots appear to be coalescing into larger dark lesions. In humid conditions, pinkish spore masses might be visible in the center of older spots (though not clearly visible here).
+Sugarcane - Early Shoot Borer Damage: Focus is on the base of young sugarcane tillers (shoots). The central whorl of leaves is dried up, creating a characteristic "dead heart" symptom, which can be easily pulled out. Small bore holes may be visible near the base of the affected shoot.
+Potato - Late Blight: Image shows potato leaves with large, irregular, water-soaked lesions, often starting at the leaf margins or tips. The lesions rapidly turn dark brown to black. A fuzzy white fungal growth (sporangiophores) might be visible on the underside of the leaves near the edge of the lesions, especially in moist conditions.
+Brinjal (Eggplant) - Aphid Infestation: Close-up shows the underside of several young brinjal leaves and stems. They are heavily infested with clusters of small, pear-shaped insects, mostly greenish or blackish in color. Some shiny, sticky honeydew secretions might be visible on the leaf surfaces below the infestation.
+Okra (Lady's Finger) - Yellow Vein Mosaic Virus: The image shows several leaves of an okra plant. The veins of the leaves are prominently yellow, and the yellow network extends into the surrounding green leaf tissue, creating a distinct mosaic pattern. The leaves might be slightly reduced in size.
+Cabbage - Diamondback Moth Larvae: Several outer leaves of a cabbage head are shown. Small (around 1 cm), greenish larvae with a slightly tapered body are visible, actively feeding on the leaf tissue. They have created irregular holes or "window panes" (where only one layer of the leaf epidermis is left). Some silken threads might be present.
+Pigeon Pea (Tur/Arhar) - Pod Borer Damage: Image shows several developing pigeon pea pods. Some pods have distinct circular bore holes on the surface. One pod is broken open slightly, revealing a visible greenish or brownish larva feeding inside on the developing seeds. Frass might be present near the holes.
+Mustard - White Rust: Focus is on mustard leaves and potentially flower stalks. White, blister-like pustules, somewhat raised and initially smooth, are scattered on the underside of the leaves. Corresponding yellow spots may be visible on the upper leaf surface. Affected flower parts can become swollen and distorted.
+Apple - Powdery Mildew: Young leaves and shoot tips of an apple sapling are shown. They are covered with a characteristic white to grayish powdery fungal growth. Affected leaves may appear distorted, curled, or stunted.
+Grapes - Downy Mildew: Image displays the upper surface of grape leaves showing distinct, angular, yellowish-green spots ("oil spots"). On the underside of the leaf, corresponding to these spots, a white, downy fungal growth is visible, especially under humid conditions.
+Papaya - Mealybug Infestation: Close-up on the stem and leaf petioles of a papaya plant, especially near the growing point. Clusters of white, cottony masses are visible, covering small, oval, soft-bodied insects (mealybugs). Ants may also be present, attracted to the honeydew secreted by the mealybugs.
+Banana - Sigatoka Leaf Spot: Several mature banana leaves are shown. They exhibit numerous small, elliptical spots parallel to the leaf veins. Initially yellowish-green, the spots enlarge, become dark brown or black, and often develop a grayish center surrounded by a dark border and a yellow halo. Severe spotting leads to leaf necrosis.
+""",
+    "The image shows multiple leaves of a rice plant during the tillering stage. Many leaves exhibit diamond-shaped lesions, approximately 1-2 cm long, with grayish-white centers and distinct dark brown borders. Some lesions are merging, causing larger blighted areas. No insects are visible.",
+    "Close-up view of a developing cotton boll. Several small (<1cm), pinkish-white larvae are seen boring into the boll surface near the bracts. Small entry holes with some dark, granular frass (excrement) are visible. The surrounding leaves appear relatively healthy.",
+    "Image displays the lower leaves of a young maize plant. The leaves show distinct yellowing starting at the leaf tip and progressing down the midrib in a characteristic V-shape pattern. The leaf margins remain relatively green initially. Upper leaves appear greener.",
+    "Several lower leaves of a mature tomato plant are shown. They exhibit circular to irregular dark brown lesions, 0.5-1.5 cm in diameter. Many lesions display characteristic concentric rings, giving a 'target board' appearance. Significant yellowing (chlorosis) surrounds the lesions, and some affected leaves are starting to wither.",
+    "Picture shows the upper leaves of a wheat plant nearing the flag leaf stage. Numerous small, bright yellow-orange pustules are arranged in distinct, long stripes running parallel to the leaf veins. When touched, a yellowish powder (spores) rubs off.",
+    "The image focuses on the growing tip of a chili plant. The young leaves are severely deformed, showing upward curling, puckering, and twisting. The leaves appear thickened and brittle, and veins are sometimes noticeably thicker. Plant growth appears stunted overall. Tiny whitefly insects may be subtly present but are not the primary focus.",
+    "Several leaves of a groundnut plant are displayed. They are covered with numerous circular, dark brown to black spots, about 2-8 mm wide. Each spot is surrounded by a distinct yellow halo. Some older spots on lower leaves might lack the clear halo.",
+    "Image shows a section of a healthy, vigorously growing soybean plant canopy during the vegetative stage. The leaves are uniformly deep green, fully expanded, and show no signs of spotting, yellowing, mosaic patterns, insect feeding, or wilting. Stems appear sturdy.",
+    "A near-ripe mango fruit is shown. It displays several sunken, irregular-shaped black spots on the peel. Some spots appear to be coalescing into larger dark lesions. In humid conditions, pinkish spore masses might be visible in the center of older spots (though not clearly visible here).",
+    "Focus is on the base of young sugarcane tillers (shoots). The central whorl of leaves is dried up, creating a characteristic \"dead heart\" symptom, which can be easily pulled out. Small bore holes may be visible near the base of the affected shoot.",
+    "Image shows potato leaves with large, irregular, water-soaked lesions, often starting at the leaf margins or tips. The lesions rapidly turn dark brown to black. A fuzzy white fungal growth (sporangiophores) might be visible on the underside of the leaves near the edge of the lesions, especially in moist conditions.",
+    "Close-up shows the underside of several young brinjal leaves and stems. They are heavily infested with clusters of small, pear-shaped insects, mostly greenish or blackish in color. Some shiny, sticky honeydew secretions might be visible on the leaf surfaces below the infestation.",
+    "The image shows several leaves of an okra plant. The veins of the leaves are prominently yellow, and the yellow network extends into the surrounding green leaf tissue, creating a distinct mosaic pattern. The leaves might be slightly reduced in size.",
+    "Several outer leaves of a cabbage head are shown. Small (around 1 cm), greenish larvae with a slightly tapered body are visible, actively feeding on the leaf tissue. They have created irregular holes or \"window panes\" (where only one layer of the leaf epidermis is left). Some silken threads might be present.",
+    "Image shows several developing pigeon pea pods. Some pods have distinct circular bore holes on the surface. One pod is broken open slightly, revealing a visible greenish or brownish larva feeding inside on the developing seeds. Frass might be present near the holes.",
+    "Focus is on mustard leaves and potentially flower stalks. White, blister-like pustules, somewhat raised and initially smooth, are scattered on the underside of the leaves. Corresponding yellow spots may be visible on the upper leaf surface. Affected flower parts can become swollen and distorted.",
+    "Young leaves and shoot tips of an apple sapling are shown. They are covered with a characteristic white to grayish powdery fungal growth. Affected leaves may appear distorted, curled, or stunted.",
+    "Image displays the upper surface of grape leaves showing distinct, angular, yellowish-green spots (\"oil spots\"). On the underside of the leaf, corresponding to these spots, a white, downy fungal growth is visible, especially under humid conditions.",
+    "Close-up on the stem and leaf petioles of a papaya plant, especially near the growing point. Clusters of white, cottony masses are visible, covering small, oval, soft-bodied insects (mealybugs). Ants may also be present, attracted to the honeydew secreted by the mealybugs.",
+    "Several mature banana leaves are shown. They exhibit numerous small, elliptical spots parallel to the leaf veins. Initially yellowish-green, the spots enlarge, become dark brown or black, and often develop a grayish center surrounded by a dark border and a yellow halo. Severe spotting leads to leaf necrosis."
+    # === END OF DESCRIPTION LIST ===
+]
+
+OUTPUT_FILENAME = "agri_synthetic_data_generated.jsonl"
+DESCRIPTIONS_PER_BATCH = 5
+# Consider slightly higher temperature for variation across runs, but check quality
+GENERATION_TEMPERATURE = 0.5
+# Adjust sleep time based on API rate limits (seconds)
+SLEEP_TIME_PER_API_CALL = 20
+
+# --- Prompt Templates (Remain the same as previous script) ---
+# template_detailed, template_short, template_easy...
+template_detailed = """
+You are an AI assistant generating synthetic fine-tuning data for an agricultural LLM (Gemma-3). You act as a **senior agricultural scientist**. You will be given {num_descriptions} distinct text descriptions simulating field observations. For **EACH** description, you must generate a corresponding **detailed and technically accurate** diagnosis and management strategy, formatted **exactly** as specified.
+
+**INPUT DESCRIPTIONS:**
+{input_descriptions_formatted}
+
+**TASK:**
+For **EACH** of the {num_descriptions} Input Descriptions provided above, generate a **single output string** strictly following the Gemma-3 format. The response part within each string should contain your detailed, technical diagnosis and recommendations based *only* on the corresponding Input Description.
+
+**OUTPUT FORMAT (Generate one block containing all {num_descriptions} strings consecutively, separated by '---'):**
+{output_format_structure}
+
+**Formatting & Constraints:**
+*   Generate **one single continuous block of text** containing all {num_descriptions} formatted `OUTPUT_STRING` sections.
+*   Use `\\n---\\n` as a separator between each formatted string output.
+*   Each of the {num_descriptions} output strings MUST individually adhere strictly to the `<bos>...<end_of_turn>` structure.
+*   Ensure the correct description is pasted inside the user turn for each corresponding output string.
+*   Base diagnosis/recommendations ONLY on the corresponding Input Description.
+*   Responses should be thorough and use appropriate technical terms.
+*   Do NOT add any extra text outside this structure.
+--- Produce only the final formatted block based on the Input Descriptions. ---
+"""
+template_short = """
+You are an AI assistant generating synthetic fine-tuning data for an agricultural LLM (Gemma-3). You act as a **quick diagnostic assistant**. You will be given {num_descriptions} distinct text descriptions simulating field observations. For **EACH** description, you must generate a corresponding **very brief identification** and key point, formatted **exactly** as specified.
+
+**INPUT DESCRIPTIONS:**
+{input_descriptions_formatted}
+
+**TASK:**
+For **EACH** of the {num_descriptions} Input Descriptions provided above, generate a **single output string** strictly following the Gemma-3 format. The response part within each string should contain your extremely concise identification and key point/action (1-3 sentences max) based *only* on the corresponding Input Description.
+
+**OUTPUT FORMAT (Generate one block containing all {num_descriptions} strings consecutively, separated by '---'):**
+{output_format_structure}
+
+**Formatting & Constraints:**
+*   Generate **one single continuous block of text** containing all {num_descriptions} formatted `OUTPUT_STRING` sections.
+*   Use `\\n---\\n` as a separator between each formatted string output.
+*   Each of the {num_descriptions} output strings MUST individually adhere strictly to the `<bos>...<end_of_turn>` structure.
+*   Ensure the correct description is pasted inside the user turn for each corresponding output string.
+*   Base identification ONLY on the corresponding Input Description.
+*   Responses should be extremely concise.
+*   Do NOT add any extra text outside this structure.
+--- Produce only the final formatted block based on the Input Descriptions. ---
+"""
+template_easy = """
+You are an AI assistant generating synthetic fine-tuning data for an agricultural LLM (Gemma-3). You act as a **friendly local agricultural advisor**. You will be given {num_descriptions} distinct text descriptions simulating field observations. For **EACH** description, you must generate a corresponding explanation and recommendation in **simple, easy-to-understand language**, formatted **exactly** as specified.
+
+**INPUT DESCRIPTIONS:**
+{input_descriptions_formatted}
+
+**TASK:**
+For **EACH** of the {num_descriptions} Input Descriptions provided above, generate a **single output string** strictly following the Gemma-3 format. The response part within each string should explain the likely problem and suggest first steps in **simple, non-technical language** suitable for an Indian farmer, based *only* on the corresponding Input Description.
+
+**OUTPUT FORMAT (Generate one block containing all {num_descriptions} strings consecutively, separated by '---'):**
+{output_format_structure}
+
+**Formatting & Constraints:**
+*   Generate **one single continuous block of text** containing all {num_descriptions} formatted `OUTPUT_STRING` sections.
+*   Use `\\n---\\n` as a separator between each formatted string output.
+*   Each of the {num_descriptions} output strings MUST individually adhere strictly to the `<bos>...<end_of_turn>` structure.
+*   Ensure the correct description is pasted inside the user turn for each corresponding output string.
+*   Base explanation ONLY on the corresponding Input Description.
+*   Use simple phrasing, avoid jargon.
+*   Do NOT add any extra text outside this structure.
+--- Produce only the final formatted block based the Input Descriptions. ---
+"""
+templates = { "detailed": template_detailed, "short": template_short, "easy": template_easy }
+user_prompts_for_template = {
+     "detailed": "Analyze the provided description of plant symptoms and provide a detailed technical diagnosis and recommendations.",
+     "short": "Briefly identify the problem shown in the description and give the main point.",
+     "easy": "Explain what might be wrong based on this description, using simple words, and what I should do first."
+}
+
+# --- Helper Function (Remains the same) ---
+# format_prompt_for_batch function... (kept concise for brevity, same as before)
+
+# --- Load Existing Data ---
+existing_texts = set()
+if os.path.exists(OUTPUT_FILENAME):
+    print(f"Loading existing entries from {OUTPUT_FILENAME} to avoid duplicates...")
+    try:
+        with open(OUTPUT_FILENAME, 'r', encoding='utf-8') as infile:
+            for line in infile:
+                try:
+                    data = json.loads(line)
+                    if 'text' in data:
+                        existing_texts.add(data['text'])
+                except (json.JSONDecodeError, KeyError) as e:
+                    print(f"  Skipping invalid line: {line.strip()} - Error: {e}")
+        print(f"Loaded {len(existing_texts)} unique existing entries.")
+    except Exception as e:
+        print(f"Error loading existing file: {e}. Starting fresh.")
+else:
+    print(f"Output file {OUTPUT_FILENAME} not found. Starting fresh.")
+
+# --- Main Generation Logic ---
+genai.configure(api_key=API_KEY)
+model = genai.GenerativeModel(MODEL_NAME)
+
+generation_config = genai.GenerationConfig(
+    temperature=GENERATION_TEMPERATURE,
+    # Add other parameters if needed
+)
+safety_settings = [ # Adjust as needed
+    {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_MEDIUM_AND_ABOVE"},
+    {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_MEDIUM_AND_ABOVE"},
+    {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_MEDIUM_AND_ABOVE"},
+    {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_MEDIUM_AND_ABOVE"},
+]
+
+total_entries_generated_this_run = 0
+num_duplicates_skipped_this_run = 0
+num_batches = math.ceil(len(input_descriptions) / DESCRIPTIONS_PER_BATCH)
+
+print(f"\nStarting data generation for {len(input_descriptions)} descriptions...")
+print(f"Will process in {num_batches} batches of up to {DESCRIPTIONS_PER_BATCH}.")
+print(f"Output will be appended to: {OUTPUT_FILENAME}")
+print(f"Using model: {MODEL_NAME}, Temperature: {GENERATION_TEMPERATURE}")
+print(f"Pausing {SLEEP_TIME_PER_API_CALL} seconds between API calls.")
+
+try:
+    # Open file in append mode
+    with open(OUTPUT_FILENAME, 'a', encoding='utf-8') as f:
+        for i in range(num_batches):
+            start_index = i * DESCRIPTIONS_PER_BATCH
+            end_index = start_index + DESCRIPTIONS_PER_BATCH
+            batch_descriptions = input_descriptions[start_index:end_index]
+            actual_batch_size = len(batch_descriptions)
+
+            print(f"\n--- Processing Batch {i+1}/{num_batches} ({actual_batch_size} descriptions) ---")
+
+            if not batch_descriptions:
+                print("Skipping empty batch.")
+                continue
+
+            for template_key in templates.keys():
+                print(f"  Generating '{template_key}' style responses...")
+
+                # --- Format the prompt (using the same logic as previous script) ---
+                formatted_prompt = ""
+                input_desc_block = ""
+                output_struct_block = ""
+                user_prompt = user_prompts_for_template[template_key]
+                for idx, desc in enumerate(batch_descriptions, 1):
+                     input_desc_block += f"INPUT_DESCRIPTION_{idx}:\n{desc}\n\n"
+                     pasted_desc_safe = desc.replace('`', r'\`').replace('{', '{{').replace('}', '}}')
+                     output_struct_block += f"OUTPUT_STRING_{idx}:\n"
+                     output_struct_block += f"<bos><start_of_turn>user\n{user_prompt}\n{pasted_desc_safe}<end_of_turn>\n<start_of_turn>model\n[MODEL'S RESPONSE FOR DESCRIPTION {idx} HERE]<end_of_turn>\n"
+                     if idx < actual_batch_size:
+                         output_struct_block += "---\n"
+                formatted_prompt = templates[template_key].format(
+                    num_descriptions=actual_batch_size,
+                    input_descriptions_formatted=input_desc_block.strip(),
+                    output_format_structure=output_struct_block.strip()
+                 )
+
+                # --- Call the API (with retries) ---
+                # (API call logic with retries remains the same as previous script)
+                api_call_successful = False
+                retry_count = 0
+                max_retries = 2
+                response_text = None # Initialize response_text
+                while not api_call_successful and retry_count <= max_retries:
+                    try:
+                        print(f"    Sending request to Gemini (Attempt {retry_count + 1})...")
+                        response = model.generate_content(
+                            formatted_prompt,
+                            generation_config=generation_config,
+                            safety_settings=safety_settings
+                        )
+                        # Check for empty response or potential errors in the response object itself
+                        if response.parts:
+                             response_text = response.text
+                        else:
+                             # Handle cases where the response might be blocked or empty
+                             print("    WARNING: Received response with no processable parts. Checking prompt feedback.")
+                             if hasattr(response, 'prompt_feedback') and response.prompt_feedback:
+                                 print(f"    Prompt Feedback: {response.prompt_feedback}")
+                             response_text = None # Treat as failure for processing
+                             # Decide if this is a retryable error or skip
+                             if response.prompt_feedback and response.prompt_feedback.block_reason:
+                                  print("      Response blocked, skipping.")
+                                  break # Exit retry loop for blocked content
+                             else:
+                                 print("      Empty response, retrying...")
+
+
+                        # If response_text is valid, mark successful
+                        if response_text:
+                            api_call_successful = True
+                        elif not api_call_successful and retry_count < max_retries: # Only retry if not blocked and retries left
+                            raise ValueError("Empty or non-text response part received") # Trigger retry
+                        elif not api_call_successful: # Max retries reached or blocked
+                             break # Exit retry loop
+
+                    except Exception as e:
+                        retry_count += 1
+                        print(f"    ERROR during API call (Attempt {retry_count}): {e}")
+                        if retry_count > max_retries:
+                            print("    Max retries reached. Skipping this batch/template.")
+                            response_text = None
+                            break
+                        print(f"    Retrying in {SLEEP_TIME_PER_API_CALL} seconds...")
+                        time.sleep(SLEEP_TIME_PER_API_CALL)
+
+                # --- Process and Save Response (with duplicate check) ---
+                if api_call_successful and response_text:
+                    split_parts = re.split(r'\n---\n', response_text.strip())
+                    generated_count_in_batch = 0
+                    skipped_count_in_batch = 0
+
+                    if len(split_parts) == actual_batch_size:
+                        for entry_text in split_parts:
+                            cleaned_text = entry_text.strip()
+                            if cleaned_text.startswith("OUTPUT_STRING_"):
+                                cleaned_text = cleaned_text.split('\n', 1)[-1].strip()
+
+                            if cleaned_text.startswith("<bos>") and cleaned_text.endswith("<end_of_turn>"):
+                                # *** Check for duplicates ***
+                                if cleaned_text not in existing_texts:
+                                    data_entry = {"text": cleaned_text}
+                                    json.dump(data_entry, f, ensure_ascii=False)
+                                    f.write('\n')
+                                    existing_texts.add(cleaned_text) # Add to set *after* writing
+                                    total_entries_generated_this_run += 1
+                                    generated_count_in_batch += 1
+                                else:
+                                    num_duplicates_skipped_this_run += 1
+                                    skipped_count_in_batch += 1
+                                    # print(f"    Skipped duplicate entry.") # Optional: uncomment for verbose logging
+                            else:
+                                print(f"    WARNING: Invalid format detected in generated entry part:\n{cleaned_text[:100]}...")
+                        print(f"    Processed batch for '{template_key}': Saved {generated_count_in_batch} new entries, Skipped {skipped_count_in_batch} duplicates.")
+                    else:
+                         print(f"    WARNING: Expected {actual_batch_size} parts but got {len(split_parts)} for '{template_key}'. Skipping save for this malformed batch response.")
+                         print(f"    Raw response snippet:\n{response_text[:500]}...")
+
+                elif api_call_successful and not response_text:
+                     print("    WARNING: API call successful but received empty/blocked response, no entries saved.")
+                # else: (Handled by retry loop failure message)
+                #    pass
+
+                # --- Rate Limiting Pause ---
+                print(f"    Pausing for {SLEEP_TIME_PER_API_CALL} seconds...")
+                time.sleep(SLEEP_TIME_PER_API_CALL)
+
+except Exception as e:
+    print(f"\nAn unexpected error occurred outside the main loop: {e}")
+    import traceback
+    traceback.print_exc() # Print full traceback for debugging
+
+finally:
+    print(f"\n--- Run Finished ---")
+    print(f"Entries generated in THIS run: {total_entries_generated_this_run}")
+    print(f"Duplicates skipped in THIS run: {num_duplicates_skipped_this_run}")
+    print(f"Total unique entries now in {OUTPUT_FILENAME}: {len(existing_texts)}") # Reflects total after run completion

generate_synthetic_data_lmstudio.py

@@ -0,0 +1,275 @@
+import requests
+import os
+import json
+import time
+import re
+import sys
+import math
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# --- Configuration ---
+# Set LM Studio API URL - default is local server
+LMSTUDIO_API_URL = os.environ.get("LMSTUDIO_API_URL", "http://localhost:1234/v1/chat/completions")
+
+# Choose a model loaded in your LM Studio instance
+MODEL_NAME = "default"  # LM Studio uses loaded model by default
+
+# Your list of detailed input descriptions
+input_descriptions = [
+    "Rice - Blast: The image shows multiple leaves of a rice plant during the tillering stage. Many leaves exhibit diamond-shaped lesions, approximately 1-2 cm long, with grayish-white centers and distinct dark brown borders. Some lesions are merging, causing larger blighted areas. No insects are visible.",
+    "Cotton - Bollworm: Close-up view of a developing cotton boll. Several small (<1cm), pinkish-white larvae are seen boring into the boll surface near the bracts. Small entry holes with some dark, granular frass (excrement) are visible. The surrounding leaves appear relatively healthy.",
+    "Maize - Nitrogen Deficiency: Image displays the lower leaves of a young maize plant. The leaves show distinct yellowing starting at the leaf tip and progressing down the midrib in a characteristic V-shape pattern. The leaf margins remain relatively green initially. Upper leaves appear greener.",
+    "Tomato - Early Blight: Several lower leaves of a mature tomato plant are shown. They exhibit circular to irregular dark brown lesions, 0.5-1.5 cm in diameter. Many lesions display characteristic concentric rings, giving a 'target board' appearance. Significant yellowing (chlorosis) surrounds the lesions, and some affected leaves are starting to wither.",
+    "Wheat - Stripe Rust: Picture shows the upper leaves of a wheat plant nearing the flag leaf stage. Numerous small, bright yellow-orange pustules are arranged in distinct, long stripes running parallel to the leaf veins. When touched, a yellowish powder (spores) rubs off.",
+    "Chili - Leaf Curl Virus: The image focuses on the growing tip of a chili plant. The young leaves are severely deformed, showing upward curling, puckering, and twisting. The leaves appear thickened and brittle, and veins are sometimes noticeably thicker. Plant growth appears stunted overall. Tiny whitefly insects may be subtly present but are not the primary focus.",
+    "Groundnut - Tikka Disease (Leaf Spot): Several leaves of a groundnut plant are displayed. They are covered with numerous circular, dark brown to black spots, about 2-8 mm wide. Each spot is surrounded by a distinct yellow halo. Some older spots on lower leaves might lack the clear halo.",
+    "Soybean - Healthy: Image shows a section of a healthy, vigorously growing soybean plant canopy during the vegetative stage. The leaves are uniformly deep green, fully expanded, and show no signs of spotting, yellowing, mosaic patterns, insect feeding, or wilting. Stems appear sturdy.",
+    "Mango - Anthracnose on Fruit: A near-ripe mango fruit is shown. It displays several sunken, irregular-shaped black spots on the peel. Some spots appear to be coalescing into larger dark lesions. In humid conditions, pinkish spore masses might be visible in the center of older spots (though not clearly visible here).",
+    "Sugarcane - Early Shoot Borer Damage: Focus is on the base of young sugarcane tillers (shoots). The central whorl of leaves is dried up, creating a characteristic \"dead heart\" symptom, which can be easily pulled out. Small bore holes may be visible near the base of the affected shoot.",
+    "Potato - Late Blight: Image shows potato leaves with large, irregular, water-soaked lesions, often starting at the leaf margins or tips. The lesions rapidly turn dark brown to black. A fuzzy white fungal growth (sporangiophores) might be visible on the underside of the leaves near the edge of the lesions, especially in moist conditions.",
+    "Brinjal (Eggplant) - Aphid Infestation: Close-up shows the underside of several young brinjal leaves and stems. They are heavily infested with clusters of small, pear-shaped insects, mostly greenish or blackish in color. Some shiny, sticky honeydew secretions might be visible on the leaf surfaces below the infestation.",
+    "Okra (Lady's Finger) - Yellow Vein Mosaic Virus: The image shows several leaves of an okra plant. The veins of the leaves are prominently yellow, and the yellow network extends into the surrounding green leaf tissue, creating a distinct mosaic pattern. The leaves might be slightly reduced in size.",
+    "Cabbage - Diamondback Moth Larvae: Several outer leaves of a cabbage head are shown. Small (around 1 cm), greenish larvae with a slightly tapered body are visible, actively feeding on the leaf tissue. They have created irregular holes or \"window panes\" (where only one layer of the leaf epidermis is left). Some silken threads might be present.",
+    "Pigeon Pea (Tur/Arhar) - Pod Borer Damage: Image shows several developing pigeon pea pods. Some pods have distinct circular bore holes on the surface. One pod is broken open slightly, revealing a visible greenish or brownish larva feeding inside on the developing seeds. Frass might be present near the holes.",
+    "Mustard - White Rust: Focus is on mustard leaves and potentially flower stalks. White, blister-like pustules, somewhat raised and initially smooth, are scattered on the underside of the leaves. Corresponding yellow spots may be visible on the upper leaf surface. Affected flower parts can become swollen and distorted.",
+    "Apple - Powdery Mildew: Young leaves and shoot tips of an apple sapling are shown. They are covered with a characteristic white to grayish powdery fungal growth. Affected leaves may appear distorted, curled, or stunted.",
+    "Grapes - Downy Mildew: Image displays the upper surface of grape leaves showing distinct, angular, yellowish-green spots (\"oil spots\"). On the underside of the leaf, corresponding to these spots, a white, downy fungal growth is visible, especially under humid conditions.",
+    "Papaya - Mealybug Infestation: Close-up on the stem and leaf petioles of a papaya plant, especially near the growing point. Clusters of white, cottony masses are visible, covering small, oval, soft-bodied insects (mealybugs). Ants may also be present, attracted to the honeydew secreted by the mealybugs.",
+    "Banana - Sigatoka Leaf Spot: Several mature banana leaves are shown. They exhibit numerous small, elliptical spots parallel to the leaf veins. Initially yellowish-green, the spots enlarge, become dark brown or black, and often develop a grayish center surrounded by a dark border and a yellow halo. Severe spotting leads to leaf necrosis."
+]
+
+OUTPUT_FILENAME = "agri_synthetic_data_generated_lmstudio.jsonl"
+GENERATION_TEMPERATURE = 0.7
+SLEEP_TIME_PER_API_CALL = 1  # LM Studio is local, so we can typically be a bit faster
+
+# --- Prompt Template ---
+single_prompt_template = """
+You are an AI assistant generating fine-tuning data for an agricultural diagnosis chatbot.
+Your task is to generate a response in a specific format based on the provided plant symptom description and the user's implied question style.
+
+Follow these instructions STRICTLY:
+
+1.  Analyze the User Question Style: Understand the level of detail requested by the user prompt ({user_question_style}).
+2.  Analyze the Input Description: Carefully read the provided plant symptom description.
+3.  Generate the Response: Create a helpful and accurate response that directly addresses the user's question style, based *only* on the information in the input description.
+4.  Format the Output EXACTLY as specified below and make sure to include ALL the tags, especially the final <end_of_turn> tag:
+
+<bos><start_of_turn>user
+{user_question}
+{input_description}<end_of_turn>
+<start_of_turn>model
+[YOUR GENERATED RESPONSE HERE BASED ON THE DESCRIPTION AND QUESTION STYLE]<end_of_turn>
+
+IMPORTANT: Make sure your response includes all required tags, especially ending with <end_of_turn>
+
+--- Input Data ---
+
+User Question Style: {user_question_style}
+User Question: {user_question}
+Input Description:
+{input_description}
+
+--- Generate the Output Below ---
+"""
+
+# Define the different user question styles
+user_prompts_for_style = {
+     "detailed": "Analyze the provided description of plant symptoms and provide a detailed technical diagnosis and recommendations.",
+     "short": "Briefly identify the problem shown in the description and give the main point.",
+     "easy": "Explain what might be wrong based on this description, using simple words, and what I should do first."
+}
+
+# --- Load Existing Data ---
+existing_texts = set()
+if os.path.exists(OUTPUT_FILENAME):
+    print(f"Loading existing entries from {OUTPUT_FILENAME} to avoid duplicates...")
+    try:
+        with open(OUTPUT_FILENAME, 'r', encoding='utf-8') as infile:
+            for line in infile:
+                try:
+                    data = json.loads(line)
+                    if 'text' in data:
+                        existing_texts.add(data['text'])
+                except (json.JSONDecodeError, KeyError) as e:
+                    print(f"  Skipping invalid line: {line.strip()} - Error: {e}")
+        print(f"Loaded {len(existing_texts)} unique existing entries.")
+    except Exception as e:
+        print(f"Error loading existing file: {e}. Starting fresh.")
+else:
+    print(f"Output file {OUTPUT_FILENAME} not found. Starting fresh.")
+
+# --- Main Generation Logic ---
+total_entries_generated_this_run = 0
+num_duplicates_skipped_this_run = 0
+total_api_calls = len(input_descriptions) * len(user_prompts_for_style)
+current_api_call = 0
+
+print(f"\nStarting data generation for {len(input_descriptions)} descriptions...")
+print(f"Will make {total_api_calls} individual API calls via LM Studio.")
+print(f"Output will be appended to: {OUTPUT_FILENAME}")
+print(f"Using LM Studio model: {MODEL_NAME}, Temperature: {GENERATION_TEMPERATURE}")
+print(f"Pausing {SLEEP_TIME_PER_API_CALL} seconds between API calls.")
+
+try:
+    with open(OUTPUT_FILENAME, 'a', encoding='utf-8') as f:
+        # Iterate through each description
+        for i, description in enumerate(input_descriptions):
+            print(f"\n--- Processing Description {i+1}/{len(input_descriptions)} ---")
+            print(f"Input: {description[:100]}...") # Print snippet
+
+            # Iterate through each prompt style for the current description
+            for style_key, user_question in user_prompts_for_style.items():
+                current_api_call += 1
+                print(f"  Generating '{style_key}' style response ({current_api_call}/{total_api_calls})...")
+
+                # Format the prompt
+                pasted_desc_safe = description.replace('{', '{{').replace('}', '}}')
+                formatted_prompt = single_prompt_template.format(
+                    user_question_style=style_key,
+                    user_question=user_question,
+                    input_description=pasted_desc_safe
+                )
+
+                # Prepare payload for LM Studio API
+                payload = {
+                    "messages": [
+                        {
+                            "role": "user",
+                            "content": formatted_prompt
+                        }
+                    ],
+                    "temperature": GENERATION_TEMPERATURE,
+                    "max_tokens": 2048,
+                    "stream": False
+                }
+
+                # Call the LM Studio API (with retries)
+                api_call_successful = False
+                retry_count = 0
+                max_retries = 3
+                response_text = None
+                
+                while not api_call_successful and retry_count <= max_retries:
+                    try:
+                        print(f"    Sending request to LM Studio (Attempt {retry_count + 1})...")
+                        start_time = time.time()
+
+                        # Make the POST request to LM Studio
+                        response = requests.post(
+                            LMSTUDIO_API_URL,
+                            json=payload,
+                            headers={"Content-Type": "application/json"}
+                        )
+
+                        end_time = time.time()
+                        print(f"    LM Studio API call took {end_time - start_time:.2f} seconds.")
+
+                        # Check for HTTP errors
+                        response.raise_for_status()
+
+                        # Parse the JSON response
+                        response_data = response.json()
+
+                        # Extract response content (LM Studio format)
+                        if response_data.get('choices') and len(response_data['choices']) > 0:
+                            response_text = response_data['choices'][0]['message']['content']
+                            print(f"    DEBUG - Last 50 chars of response: {response_text[-50:] if response_text else 'None'}")
+
+                            # Check if response looks like expected format
+                            if response_text and "<bos>" in response_text:
+                                api_call_successful = True
+                            else:
+                                print("    WARNING: Received response doesn't contain '<bos>'. Might be malformed.")
+                                print(f"    Raw Response Snippet: {response_text[:200]}...")
+                                # Extract what looks like the format we want
+                                if "<start_of_turn>" in response_text:
+                                    # Try to extract just the formatted part
+                                    pattern = r"<bos>.*?<end_of_turn>"
+                                    match = re.search(pattern, response_text, re.DOTALL)
+                                    if match:
+                                        response_text = match.group(0)
+                                        api_call_successful = True
+                                        print("    Found partial match in response, extracted usable content.")
+                                    else:
+                                        raise ValueError("Response format incorrect")
+                                else:
+                                    raise ValueError("Response format incorrect")
+                        else:
+                            print("    WARNING: Received unexpected response structure from LM Studio.")
+                            print(f"    Raw response data: {response_data}")
+                            response_text = None
+                            raise ValueError("Invalid LM Studio response structure")
+
+                    except requests.exceptions.RequestException as e:
+                        retry_count += 1
+                        print(f"    ERROR during LM Studio API call (Attempt {retry_count}): {e}")
+                        
+                        wait_time = SLEEP_TIME_PER_API_CALL * (2 ** (retry_count - 1))
+
+                        if retry_count > max_retries:
+                            print(f"    Max retries reached for Description {i+1}, Style '{style_key}'. Skipping.")
+                            response_text = None
+                            break
+                        print(f"    Retrying in {wait_time:.2f} seconds...")
+                        time.sleep(wait_time)
+                    except Exception as e:
+                        retry_count += 1
+                        print(f"    ERROR processing request/response (Attempt {retry_count}): {e}")
+                        wait_time = SLEEP_TIME_PER_API_CALL * (2 ** (retry_count - 1))
+                        if retry_count > max_retries:
+                            print(f"    Max retries reached for Description {i+1}, Style '{style_key}'. Skipping.")
+                            response_text = None
+                            break
+                        print(f"    Retrying in {wait_time:.2f} seconds...")
+                        time.sleep(wait_time)
+
+                # Process and Save Response
+                if api_call_successful and response_text:
+                    cleaned_text = response_text.strip()
+                    print(f"    DEBUG - Response starts with: {cleaned_text[:50]}...")
+                    print(f"    DEBUG - Response ends with: ...{cleaned_text[-50:]}")
+
+                    # Ensure response has both starting and ending tags
+                    if "<bos>" in cleaned_text:
+                        # Check for end tag and add if missing
+                        if "<end_of_turn>" not in cleaned_text:
+                            print("    WARNING: Response missing <end_of_turn> tag. Adding it for consistency.")
+                            cleaned_text = cleaned_text + "<end_of_turn>"
+                        
+                        # Check for duplicates
+                        if cleaned_text not in existing_texts:
+                            data_entry = {"text": cleaned_text}
+                            json.dump(data_entry, f, ensure_ascii=False)
+                            f.write('\n')
+                            existing_texts.add(cleaned_text)
+                            total_entries_generated_this_run += 1
+                            print(f"    Saved 1 new entry for '{style_key}'.")
+                        else:
+                            num_duplicates_skipped_this_run += 1
+                            print(f"    Skipped 1 duplicate entry for '{style_key}'.")
+                    else:
+                        print(f"    WARNING: Invalid format detected in final generated entry for '{style_key}'.")
+                        print(f"    Failed validation. Raw text: {cleaned_text[:150]}...")
+
+                elif not api_call_successful:
+                    print(f"    Skipping save for '{style_key}' due to API call failures after retries.")
+
+                # Rate Limiting Pause (after each API call)
+                if current_api_call < total_api_calls:  # Don't sleep after the very last call
+                    time.sleep(SLEEP_TIME_PER_API_CALL)
+
+except KeyboardInterrupt:
+    print("\n--- KeyboardInterrupt detected. Stopping script early. ---")
+except Exception as e:
+    print(f"\nAn unexpected error occurred outside the main loop: {e}")
+    import traceback
+    traceback.print_exc()
+
+finally:
+    print(f"\n--- Run Finished ---")
+    print(f"Total API calls attempted: {current_api_call}")
+    print(f"Entries generated in THIS run: {total_entries_generated_this_run}")
+    print(f"Duplicates skipped in THIS run: {num_duplicates_skipped_this_run}")
+    print(f"Total unique entries now in {OUTPUT_FILENAME}: {len(existing_texts)}")

generate_synthetic_data_ollama.py

@@ -0,0 +1,272 @@
+import requests
+import os
+import json
+import time
+import re
+import sys
+import math
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# --- Configuration ---
+# Set Ollama API URL - default is local server
+OLLAMA_API_URL = os.environ.get("OLLAMA_API_URL", "hhttp://192.168.38.87:1234/v1/completions")
+
+# Choose a model available in your Ollama instance
+# Examples: "llama3", "mistral", "gemma:7b", etc.
+MODEL_NAME = "lmstudio-community/gemma-3-4b-it-GGUF"
+
+# Your list of detailed input descriptions
+input_descriptions = [
+    "Rice - Blast: The image shows multiple leaves of a rice plant during the tillering stage. Many leaves exhibit diamond-shaped lesions, approximately 1-2 cm long, with grayish-white centers and distinct dark brown borders. Some lesions are merging, causing larger blighted areas. No insects are visible.",
+    "Cotton - Bollworm: Close-up view of a developing cotton boll. Several small (<1cm), pinkish-white larvae are seen boring into the boll surface near the bracts. Small entry holes with some dark, granular frass (excrement) are visible. The surrounding leaves appear relatively healthy.",
+    "Maize - Nitrogen Deficiency: Image displays the lower leaves of a young maize plant. The leaves show distinct yellowing starting at the leaf tip and progressing down the midrib in a characteristic V-shape pattern. The leaf margins remain relatively green initially. Upper leaves appear greener.",
+    "Tomato - Early Blight: Several lower leaves of a mature tomato plant are shown. They exhibit circular to irregular dark brown lesions, 0.5-1.5 cm in diameter. Many lesions display characteristic concentric rings, giving a 'target board' appearance. Significant yellowing (chlorosis) surrounds the lesions, and some affected leaves are starting to wither.",
+    "Wheat - Stripe Rust: Picture shows the upper leaves of a wheat plant nearing the flag leaf stage. Numerous small, bright yellow-orange pustules are arranged in distinct, long stripes running parallel to the leaf veins. When touched, a yellowish powder (spores) rubs off.",
+    "Chili - Leaf Curl Virus: The image focuses on the growing tip of a chili plant. The young leaves are severely deformed, showing upward curling, puckering, and twisting. The leaves appear thickened and brittle, and veins are sometimes noticeably thicker. Plant growth appears stunted overall. Tiny whitefly insects may be subtly present but are not the primary focus.",
+    "Groundnut - Tikka Disease (Leaf Spot): Several leaves of a groundnut plant are displayed. They are covered with numerous circular, dark brown to black spots, about 2-8 mm wide. Each spot is surrounded by a distinct yellow halo. Some older spots on lower leaves might lack the clear halo.",
+    "Soybean - Healthy: Image shows a section of a healthy, vigorously growing soybean plant canopy during the vegetative stage. The leaves are uniformly deep green, fully expanded, and show no signs of spotting, yellowing, mosaic patterns, insect feeding, or wilting. Stems appear sturdy.",
+    "Mango - Anthracnose on Fruit: A near-ripe mango fruit is shown. It displays several sunken, irregular-shaped black spots on the peel. Some spots appear to be coalescing into larger dark lesions. In humid conditions, pinkish spore masses might be visible in the center of older spots (though not clearly visible here).",
+    "Sugarcane - Early Shoot Borer Damage: Focus is on the base of young sugarcane tillers (shoots). The central whorl of leaves is dried up, creating a characteristic \"dead heart\" symptom, which can be easily pulled out. Small bore holes may be visible near the base of the affected shoot.",
+    "Potato - Late Blight: Image shows potato leaves with large, irregular, water-soaked lesions, often starting at the leaf margins or tips. The lesions rapidly turn dark brown to black. A fuzzy white fungal growth (sporangiophores) might be visible on the underside of the leaves near the edge of the lesions, especially in moist conditions.",
+    "Brinjal (Eggplant) - Aphid Infestation: Close-up shows the underside of several young brinjal leaves and stems. They are heavily infested with clusters of small, pear-shaped insects, mostly greenish or blackish in color. Some shiny, sticky honeydew secretions might be visible on the leaf surfaces below the infestation.",
+    "Okra (Lady's Finger) - Yellow Vein Mosaic Virus: The image shows several leaves of an okra plant. The veins of the leaves are prominently yellow, and the yellow network extends into the surrounding green leaf tissue, creating a distinct mosaic pattern. The leaves might be slightly reduced in size.",
+    "Cabbage - Diamondback Moth Larvae: Several outer leaves of a cabbage head are shown. Small (around 1 cm), greenish larvae with a slightly tapered body are visible, actively feeding on the leaf tissue. They have created irregular holes or \"window panes\" (where only one layer of the leaf epidermis is left). Some silken threads might be present.",
+    "Pigeon Pea (Tur/Arhar) - Pod Borer Damage: Image shows several developing pigeon pea pods. Some pods have distinct circular bore holes on the surface. One pod is broken open slightly, revealing a visible greenish or brownish larva feeding inside on the developing seeds. Frass might be present near the holes.",
+    "Mustard - White Rust: Focus is on mustard leaves and potentially flower stalks. White, blister-like pustules, somewhat raised and initially smooth, are scattered on the underside of the leaves. Corresponding yellow spots may be visible on the upper leaf surface. Affected flower parts can become swollen and distorted.",
+    "Apple - Powdery Mildew: Young leaves and shoot tips of an apple sapling are shown. They are covered with a characteristic white to grayish powdery fungal growth. Affected leaves may appear distorted, curled, or stunted.",
+    "Grapes - Downy Mildew: Image displays the upper surface of grape leaves showing distinct, angular, yellowish-green spots (\"oil spots\"). On the underside of the leaf, corresponding to these spots, a white, downy fungal growth is visible, especially under humid conditions.",
+    "Papaya - Mealybug Infestation: Close-up on the stem and leaf petioles of a papaya plant, especially near the growing point. Clusters of white, cottony masses are visible, covering small, oval, soft-bodied insects (mealybugs). Ants may also be present, attracted to the honeydew secreted by the mealybugs.",
+    "Banana - Sigatoka Leaf Spot: Several mature banana leaves are shown. They exhibit numerous small, elliptical spots parallel to the leaf veins. Initially yellowish-green, the spots enlarge, become dark brown or black, and often develop a grayish center surrounded by a dark border and a yellow halo. Severe spotting leads to leaf necrosis."
+]
+
+OUTPUT_FILENAME = "agri_synthetic_data_generated_ollama.jsonl"
+GENERATION_TEMPERATURE = 0.7
+SLEEP_TIME_PER_API_CALL = 1  # Ollama is local, so we can typically be a bit faster
+
+# --- Prompt Template ---
+single_prompt_template = """
+You are an AI assistant generating fine-tuning data for an agricultural diagnosis chatbot.
+Your task is to generate a response in a specific format based on the provided plant symptom description and the user's implied question style.
+
+Follow these instructions STRICTLY:
+
+1.  Analyze the User Question Style: Understand the level of detail requested by the user prompt ({user_question_style}).
+2.  Analyze the Input Description: Carefully read the provided plant symptom description.
+3.  Generate the Response: Create a helpful and accurate response that directly addresses the user's question style, based *only* on the information in the input description.
+4.  Format the Output EXACTLY as specified below and make sure to include ALL the tags, especially the final <end_of_turn> tag:
+
+<bos><start_of_turn>user
+{user_question}
+{input_description}<end_of_turn>
+<start_of_turn>model
+[YOUR GENERATED RESPONSE HERE BASED ON THE DESCRIPTION AND QUESTION STYLE]<end_of_turn>
+
+IMPORTANT: Make sure your response includes all required tags, especially ending with <end_of_turn>
+
+--- Input Data ---
+
+User Question Style: {user_question_style}
+User Question: {user_question}
+Input Description:
+{input_description}
+
+--- Generate the Output Below ---
+"""
+
+# Define the different user question styles
+user_prompts_for_style = {
+     "detailed": "Analyze the provided description of plant symptoms and provide a detailed technical diagnosis and recommendations.",
+     "short": "Briefly identify the problem shown in the description and give the main point.",
+     "easy": "Explain what might be wrong based on this description, using simple words, and what I should do first."
+}
+
+# --- Load Existing Data ---
+existing_texts = set()
+if os.path.exists(OUTPUT_FILENAME):
+    print(f"Loading existing entries from {OUTPUT_FILENAME} to avoid duplicates...")
+    try:
+        with open(OUTPUT_FILENAME, 'r', encoding='utf-8') as infile:
+            for line in infile:
+                try:
+                    data = json.loads(line)
+                    if 'text' in data:
+                        existing_texts.add(data['text'])
+                except (json.JSONDecodeError, KeyError) as e:
+                    print(f"  Skipping invalid line: {line.strip()} - Error: {e}")
+        print(f"Loaded {len(existing_texts)} unique existing entries.")
+    except Exception as e:
+        print(f"Error loading existing file: {e}. Starting fresh.")
+else:
+    print(f"Output file {OUTPUT_FILENAME} not found. Starting fresh.")
+
+# --- Main Generation Logic ---
+total_entries_generated_this_run = 0
+num_duplicates_skipped_this_run = 0
+total_api_calls = len(input_descriptions) * len(user_prompts_for_style)
+current_api_call = 0
+
+print(f"\nStarting data generation for {len(input_descriptions)} descriptions...")
+print(f"Will make {total_api_calls} individual API calls via Ollama.")
+print(f"Output will be appended to: {OUTPUT_FILENAME}")
+print(f"Using Ollama model: {MODEL_NAME}, Temperature: {GENERATION_TEMPERATURE}")
+print(f"Pausing {SLEEP_TIME_PER_API_CALL} seconds between API calls.")
+
+try:
+    with open(OUTPUT_FILENAME, 'a', encoding='utf-8') as f:
+        # Iterate through each description
+        for i, description in enumerate(input_descriptions):
+            print(f"\n--- Processing Description {i+1}/{len(input_descriptions)} ---")
+            print(f"Input: {description[:100]}...") # Print snippet
+
+            # Iterate through each prompt style for the current description
+            for style_key, user_question in user_prompts_for_style.items():
+                current_api_call += 1
+                print(f"  Generating '{style_key}' style response ({current_api_call}/{total_api_calls})...")
+
+                # Format the prompt
+                pasted_desc_safe = description.replace('{', '{{').replace('}', '}}')
+                formatted_prompt = single_prompt_template.format(
+                    user_question_style=style_key,
+                    user_question=user_question,
+                    input_description=pasted_desc_safe
+                )
+
+                # Prepare payload for Ollama API
+                payload = {
+                    "model": MODEL_NAME,
+                    "prompt": formatted_prompt,
+                    "temperature": GENERATION_TEMPERATURE,
+                    "num_predict": 2048,  # Max tokens to generate
+                    "stop": ["\n\n"]  # Optional stop sequence
+                }
+
+                # Call the Ollama API (with retries)
+                api_call_successful = False
+                retry_count = 0
+                max_retries = 3
+                response_text = None
+                
+                while not api_call_successful and retry_count <= max_retries:
+                    try:
+                        print(f"    Sending request to Ollama (Attempt {retry_count + 1})...")
+                        start_time = time.time()
+
+                        # Make the POST request to Ollama
+                        response = requests.post(
+                            OLLAMA_API_URL,
+                            json=payload,
+                            headers={"Content-Type": "application/json"}
+                        )
+
+                        end_time = time.time()
+                        print(f"    Ollama API call took {end_time - start_time:.2f} seconds.")
+
+                        # Check for HTTP errors
+                        response.raise_for_status()
+
+                        # Parse the JSON response
+                        response_data = response.json()
+
+                        # Extract response content (Ollama's format differs from OpenRouter)
+                        if response_data.get('response'):
+                            response_text = response_data['response']
+                            print(f"    DEBUG - Last 50 chars of response: {response_text[-50:] if response_text else 'None'}")
+
+                            # Check if response looks like expected format
+                            if response_text and "<bos>" in response_text:
+                                api_call_successful = True
+                            else:
+                                print("    WARNING: Received response doesn't contain '<bos>'. Might be malformed.")
+                                print(f"    Raw Response Snippet: {response_text[:200]}...")
+                                # Extract what looks like the format we want
+                                if "<start_of_turn>" in response_text:
+                                    # Try to extract just the formatted part
+                                    pattern = r"<bos>.*?<end_of_turn>"
+                                    match = re.search(pattern, response_text, re.DOTALL)
+                                    if match:
+                                        response_text = match.group(0)
+                                        api_call_successful = True
+                                        print("    Found partial match in response, extracted usable content.")
+                                    else:
+                                        raise ValueError("Response format incorrect")
+                                else:
+                                    raise ValueError("Response format incorrect")
+                        else:
+                            print("    WARNING: Received unexpected response structure from Ollama.")
+                            print(f"    Raw response data: {response_data}")
+                            response_text = None
+                            raise ValueError("Invalid Ollama response structure")
+
+                    except requests.exceptions.RequestException as e:
+                        retry_count += 1
+                        print(f"    ERROR during Ollama API call (Attempt {retry_count}): {e}")
+                        
+                        wait_time = SLEEP_TIME_PER_API_CALL * (2 ** (retry_count - 1))
+
+                        if retry_count > max_retries:
+                            print(f"    Max retries reached for Description {i+1}, Style '{style_key}'. Skipping.")
+                            response_text = None
+                            break
+                        print(f"    Retrying in {wait_time:.2f} seconds...")
+                        time.sleep(wait_time)
+                    except Exception as e:
+                        retry_count += 1
+                        print(f"    ERROR processing request/response (Attempt {retry_count}): {e}")
+                        wait_time = SLEEP_TIME_PER_API_CALL * (2 ** (retry_count - 1))
+                        if retry_count > max_retries:
+                            print(f"    Max retries reached for Description {i+1}, Style '{style_key}'. Skipping.")
+                            response_text = None
+                            break
+                        print(f"    Retrying in {wait_time:.2f} seconds...")
+                        time.sleep(wait_time)
+
+                # Process and Save Response
+                if api_call_successful and response_text:
+                    cleaned_text = response_text.strip()
+                    print(f"    DEBUG - Response starts with: {cleaned_text[:50]}...")
+                    print(f"    DEBUG - Response ends with: ...{cleaned_text[-50:]}")
+
+                    # Ensure response has both starting and ending tags
+                    if "<bos>" in cleaned_text:
+                        # Check for end tag and add if missing
+                        if "<end_of_turn>" not in cleaned_text:
+                            print("    WARNING: Response missing <end_of_turn> tag. Adding it for consistency.")
+                            cleaned_text = cleaned_text + "<end_of_turn>"
+                        
+                        # Check for duplicates
+                        if cleaned_text not in existing_texts:
+                            data_entry = {"text": cleaned_text}
+                            json.dump(data_entry, f, ensure_ascii=False)
+                            f.write('\n')
+                            existing_texts.add(cleaned_text)
+                            total_entries_generated_this_run += 1
+                            print(f"    Saved 1 new entry for '{style_key}'.")
+                        else:
+                            num_duplicates_skipped_this_run += 1
+                            print(f"    Skipped 1 duplicate entry for '{style_key}'.")
+                    else:
+                        print(f"    WARNING: Invalid format detected in final generated entry for '{style_key}'.")
+                        print(f"    Failed validation. Raw text: {cleaned_text[:150]}...")
+
+                elif not api_call_successful:
+                    print(f"    Skipping save for '{style_key}' due to API call failures after retries.")
+
+                # Rate Limiting Pause (after each API call)
+                if current_api_call < total_api_calls:  # Don't sleep after the very last call
+                    time.sleep(SLEEP_TIME_PER_API_CALL)
+
+except KeyboardInterrupt:
+    print("\n--- KeyboardInterrupt detected. Stopping script early. ---")
+except Exception as e:
+    print(f"\nAn unexpected error occurred outside the main loop: {e}")
+    import traceback
+    traceback.print_exc()
+
+finally:
+    print(f"\n--- Run Finished ---")
+    print(f"Total API calls attempted: {current_api_call}")
+    print(f"Entries generated in THIS run: {total_entries_generated_this_run}")
+    print(f"Duplicates skipped in THIS run: {num_duplicates_skipped_this_run}")
+    print(f"Total unique entries now in {OUTPUT_FILENAME}: {len(existing_texts)}")

generate_synthetic_data_openrouter.py

@@ -0,0 +1,302 @@
+import requests # Use requests for HTTP calls
+import os
+import json
+import time
+import re
+import sys
+import math
+from dotenv import load_dotenv
+
+load_dotenv()
+# --- Configuration ---
+# !!! GET YOUR API KEY FROM openrouter.ai !!!
+OPENROUTER_API_KEY = os.environ.get("OPENROUTER_API_KEY") # Use environment variable
+# Or paste directly: OPENROUTER_API_KEY = "sk-or-v1-..."
+if not OPENROUTER_API_KEY:
+    raise ValueError("Please set the OPENROUTER_API_KEY environment variable.")
+
+# Set your site URL and app name (optional, but recommended by OpenRouter)
+# Replace with your actual info or leave as is
+SITE_URL = "http://localhost" # Or your app's URL
+APP_NAME = "SyntheticDataGen"
+
+# Choose a model available on OpenRouter (Check openrouter.ai/models)
+# Use the OpenRouter format: provider/model-name
+MODEL_NAME = "meta-llama/llama-4-maverick:free"
+# Other examples: "openai/gpt-4o", "anthropic/claude-3-haiku", "google/gemini-pro"
+
+# Your list of detailed input descriptions (Paste the 20 descriptions here)
+input_descriptions = [
+    # === PASTE THE 20 DESCRIPTIONS FROM THE PREVIOUS ANSWER HERE ===
+    
+    "Rice - Blast: The image shows multiple leaves of a rice plant during the tillering stage. Many leaves exhibit diamond-shaped lesions, approximately 1-2 cm long, with grayish-white centers and distinct dark brown borders. Some lesions are merging, causing larger blighted areas. No insects are visible.",
+    "Cotton - Bollworm: Close-up view of a developing cotton boll. Several small (<1cm), pinkish-white larvae are seen boring into the boll surface near the bracts. Small entry holes with some dark, granular frass (excrement) are visible. The surrounding leaves appear relatively healthy.",
+    "Maize - Nitrogen Deficiency: Image displays the lower leaves of a young maize plant. The leaves show distinct yellowing starting at the leaf tip and progressing down the midrib in a characteristic V-shape pattern. The leaf margins remain relatively green initially. Upper leaves appear greener.",
+    "Tomato - Early Blight: Several lower leaves of a mature tomato plant are shown. They exhibit circular to irregular dark brown lesions, 0.5-1.5 cm in diameter. Many lesions display characteristic concentric rings, giving a 'target board' appearance. Significant yellowing (chlorosis) surrounds the lesions, and some affected leaves are starting to wither.",
+    "Wheat - Stripe Rust: Picture shows the upper leaves of a wheat plant nearing the flag leaf stage. Numerous small, bright yellow-orange pustules are arranged in distinct, long stripes running parallel to the leaf veins. When touched, a yellowish powder (spores) rubs off.",
+    "Chili - Leaf Curl Virus: The image focuses on the growing tip of a chili plant. The young leaves are severely deformed, showing upward curling, puckering, and twisting. The leaves appear thickened and brittle, and veins are sometimes noticeably thicker. Plant growth appears stunted overall. Tiny whitefly insects may be subtly present but are not the primary focus.",
+    "Groundnut - Tikka Disease (Leaf Spot): Several leaves of a groundnut plant are displayed. They are covered with numerous circular, dark brown to black spots, about 2-8 mm wide. Each spot is surrounded by a distinct yellow halo. Some older spots on lower leaves might lack the clear halo.",
+    "Soybean - Healthy: Image shows a section of a healthy, vigorously growing soybean plant canopy during the vegetative stage. The leaves are uniformly deep green, fully expanded, and show no signs of spotting, yellowing, mosaic patterns, insect feeding, or wilting. Stems appear sturdy.",
+    "Mango - Anthracnose on Fruit: A near-ripe mango fruit is shown. It displays several sunken, irregular-shaped black spots on the peel. Some spots appear to be coalescing into larger dark lesions. In humid conditions, pinkish spore masses might be visible in the center of older spots (though not clearly visible here).",
+    "Sugarcane - Early Shoot Borer Damage: Focus is on the base of young sugarcane tillers (shoots). The central whorl of leaves is dried up, creating a characteristic \"dead heart\" symptom, which can be easily pulled out. Small bore holes may be visible near the base of the affected shoot.",
+    "Potato - Late Blight: Image shows potato leaves with large, irregular, water-soaked lesions, often starting at the leaf margins or tips. The lesions rapidly turn dark brown to black. A fuzzy white fungal growth (sporangiophores) might be visible on the underside of the leaves near the edge of the lesions, especially in moist conditions.",
+    "Brinjal (Eggplant) - Aphid Infestation: Close-up shows the underside of several young brinjal leaves and stems. They are heavily infested with clusters of small, pear-shaped insects, mostly greenish or blackish in color. Some shiny, sticky honeydew secretions might be visible on the leaf surfaces below the infestation.",
+    "Okra (Lady's Finger) - Yellow Vein Mosaic Virus: The image shows several leaves of an okra plant. The veins of the leaves are prominently yellow, and the yellow network extends into the surrounding green leaf tissue, creating a distinct mosaic pattern. The leaves might be slightly reduced in size.",
+    "Cabbage - Diamondback Moth Larvae: Several outer leaves of a cabbage head are shown. Small (around 1 cm), greenish larvae with a slightly tapered body are visible, actively feeding on the leaf tissue. They have created irregular holes or \"window panes\" (where only one layer of the leaf epidermis is left). Some silken threads might be present.",
+    "Pigeon Pea (Tur/Arhar) - Pod Borer Damage: Image shows several developing pigeon pea pods. Some pods have distinct circular bore holes on the surface. One pod is broken open slightly, revealing a visible greenish or brownish larva feeding inside on the developing seeds. Frass might be present near the holes.",
+    "Mustard - White Rust: Focus is on mustard leaves and potentially flower stalks. White, blister-like pustules, somewhat raised and initially smooth, are scattered on the underside of the leaves. Corresponding yellow spots may be visible on the upper leaf surface. Affected flower parts can become swollen and distorted.",
+    "Apple - Powdery Mildew: Young leaves and shoot tips of an apple sapling are shown. They are covered with a characteristic white to grayish powdery fungal growth. Affected leaves may appear distorted, curled, or stunted.",
+    "Grapes - Downy Mildew: Image displays the upper surface of grape leaves showing distinct, angular, yellowish-green spots (\"oil spots\"). On the underside of the leaf, corresponding to these spots, a white, downy fungal growth is visible, especially under humid conditions.",
+    "Papaya - Mealybug Infestation: Close-up on the stem and leaf petioles of a papaya plant, especially near the growing point. Clusters of white, cottony masses are visible, covering small, oval, soft-bodied insects (mealybugs). Ants may also be present, attracted to the honeydew secreted by the mealybugs.",
+    "Banana - Sigatoka Leaf Spot: Several mature banana leaves are shown. They exhibit numerous small, elliptical spots parallel to the leaf veins. Initially yellowish-green, the spots enlarge, become dark brown or black, and often develop a grayish center surrounded by a dark border and a yellow halo. Severe spotting leads to leaf necrosis."
+    # === END OF DESCRIPTION LIST ===
+]
+
+OUTPUT_FILENAME = "agri_synthetic_data_generated_openrouter.jsonl" # Changed filename
+# Consider slightly higher temperature for variation across runs, but check quality
+GENERATION_TEMPERATURE = 0.7 # Temperature control works similarly
+# Adjust sleep time based on OpenRouter rate limits (vary per model and user tier)
+SLEEP_TIME_PER_API_CALL = 2 # Start with 2, adjust if you hit rate limits
+
+# OpenRouter API Endpoint
+OPENROUTER_API_URL = "https://openrouter.ai/api/v1/chat/completions"
+
+# --- Modified Prompt Template with More Explicit Instructions ---
+single_prompt_template = """
+You are an AI assistant generating fine-tuning data for an agricultural diagnosis chatbot.
+Your task is to generate a response in a specific format based on the provided plant symptom description and the user's implied question style.
+
+Follow these instructions STRICTLY:
+
+1.  Analyze the User Question Style: Understand the level of detail requested by the user prompt ({user_question_style}).
+2.  Analyze the Input Description: Carefully read the provided plant symptom description.
+3.  Generate the Response: Create a helpful and accurate response that directly addresses the user's question style, based *only* on the information in the input description.
+4.  Format the Output EXACTLY as specified below and make sure to include ALL the tags, especially the final <end_of_turn> tag:
+
+<bos><start_of_turn>user
+{user_question}
+{input_description}<end_of_turn>
+<start_of_turn>model
+[YOUR GENERATED RESPONSE HERE BASED ON THE DESCRIPTION AND QUESTION STYLE]<end_of_turn>
+
+IMPORTANT: Make sure your response includes all required tags, especially ending with <end_of_turn>
+
+--- Input Data ---
+
+User Question Style: {user_question_style}
+User Question: {user_question}
+Input Description:
+{input_description}
+
+--- Generate the Output Below ---
+"""
+
+# Define the different user question styles (Remains the same)
+user_prompts_for_style = {
+     "detailed": "Analyze the provided description of plant symptoms and provide a detailed technical diagnosis and recommendations.",
+     "short": "Briefly identify the problem shown in the description and give the main point.",
+     "easy": "Explain what might be wrong based on this description, using simple words, and what I should do first."
+}
+
+
+# --- Load Existing Data (Remains the same) ---
+existing_texts = set()
+if os.path.exists(OUTPUT_FILENAME):
+    print(f"Loading existing entries from {OUTPUT_FILENAME} to avoid duplicates...")
+    try:
+        with open(OUTPUT_FILENAME, 'r', encoding='utf-8') as infile:
+            for line in infile:
+                try:
+                    data = json.loads(line)
+                    if 'text' in data:
+                        existing_texts.add(data['text'])
+                except (json.JSONDecodeError, KeyError) as e:
+                    print(f"  Skipping invalid line: {line.strip()} - Error: {e}")
+        print(f"Loaded {len(existing_texts)} unique existing entries.")
+    except Exception as e:
+        print(f"Error loading existing file: {e}. Starting fresh.")
+else:
+    print(f"Output file {OUTPUT_FILENAME} not found. Starting fresh.")
+
+# --- Main Generation Logic (Revised for OpenRouter with debug info) ---
+total_entries_generated_this_run = 0
+num_duplicates_skipped_this_run = 0
+total_api_calls = len(input_descriptions) * len(user_prompts_for_style)
+current_api_call = 0
+
+print(f"\nStarting data generation for {len(input_descriptions)} descriptions...")
+print(f"Will make {total_api_calls} individual API calls via OpenRouter.")
+print(f"Output will be appended to: {OUTPUT_FILENAME}")
+print(f"Using OpenRouter model: {MODEL_NAME}, Temperature: {GENERATION_TEMPERATURE}")
+print(f"Pausing {SLEEP_TIME_PER_API_CALL} seconds between API calls.")
+
+# Prepare headers for OpenRouter API calls
+headers = {
+    "Authorization": f"Bearer {OPENROUTER_API_KEY}",
+    "HTTP-Referer": SITE_URL, # Optional, but recommended
+    "X-Title": APP_NAME,      # Optional, but recommended
+    "Content-Type": "application/json"
+}
+
+try:
+    with open(OUTPUT_FILENAME, 'a', encoding='utf-8') as f:
+        # Iterate through each description
+        for i, description in enumerate(input_descriptions):
+            print(f"\n--- Processing Description {i+1}/{len(input_descriptions)} ---")
+            print(f"Input: {description[:100]}...") # Print snippet
+
+            # Iterate through each prompt style for the current description
+            for style_key, user_question in user_prompts_for_style.items():
+                current_api_call += 1
+                print(f"  Generating '{style_key}' style response ({current_api_call}/{total_api_calls})...")
+
+                # --- Format the SINGLE prompt (Remains the same) ---
+                pasted_desc_safe = description.replace('{', '{{').replace('}', '}}')
+                formatted_prompt = single_prompt_template.format(
+                    user_question_style=style_key,
+                    user_question=user_question,
+                    input_description=pasted_desc_safe
+                )
+
+                # --- Structure messages for the API (Remains the same) ---
+                messages = [
+                    {
+                        "role": "user",
+                        "content": formatted_prompt
+                    }
+                ]
+
+                # --- Prepare payload for OpenRouter ---
+                payload = {
+                    "model": MODEL_NAME,
+                    "messages": messages,
+                    "temperature": GENERATION_TEMPERATURE,
+                    "max_tokens": 2048,  # Set a higher limit to avoid truncation
+                    # "top_p": 0.9,     # Optional: Set top_p if needed
+                    # "stream": False   # We want the full response, not streaming
+                }
+
+                # --- Call the OpenRouter API (with retries) ---
+                api_call_successful = False
+                retry_count = 0
+                max_retries = 3
+                response_text = None
+                while not api_call_successful and retry_count <= max_retries:
+                    try:
+                        print(f"    Sending request to OpenRouter (Attempt {retry_count + 1})...")
+                        start_time = time.time()
+
+                        # Make the POST request
+                        response = requests.post(
+                            OPENROUTER_API_URL,
+                            headers=headers,
+                            json=payload # Send data as JSON
+                        )
+
+                        end_time = time.time()
+                        print(f"    OpenRouter API call took {end_time - start_time:.2f} seconds.")
+
+                        # Check for HTTP errors (4xx or 5xx)
+                        response.raise_for_status()
+
+                        # Parse the JSON response
+                        response_data = response.json()
+
+                        # Extract response content
+                        if (response_data.get('choices') and
+                            isinstance(response_data['choices'], list) and
+                            len(response_data['choices']) > 0 and
+                            response_data['choices'][0].get('message') and
+                            response_data['choices'][0]['message'].get('content')):
+
+                            response_text = response_data['choices'][0]['message']['content']
+                            print(f"    DEBUG - Last 50 chars of response: {response_text[-50:] if response_text else 'None'}")
+
+                            # Basic check if the response looks like the start of our format
+                            if response_text and response_text.strip().startswith("<bos>"):
+                                api_call_successful = True
+                            else:
+                                print("    WARNING: Received response doesn't start with '<bos>'. Might be malformed.")
+                                print(f"    Raw Response Snippet: {response_text[:200]}...")
+                                raise ValueError("Response format incorrect")
+                        else:
+                           # Handle unexpected response structure
+                           print("    WARNING: Received unexpected response structure from OpenRouter.")
+                           print(f"    Raw response data: {response_data}")
+                           response_text = None
+                           raise ValueError("Invalid OpenRouter response structure") # Trigger retry
+
+                    except requests.exceptions.RequestException as e:
+                        retry_count += 1
+                        print(f"    ERROR during OpenRouter API call (Attempt {retry_count}): {e}")
+                        # Check for specific status codes if needed (e.g., 429 Too Many Requests)
+                        if hasattr(e, 'response') and e.response and e.response.status_code == 429:
+                             print("    Rate limit likely exceeded.")
+                             # Increase wait time significantly for rate limits
+                             wait_time = SLEEP_TIME_PER_API_CALL * (2 ** retry_count) * 5
+                        else:
+                            wait_time = SLEEP_TIME_PER_API_CALL * (2 ** (retry_count - 1))
+
+                        if retry_count > max_retries:
+                            print(f"    Max retries reached for Description {i+1}, Style '{style_key}'. Skipping.")
+                            response_text = None
+                            break # Break the retry loop for this specific call
+                        print(f"    Retrying in {wait_time:.2f} seconds...")
+                        time.sleep(wait_time)
+                    except Exception as e: # Catch other potential errors (like ValueError from format check)
+                        retry_count += 1
+                        print(f"    ERROR processing request/response (Attempt {retry_count}): {e}")
+                        wait_time = SLEEP_TIME_PER_API_CALL * (2 ** (retry_count - 1))
+                        if retry_count > max_retries:
+                            print(f"    Max retries reached for Description {i+1}, Style '{style_key}'. Skipping.")
+                            response_text = None
+                            break # Break the retry loop
+                        print(f"    Retrying in {wait_time:.2f} seconds...")
+                        time.sleep(wait_time)
+
+                # --- Process and Save Response (Modified with more flexible validation) ---
+                if api_call_successful and response_text:
+                    cleaned_text = response_text.strip()
+                    print(f"    DEBUG - Response starts with: {cleaned_text[:50]}...")
+                    print(f"    DEBUG - Response ends with: ...{cleaned_text[-50:]}")
+
+                    # More flexible check to troubleshoot
+                    if cleaned_text.startswith("<bos>"):
+                        if "<end_of_turn>" not in cleaned_text:
+                            print("    WARNING: Response missing <end_of_turn> tag. Adding it for consistency.")
+                            cleaned_text = cleaned_text + "<end_of_turn>"
+                        
+                        if cleaned_text not in existing_texts:
+                            data_entry = {"text": cleaned_text}
+                            json.dump(data_entry, f, ensure_ascii=False)
+                            f.write('\n')
+                            existing_texts.add(cleaned_text)
+                            total_entries_generated_this_run += 1
+                            print(f"    Saved 1 new entry for '{style_key}'.")
+                        else:
+                            num_duplicates_skipped_this_run += 1
+                            print(f"    Skipped 1 duplicate entry for '{style_key}'.")
+                    else:
+                        print(f"    WARNING: Invalid format detected in final generated entry for '{style_key}'.")
+                        print(f"    Failed validation. Raw text: {cleaned_text[:150]}...")
+
+                elif not api_call_successful:
+                     print(f"    Skipping save for '{style_key}' due to API call failures after retries.")
+
+                # --- Rate Limiting Pause (after each API call) ---
+                if current_api_call < total_api_calls: # Don't sleep after the very last call
+                    time.sleep(SLEEP_TIME_PER_API_CALL)
+
+except KeyboardInterrupt:
+    print("\n--- KeyboardInterrupt detected. Stopping script early. ---")
+except Exception as e:
+    print(f"\nAn unexpected error occurred outside the main loop: {e}")
+    import traceback
+    traceback.print_exc()
+
+finally:
+    print(f"\n--- Run Finished ---")
+    print(f"Total API calls attempted: {current_api_call}")
+    print(f"Entries generated in THIS run: {total_entries_generated_this_run}")
+    print(f"Duplicates skipped in THIS run: {num_duplicates_skipped_this_run}")
+    print(f"Total unique entries now in {OUTPUT_FILENAME}: {len(existing_texts)}")