Isibor commited on
Commit
d8882c0
·
verified ·
1 Parent(s): f9f3a99

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +626 -70
app.py CHANGED
@@ -1,70 +1,626 @@
1
- import gradio as gr
2
- from huggingface_hub import InferenceClient
3
-
4
-
5
- def respond(
6
- message,
7
- history: list[dict[str, str]],
8
- system_message,
9
- max_tokens,
10
- temperature,
11
- top_p,
12
- hf_token: gr.OAuthToken,
13
- ):
14
- """
15
- For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
16
- """
17
- client = InferenceClient(token=hf_token.token, model="openai/gpt-oss-20b")
18
-
19
- messages = [{"role": "system", "content": system_message}]
20
-
21
- messages.extend(history)
22
-
23
- messages.append({"role": "user", "content": message})
24
-
25
- response = ""
26
-
27
- for message in client.chat_completion(
28
- messages,
29
- max_tokens=max_tokens,
30
- stream=True,
31
- temperature=temperature,
32
- top_p=top_p,
33
- ):
34
- choices = message.choices
35
- token = ""
36
- if len(choices) and choices[0].delta.content:
37
- token = choices[0].delta.content
38
-
39
- response += token
40
- yield response
41
-
42
-
43
- """
44
- For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
45
- """
46
- chatbot = gr.ChatInterface(
47
- respond,
48
- type="messages",
49
- additional_inputs=[
50
- gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
51
- gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
52
- gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
53
- gr.Slider(
54
- minimum=0.1,
55
- maximum=1.0,
56
- value=0.95,
57
- step=0.05,
58
- label="Top-p (nucleus sampling)",
59
- ),
60
- ],
61
- )
62
-
63
- with gr.Blocks() as demo:
64
- with gr.Sidebar():
65
- gr.LoginButton()
66
- chatbot.render()
67
-
68
-
69
- if __name__ == "__main__":
70
- demo.launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import io
3
+ import torch
4
+ import torch.nn as nn
5
+ from torchvision import transforms
6
+ from torchvision.models import efficientnet_b3, EfficientNet_B3_Weights
7
+ from PIL import Image
8
+ from transformers import AutoTokenizer, AutoModelForCausalLM
9
+ import gradio as gr
10
+ from typing import Dict, Any, List, Tuple
11
+
12
+ KNOWLEDGE_BASE: Dict[str, Dict[str, Any]] = {
13
+ "DROUGHT_LEAVES": {
14
+ "keywords": ["drought", "wilt", "dehydrated", "scorched leaf", "shriveled leaf", "water stress", "leaf margin"],
15
+ "chunks": {
16
+ "DESCRIPTION_AND_CAUSE": "**The leaves are undergoing desiccation (drying out) because the plant lacks sufficient soil moisture.** This is often triggered by ** prolonged drought, high heat, or windy conditions ** that cause the plant to lose water faster than its roots can supply it.",
17
+ "DIAGNOSTIC_CLUES": "Look for leaf margins that are ** brown, brittle, or curled inwards **. The leaves will ** wilt noticeably during the midday sun **, even if they recover slightly overnight. Check soil is dry 4-6 inches deep.",
18
+ "IMMEDIATE_ACTION": "Water the plants ** deeply and evenly ** using ** drip irrigation ** or a soaker hose, checking the soil moisture 6 inches down. If possible, ** apply shade nets ** during the peak afternoon heat.",
19
+ "PREVENTION_AND_LONGTERM": "** Apply a thick layer of organic mulch ** (straw or dried leaves) to cool the soil and drastically reduce water evaporation. Ensure your irrigation system is ** consistent and efficient **."
20
+ }
21
+ },
22
+ "DROUGHT_FRUITS": {
23
+ "keywords": ["drought fruit", "dry fruit", "shriveled fruit", "dried fruit", "leathery fruit", "fruit desiccation"],
24
+ "chunks": {
25
+ "DESCRIPTION_AND_CAUSE": "**The fruits are small, hard, and shriveled because water stress limits the plant’s ability to send enough water to the developing fruit tissue.** This is a symptom of ** severe or prolonged drought ** during the critical fruit enlargement stage.",
26
+ "DIAGNOSTIC_CLUES": "The fruits will feel ** hard or leathery ** instead of plump. They may show ** uneven ripening ** or stop enlarging completely.",
27
+ "IMMEDIATE_ACTION": "Immediately ** ensure consistent, deep irrigation ** to stabilize soil moisture. Lightly misting the foliage early in the morning can provide temporary relief.",
28
+ "PREVENTION_AND_LONGTERM": "Maintain a ** strict irrigation schedule ** based on the weather forecast and plant stage. Consider a ** foliar spray of potassium ** during fruiting to improve the fruit's water-holding capacity."
29
+ }
30
+ },
31
+ "UNRIPE_FRUITS": {
32
+ "keywords": ["unripe", "green fruit", "immature", "delayed", "slow color", "potassium deficiency"],
33
+ "chunks": {
34
+ "DESCRIPTION_AND_CAUSE": "**Ripening is delayed because the necessary sugar accumulation and pigment production enzymes are inhibited.** Common causes include ** low temperatures, insufficient sunlight due to shading, or nutrient imbalances **, particularly ** low phosphorus or potassium ** and excessive nitrogen.",
35
+ "DIAGNOSTIC_CLUES": "Fruits remain ** firm and primarily green or pale ** for an extended period after reaching full size. Check for ** dense foliage ** that is blocking light.",
36
+ "IMMEDIATE_ACTION": "** Remove excessive foliage ** (light pruning) to expose the fruits to 6–8 hours of direct sunlight per day. If available, apply a quick-release ** potassium-rich fertilizer ** near the plants.",
37
+ "PREVENTION_AND_LONGTERM": "** Conduct a soil test ** to check your P:K:N balance. Ensure adequate potassium and phosphorus levels are maintained before and during the fruiting period. Choose a variety suited to your local climate."
38
+ }
39
+ },
40
+ "HEALTHY_RIPE": {
41
+ "keywords": ["ripe", "mature", "healthy", "lush", "uniformly red", "quality", "post-harvest", "no spots"],
42
+ "chunks": {
43
+ "DESCRIPTION_AND_CAUSE": "The plant is in ** optimal health with successful maturity **. The fruit's uniform color and firmness are due to ** balanced water supply, sufficient nutrients, and proper cultural practices ** that allow natural ripening processes to proceed efficiently.",
44
+ "DIAGNOSTIC_CLUES": "** Fruits are uniformly red, glossy, firm, and aromatic **, without any signs of spots, mold, or shriveling. Leaves are a ** vibrant, dark green **.",
45
+ "IMMEDIATE_ACTION": "** Harvest the fruit in the morning ** when the fruits are cool. ** Handle gently ** to avoid bruising and cool the fruit quickly after picking to prolong shelf life.",
46
+ "PREVENTION_AND_LONGTERM": "Maintain a ** balanced fertilization program ** and continue ** regular scouting ** for early signs of pests and diseases. ** Prioritize good drainage ** to prevent waterlogging."
47
+ }
48
+ },
49
+ "FUNGAL_LEAVES": {
50
+ "keywords": ["dark spot", "purplish spot", "leaf spot", "blight", "leaf mildew", "fruit mildew", "white powder leaf"],
51
+ "chunks": {
52
+ "DESCRIPTION_AND_CAUSE": "**Leaves are infected by a fungal pathogen**, causing cell necrosis (death) or surface growth. This infection is ** favored by extended periods of leaf wetness, high humidity, or poor air circulation **.",
53
+ "DIAGNOSTIC_CLUES": "Look for ** dark or purplish circular spots ** on the leaves, or a ** fuzzy white/gray powder ** coating the leaf surface. New growth may appear stunted or distorted.",
54
+ "IMMEDIATE_ACTION": "** Immediately remove and destroy ** all infected leaves and plant debris. ** Switch from overhead irrigation ** to drip or soaker methods, and ** water only at the base ** of the plant in the morning.",
55
+ "PREVENTION_AND_LONGTERM": "** Ensure adequate plant spacing ** to improve air circulation. Consider applying an approved ** copper-based or systemic fungicide ** according to local guidelines, and practice ** crop rotation ** (3–4 years)."
56
+ }
57
+ },
58
+ "FUNGAL_FRUITS": {
59
+ "keywords": ["fruit rot", "gray mold", "botrytis", "moldy fruit", "soft fruit", "fruit mildew", "white powder fruit"],
60
+ "chunks": {
61
+ "DESCRIPTION_AND_CAUSE": "**Fruit tissue is being decomposed by fungi (like *Botrytis cinerea*) that colonize the fruit.** This is often caused by ** extended wet periods, poor ventilation in the canopy, or damage/wounds ** on the fruit surface.",
62
+ "DIAGNOSTIC_CLUES": "The fruit becomes ** soft, mushy, and often develops a fuzzy gray mold ** or a ** white powdery coating **. The rot spreads quickly, especially where fruits are clustered or touch the ground.",
63
+ "IMMEDIATE_ACTION": "** Harvest frequently and immediately remove and discard (do not compost) ** all rotten and infected fruits. ** Apply mulch ** beneath the plants to prevent fruit contact with the soil.",
64
+ "PREVENTION_AND_LONGTERM": "Maintain a ** clean field environment **. Implement a preventative fungicide or bio-control program during the flowering and fruiting stage, and ensure ** good airflow ** within the plant canopy."
65
+ }
66
+ }
67
+ }
68
+
69
+ def retrieve_knowledge(caption: str, knowledge_base: Dict[str, Dict[str, Any]]) -> List[Tuple[str, str]]:
70
+ caption_lower = caption.lower()
71
+ best_match_key = None
72
+ max_matches = 0
73
+ priority_order = list(knowledge_base.keys())
74
+ for key in priority_order:
75
+ matches = sum(1 for keyword in knowledge_base[key]["keywords"] if keyword in caption_lower)
76
+ phrase_boost = sum(1 for keyword in knowledge_base[key]["keywords"] if " " in keyword and keyword in caption_lower)
77
+ matches += phrase_boost
78
+ if matches > max_matches:
79
+ max_matches = matches
80
+ best_match_key = key
81
+ retrieved_chunks = []
82
+ if best_match_key and max_matches > 0:
83
+ for label, text in knowledge_base[best_match_key]["chunks"].items():
84
+ retrieved_chunks.append((label, text))
85
+ return retrieved_chunks
86
+ if not retrieved_chunks and any(kw in caption_lower for kw in KNOWLEDGE_BASE.get("HEALTHY_RIPE", {}).get("keywords", [])):
87
+ for label, text in KNOWLEDGE_BASE.get("HEALTHY_RIPE", {}).get("chunks", {}).items():
88
+ retrieved_chunks.append((label, text))
89
+ return retrieved_chunks
90
+ return []
91
+
92
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
93
+ print(f"Using device: {device}")
94
+ dtype = torch.float16 if device.type == "cuda" else torch.float32
95
+
96
+ try:
97
+ from transformers import LlamaTokenizerFast
98
+ tokenizer = LlamaTokenizerFast.from_pretrained("hf-internal-testing/llama-tokenizer")
99
+ except Exception:
100
+ tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
101
+
102
+ if tokenizer.pad_token is None:
103
+ tokenizer.pad_token = tokenizer.eos_token if hasattr(tokenizer, "eos_token") else "<|pad|>"
104
+
105
+ VOCAB_SIZE = getattr(tokenizer, "vocab_size", 30522)
106
+
107
+ ENCODER_FEATURE_DIM = 1536
108
+ MAX_LEN = 30
109
+ DROPOUT_RATE = 0.45
110
+
111
+ class ImageEncoder(nn.Module):
112
+ def __init__(self, embed_dim=ENCODER_FEATURE_DIM):
113
+ super().__init__()
114
+ backbone = efficientnet_b3(weights=EfficientNet_B3_Weights.IMAGENET1K_V1)
115
+ self.feature_extractor = backbone.features
116
+
117
+ def forward(self, x):
118
+ x = self.feature_extractor(x)
119
+ x = x.flatten(2).permute(0, 2, 1)
120
+ return x
121
+
122
+ class CaptionModel(nn.Module):
123
+ def __init__(
124
+ self,
125
+ encoder,
126
+ vocab_size=VOCAB_SIZE,
127
+ d_model=512,
128
+ nhead=8,
129
+ num_layers=4,
130
+ max_len=MAX_LEN,
131
+ dropout_rate=DROPOUT_RATE,
132
+ encoder_feature_dim=ENCODER_FEATURE_DIM,
133
+ ):
134
+ super().__init__()
135
+ self.encoder = encoder
136
+ self.feature_proj = nn.Linear(encoder_feature_dim, d_model)
137
+ self.embedding = nn.Embedding(vocab_size, d_model)
138
+ self.pos_encoder = nn.Parameter(torch.zeros(1, max_len, d_model))
139
+ decoder_layer = nn.TransformerDecoderLayer(
140
+ d_model,
141
+ nhead,
142
+ dim_feedforward=d_model * 4,
143
+ dropout=dropout_rate,
144
+ batch_first=True,
145
+ )
146
+ self.transformer_decoder = nn.TransformerDecoder(decoder_layer, num_layers)
147
+ self.fc_out = nn.Linear(d_model, vocab_size)
148
+
149
+ def forward(self, images, captions):
150
+ features = self.encoder(images)
151
+ features = self.feature_proj(features)
152
+ embeddings = self.embedding(captions) + self.pos_encoder[:, : captions.size(1)]
153
+ T = captions.size(1)
154
+ tgt_mask = nn.Transformer.generate_square_subsequent_mask(T).to(captions.device)
155
+ output = self.transformer_decoder(tgt=embeddings, memory=features, tgt_mask=tgt_mask)
156
+ return self.fc_out(output)
157
+
158
+ def generate_caption_beam(
159
+ model,
160
+ img_tensor,
161
+ device,
162
+ max_len=MAX_LEN,
163
+ num_beams=3,
164
+ repetition_penalty=1.5,
165
+ length_penalty=0.7,
166
+ ):
167
+ model.eval()
168
+ with torch.no_grad():
169
+ img = img_tensor.unsqueeze(0).to(device)
170
+ features = model.encoder(img)
171
+ features = model.feature_proj(features)
172
+ bos_id = tokenizer.bos_token_id if hasattr(tokenizer, "bos_token_id") else 0
173
+ beam = [(torch.tensor([[bos_id]], device=device), 0.0)]
174
+ finished_beams = []
175
+
176
+ for _ in range(max_len):
177
+ new_beam = []
178
+ if len(finished_beams) >= num_beams:
179
+ break
180
+ for seq, raw_score in beam:
181
+ if hasattr(tokenizer, "eos_token_id") and seq[0, -1].item() == tokenizer.eos_token_id:
182
+ normalized_score = raw_score / (seq.size(1) ** length_penalty)
183
+ finished_beams.append((seq, normalized_score))
184
+ continue
185
+ T = seq.size(1)
186
+ tgt_mask = nn.Transformer.generate_square_subsequent_mask(T).to(device)
187
+ embeddings = model.embedding(seq) + model.pos_encoder[:, :T]
188
+ output = model.transformer_decoder(tgt=embeddings, memory=features, tgt_mask=tgt_mask)
189
+ logits = model.fc_out(output)[:, -1, :].squeeze()
190
+ for prev_id in seq.squeeze(0).tolist():
191
+ if logits[prev_id] > 0:
192
+ logits[prev_id] /= repetition_penalty
193
+ else:
194
+ logits[prev_id] *= repetition_penalty
195
+ probs = torch.log_softmax(logits, dim=-1)
196
+ topk_probs, topk_idx = torch.topk(probs, num_beams)
197
+ for i in range(num_beams):
198
+ next_id = topk_idx[i].unsqueeze(0).unsqueeze(0)
199
+ new_seq = torch.cat([seq, next_id], dim=1)
200
+ new_raw_score = raw_score + topk_probs[i].item()
201
+ new_beam.append((new_seq, new_raw_score))
202
+ new_beam.sort(key=lambda x: x[1], reverse=True)
203
+ beam = new_beam[:num_beams]
204
+
205
+ for seq, raw_score in beam:
206
+ normalized_score = raw_score / (seq.size(1) ** length_penalty)
207
+ finished_beams.append((seq, normalized_score))
208
+
209
+ if not finished_beams:
210
+ return "Caption generation failed."
211
+
212
+ best_seq, _ = sorted(finished_beams, key=lambda x: x[1], reverse=True)[0]
213
+ caption = tokenizer.decode(best_seq.squeeze().tolist(), skip_special_tokens=True)
214
+ caption = caption.replace("..", ".").replace(". .", ".").strip()
215
+ caption = " ".join(caption.split())
216
+ if caption:
217
+ first_period_index = caption.find(".")
218
+ if first_period_index != -1:
219
+ caption = caption[: first_period_index + 1]
220
+ elif not caption.endswith("."):
221
+ caption += "."
222
+ return caption
223
+
224
+ MODEL_PATH = "EfficientNetB3_model.pth"
225
+ model_loaded_successfully = False
226
+ try:
227
+ if os.path.exists(MODEL_PATH):
228
+ encoder = ImageEncoder()
229
+ caption_model = CaptionModel(encoder, vocab_size=VOCAB_SIZE, dropout_rate=DROPOUT_RATE).to(device)
230
+ caption_model.load_state_dict(torch.load(MODEL_PATH, map_location=device))
231
+ caption_model.eval()
232
+ model_loaded_successfully = True
233
+ else:
234
+ raise FileNotFoundError
235
+ except Exception:
236
+ class MockCaptionModel(nn.Module):
237
+ def __init__(self):
238
+ super().__init__()
239
+
240
+ def eval(self):
241
+ pass
242
+
243
+ caption_model = MockCaptionModel()
244
+
245
+ transform = transforms.Compose(
246
+ [
247
+ transforms.Resize((224, 224)),
248
+ transforms.ToTensor(),
249
+ transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
250
+ ]
251
+ )
252
+
253
+ LLM_MODEL_ID = "Qwen/Qwen2.5-3B-Instruct"
254
+ llm = None
255
+ llm_tokenizer = None
256
+ try:
257
+ llm_tokenizer = AutoTokenizer.from_pretrained(LLM_MODEL_ID)
258
+ if device.type == "cuda":
259
+ llm = AutoModelForCausalLM.from_pretrained(LLM_MODEL_ID, torch_dtype=dtype, device_map="auto")
260
+ else:
261
+ llm = AutoModelForCausalLM.from_pretrained(LLM_MODEL_ID, torch_dtype=dtype, device_map="cpu")
262
+ if llm_tokenizer.pad_token is None:
263
+ llm_tokenizer.pad_token = llm_tokenizer.eos_token
264
+ print("LLM loaded:", True)
265
+ except Exception as e:
266
+ print("LLM failed to load (this may be expected on CPU-only environments):", e)
267
+ llm = None
268
+ llm_tokenizer = None
269
+ print("LLM loaded:", False)
270
+
271
+ def get_multiple_recommendations(pred_caption: str, llm_model, tokenizer_model, knowledge_base):
272
+ if llm_model is None or tokenizer_model is None:
273
+ return (
274
+ "Recommendations not available: LLM failed to load. The required models could not be loaded on this device.",
275
+ [],
276
+ )
277
+
278
+ retrieved_chunks = retrieve_knowledge(pred_caption, knowledge_base)
279
+ context_text = ""
280
+ if retrieved_chunks:
281
+ context_text = "\n\n--- RAG KNOWLEDGE CONTEXT ---\n"
282
+ for label, text in retrieved_chunks:
283
+ context_text += f"**{label.replace('_', ' ')}**: {text}\n"
284
+ context_text += "------------------------------\n\n"
285
+
286
+ system_prompt = (
287
+ "You are a highly detailed and precise agricultural assistant specializing in strawberries. "
288
+ "Your task is to generate a rich, professional, and actionable recommendation strictly based on the provided caption and RAG context. "
289
+ "The output MUST be formatted into three distinct sections, each ending with a single paragraph/sentence. "
290
+ "Do not introduce unobserved problems or speculate. Do not use salutations or empathy. "
291
+ )
292
+
293
+ user_prompt = (
294
+ f'CAPTION: "{pred_caption}"\n\n'
295
+ f"{context_text}"
296
+ "INSTRUCTION: Generate a comprehensive analysis and recommendation in the following three-part stacked format, with rich descriptive text:\n"
297
+ "1. **Cause**: A detailed sentence describing the likely cause and condition based on the caption and RAG context.\n"
298
+ "2. **Immediate Action**: A comprehensive sentence detailing specific, time-sensitive actions the grower must take immediately.\n"
299
+ "3. **Long-term Action**: A forward-looking sentence outlining preventative and sustainable strategies for the future.\n"
300
+ "Ensure the output strictly follows the 'Label: Text' format below. Do not add extra text, line breaks, or numbering.\n\n"
301
+ "**Cause**: [Your descriptive text for the cause]\n"
302
+ "**Immediate Action**: [Your descriptive text for the immediate steps]\n"
303
+ "**Long-term Action**: [Your descriptive text for the long-term steps]\n"
304
+ )
305
+
306
+ messages = [
307
+ {"role": "system", "content": system_prompt},
308
+ {"role": "user", "content": user_prompt},
309
+ ]
310
+
311
+ try:
312
+ prompt = tokenizer_model.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
313
+ except Exception:
314
+ prompt = system_prompt + "\n\n" + user_prompt
315
+
316
+ inputs = tokenizer_model(prompt, return_tensors="pt", truncation=True, max_length=1024).to(llm_model.device)
317
+ output = llm_model.generate(
318
+ **inputs,
319
+ max_new_tokens=300,
320
+ temperature=0.7,
321
+ top_p=0.9,
322
+ do_sample=True,
323
+ repetition_penalty=1.1,
324
+ pad_token_id=tokenizer_model.pad_token_id,
325
+ )
326
+ text = tokenizer_model.decode(output[0], skip_special_tokens=False)
327
+ response_start_tag = "<|im_start|>assistant\n"
328
+ if response_start_tag in text:
329
+ generated_text = text.split(response_start_tag)[-1].strip()
330
+ else:
331
+ generated_text = tokenizer_model.decode(output[0][inputs.input_ids.shape[1] :], skip_special_tokens=True).strip()
332
+
333
+ final_recommendations = (
334
+ generated_text.replace(getattr(tokenizer_model, "eos_token", ""), "").replace("<|im_end|>", "").strip()
335
+ )
336
+ final_recommendations = final_recommendations.replace("Cause:", "**Cause**:")
337
+ final_recommendations = final_recommendations.replace("Immediate Action:", "**Immediate Action**:")
338
+ final_recommendations = final_recommendations.replace("Long-term Action:", "**Long-term Action**:")
339
+
340
+ return final_recommendations, retrieve_knowledge(pred_caption, knowledge_base)
341
+
342
+ def get_rag_chat_response(message: str, history: list, caption: str, rag_context: str):
343
+ if llm is None or llm_tokenizer is None:
344
+ history.append((message, "Chat not available: LLM failed to load on this device."))
345
+ return history, history
346
+
347
+ chat_system_prompt = (
348
+ "You are an expert, professional agricultural advisor for strawberry plants. "
349
+ "Base your advice STRICTLY on the visual evidence provided (Image Caption) and the expert RAG Knowledge. "
350
+ "Maintain a helpful, advisory, and professional tone. Keep responses concise unless asked for detail. "
351
+ "Do not introduce unobserved problems or speculate. "
352
+ f"--- Image Analysis ---\nCaption: {caption}\n"
353
+ f"--- RAG Knowledge ---\n{rag_context}\n"
354
+ "-----------------------\n"
355
+ "Answer the user's question, using the provided context."
356
+ )
357
+
358
+ messages = [{"role": "system", "content": chat_system_prompt}]
359
+ for user_msg, assistant_msg in history:
360
+ messages.append({"role": "user", "content": user_msg})
361
+ messages.append({"role": "assistant", "content": assistant_msg})
362
+ messages.append({"role": "user", "content": message})
363
+
364
+ try:
365
+ prompt = llm_tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
366
+ except Exception:
367
+ prompt_lines = [chat_system_prompt]
368
+ for m in messages[1:]:
369
+ prompt_lines.append(f"{m['role']}: {m['content']}")
370
+ prompt = "\n\n".join(prompt_lines)
371
+
372
+ inputs = llm_tokenizer(prompt, return_tensors="pt", truncation=True, max_length=1024).to(llm.device)
373
+ output = llm.generate(
374
+ **inputs,
375
+ max_new_tokens=256,
376
+ temperature=0.8,
377
+ top_p=0.9,
378
+ do_sample=True,
379
+ repetition_penalty=1.1,
380
+ pad_token_id=llm_tokenizer.pad_token_id,
381
+ )
382
+
383
+ text = llm_tokenizer.decode(output[0], skip_special_tokens=False)
384
+ response_start_tag = "<|im_start|>assistant\n"
385
+ if response_start_tag in text:
386
+ generated_text = text.split(response_start_tag)[-1].strip()
387
+ else:
388
+ generated_text = llm_tokenizer.decode(output[0][inputs.input_ids.shape[1] :], skip_special_tokens=True).strip()
389
+
390
+ chat_response = (
391
+ generated_text.replace(getattr(llm_tokenizer, "eos_token", ""), "").replace("<|im_end|>", "").strip()
392
+ )
393
+
394
+ history.append((message, chat_response))
395
+ return history, history
396
+
397
+ def process_image_upload(image: Image.Image):
398
+ pil_img = image.convert("RGB")
399
+ try:
400
+ img_tensor = transform(pil_img).to(device)
401
+ except Exception:
402
+ img_tensor = transform(pil_img)
403
+
404
+ if model_loaded_successfully and hasattr(caption_model, "encoder"):
405
+ try:
406
+ caption = generate_caption_beam(caption_model, img_tensor, device)
407
+ except Exception as e:
408
+ print("Caption generation error:", e)
409
+ caption = "A close-up image showing dark purplish spots on the leaves."
410
+ else:
411
+ caption = "A close-up image showing dark purplish spots on the leaves."
412
+
413
+ recommendations, retrieved_list = get_multiple_recommendations(caption, llm, llm_tokenizer, KNOWLEDGE_BASE)
414
+
415
+ if retrieved_list:
416
+ retrieved_str = "\n\n".join([f"**{lab.replace('_', ' ')}**: {txt}" for lab, txt in retrieved_list])
417
+ else:
418
+ retrieved_str = "No RAG context retrieved."
419
+
420
+ return pil_img, caption, retrieved_str, recommendations, []
421
+
422
+ title = "Chat-O-Berry Plant Health Advisor"
423
+
424
+ with gr.Blocks(title=title) as demo:
425
+ gr.Markdown("""
426
+ <style>
427
+ .gradio-container { padding: 0 !important; }
428
+ .gr-block, .gr-row, .gr-column, .gr-container {
429
+ max-width: 100% !important;
430
+ width: 100% !important;
431
+ }
432
+ .gradio-container > div { margin-top: 0 !important; }
433
+
434
+ /* Style the "Structured Recommendation" and "Advisory Chat" tabs like red buttons */
435
+ .mode-tabs .tab-nav button {
436
+ background: #e64545 !important;
437
+ color: #ffffff !important;
438
+ border-radius: 999px !important;
439
+ padding: 6px 14px !important;
440
+ border: none !important;
441
+ font-weight: 600 !important;
442
+ margin-right: 8px !important;
443
+ opacity: 0.7;
444
+ }
445
+ .mode-tabs .tab-nav button.selected {
446
+ opacity: 1;
447
+ box-shadow: 0 0 0 2px rgba(230,69,69,0.25);
448
+ }
449
+ </style>
450
+ """)
451
+
452
+ chat_history_state = gr.State(value=[])
453
+ rag_state = gr.State(value="")
454
+
455
+ with gr.Group(visible=True) as landing_group:
456
+ gr.Markdown("## 🍓 Welcome to Chat-O-Berry")
457
+
458
+ with gr.Row():
459
+ with gr.Column(scale=1):
460
+ landing_image = gr.Image(
461
+ value="/content/samples/strawberry.jpg",
462
+ label=None,
463
+ show_label=False,
464
+ interactive=False,
465
+ height=260,
466
+ elem_classes=["hero-img"],
467
+ )
468
+ with gr.Column(scale=2):
469
+ gr.Markdown(
470
+ """
471
+ <style>
472
+ .card {
473
+ background: #ffffff;
474
+ padding: 18px 22px;
475
+ border-radius: 14px;
476
+ box-shadow: 0 1px 4px rgba(0,0,0,0.08);
477
+ margin-bottom: 14px;
478
+ border-left: 5px solid #e64545;
479
+ }
480
+ .hero-img img {
481
+ border-radius: 16px !important;
482
+ box-shadow: 0 2px 6px rgba(0,0,0,0.15);
483
+ object-fit: cover;
484
+ }
485
+ </style>
486
+
487
+ <div class="card">
488
+ <h3>🍓 A Fruit Worth Knowing</h3>
489
+ Strawberries are a nutrient-dense fruit rich in vitamin C, folate, manganese, and natural antioxidants.
490
+ Their balance of sweetness, acidity, and aroma makes them both delicious and nutritionally meaningful.
491
+ </div>
492
+
493
+ <div class="card">
494
+ <h3>🌱 Understanding Strawberry Plants</h3>
495
+ Behind every berry is a plant with a shallow root system that requires steady moisture and good airflow.
496
+ Strawberries thrive in slightly acidic, well-drained soil and need protection from fungal diseases, pests, and rot.
497
+ </div>
498
+
499
+ <div class="card">
500
+ <h3>🌿 Supporting Healthy Growth</h3>
501
+ Healthy strawberries depend on consistent watering, clean foliage, proper spacing, and early detection of stress.
502
+ Chat-O-Berry helps you stay ahead by analyzing plant images and offering clear, practical guidance.
503
+ </div>
504
+ """
505
+ )
506
+
507
+ gr.Markdown(
508
+ "<p style='text-align:center; font-size:16px;'>Ready to assess your plants? Open the advisor below.</p>"
509
+ )
510
+
511
+ with gr.Row():
512
+ with gr.Column(scale=3):
513
+ gr.Markdown("")
514
+ with gr.Column(scale=4):
515
+ gr.Markdown("")
516
+ with gr.Column(scale=3):
517
+ go_to_advisor_btn = gr.Button(
518
+ "Open Chat-O-Berry Advisor",
519
+ variant="primary",
520
+ size="sm",
521
+ )
522
+
523
+ with gr.Group(visible=False) as advisor_group:
524
+ gr.Markdown("# 🍓 Chat‑O‑Berry Plant Health Advisor")
525
+ gr.Markdown("Upload a plant image for AI‑powered health analysis and agronomic recommendations.")
526
+
527
+ with gr.Row():
528
+ with gr.Column(scale=1):
529
+ image_in = gr.Image(type="pil", label="Upload Plant Image", interactive=True)
530
+ run_btn = gr.Button("Analyze Plant Health", variant="primary")
531
+ hidden_out_image = gr.Image(visible=False)
532
+
533
+ gr.Examples(
534
+ examples=[
535
+ ["samples/darkspot.jpg"],
536
+ ["samples/droughtfruits.jpg"],
537
+ ["samples/fruitrot.png"],
538
+ ["samples/healthyleaf.jpg"],
539
+ ["samples/leafmildew.png"],
540
+ ["samples/ripefruits.jpg"],
541
+ ["samples/unripefruit.jpg"],
542
+ ],
543
+ inputs=[image_in],
544
+ label="Sample strawberry images",
545
+ )
546
+
547
+ with gr.Column(scale=2):
548
+ gr.Markdown("### 🍓 **Plant Health Caption**")
549
+ caption_out = gr.Textbox(label="", lines=2, interactive=False, container=False)
550
+ with gr.Tabs(elem_classes=["mode-tabs"]):
551
+ with gr.TabItem("Structured Recommendation"):
552
+ gr.Markdown("### **Analysis and Action Plan:**")
553
+ rec_out = gr.Textbox(
554
+ label="",
555
+ lines=8,
556
+ interactive=False,
557
+ container=False,
558
+ placeholder="Upload and Analyze an image to receive a structured recommendation here.",
559
+ )
560
+ with gr.TabItem("Advisory Chat"):
561
+ gr.Markdown("### **Interactive Advisory Chat**")
562
+ chat_box = gr.Chatbot(
563
+ height=300,
564
+ label="Advisory Chat based on Image Analysis",
565
+ )
566
+ with gr.Row():
567
+ chat_input = gr.Textbox(
568
+ scale=4,
569
+ placeholder="Ask a follow-up question about the plant's health or treatment...",
570
+ show_label=False,
571
+ )
572
+ chat_send_btn = gr.Button("Send", scale=1, variant="secondary")
573
+
574
+ with gr.Row():
575
+ with gr.Column(scale=7):
576
+ gr.Markdown("")
577
+ with gr.Column(scale=3):
578
+ back_to_home_btn = gr.Button(
579
+ "Back to Home Page",
580
+ variant="primary",
581
+ size="sm",
582
+ )
583
+
584
+ run_btn.click(
585
+ process_image_upload,
586
+ inputs=[image_in],
587
+ outputs=[hidden_out_image, caption_out, rag_state, rec_out, chat_history_state],
588
+ )
589
+
590
+ chat_send_btn.click(
591
+ get_rag_chat_response,
592
+ inputs=[chat_input, chat_history_state, caption_out, rag_state],
593
+ outputs=[chat_history_state, chat_box],
594
+ ).then(lambda: "", inputs=None, outputs=[chat_input])
595
+
596
+ chat_input.submit(
597
+ get_rag_chat_response,
598
+ inputs=[chat_input, chat_history_state, caption_out, rag_state],
599
+ outputs=[chat_history_state, chat_box],
600
+ ).then(lambda: "", inputs=None, outputs=[chat_input])
601
+
602
+ def show_advisor():
603
+ return {
604
+ landing_group: gr.update(visible=False),
605
+ advisor_group: gr.update(visible=True),
606
+ }
607
+
608
+ def show_landing():
609
+ return {
610
+ landing_group: gr.update(visible=True),
611
+ advisor_group: gr.update(visible=False),
612
+ }
613
+
614
+ go_to_advisor_btn.click(
615
+ show_advisor,
616
+ outputs=[landing_group, advisor_group],
617
+ )
618
+
619
+ back_to_home_btn.click(
620
+ show_landing,
621
+ outputs=[landing_group, advisor_group],
622
+ )
623
+
624
+ if __name__ == "__main__":
625
+ print("Starting app with Landing + Chat‑O‑Berry Advisor sections.")
626
+ demo.launch()