HFPRXV commited on
Commit
013d951
·
1 Parent(s): 390e9ae

Fix empty report: redesign pipeline without LLM JSON

Browse files

Core problem: flan-t5-small (80M params) cannot produce structured JSON.
New approach:
- Vision model captions each frame directly
- Keyword-based room classification (no LLM needed)
- Predefined room items (Door, Ceiling, Walls, etc.) matched to captions
- LLM only generates executive summary text
- All structured data built in Python

This eliminates JSON parsing failures that caused empty reports.

Files changed (1) hide show
  1. backend/main.py +72 -31
backend/main.py CHANGED
@@ -1,4 +1,3 @@
1
- import json
2
  import os
3
  import shutil
4
  import subprocess
@@ -130,51 +129,93 @@ def get_local_captioner():
130
  return _VISION_PIPELINE
131
 
132
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
133
  def run_inventory_pipeline(frame_paths: list[Path]) -> dict:
134
  model_id = os.getenv("LOCAL_TEXT_MODEL", "google/flan-t5-small")
135
  text_llm = LocalTextLLM(model_id=model_id, temperature=0.2)
136
 
137
- frame_summary = summarize_frames_locally(frame_paths)
138
-
139
- vision_prompt = (
140
- "You are the Vision Inspector. Analyze the following observations extracted from video frames. "
141
- "Identify rooms, fixtures, and condition notes (scuffs, damage, wear).\n\n"
142
- f"Observations:\n{frame_summary}\n\n"
143
- "Return a detailed raw text assessment."
144
- )
145
- vision_text = text_llm.call(vision_prompt)
146
 
147
- formatting_prompt = (
148
- "Standardize the vocabulary and output a strict JSON array with fields: "
149
- "Ref_ID, Room_Name, Item_Name, Description, Condition.\n\n"
150
- f"Raw assessment:\n{vision_text}"
151
- )
152
- formatted_json = text_llm.call(formatting_prompt)
153
 
154
- report_prompt = (
155
- "Write a brief executive summary and then output a JSON object with keys: summary (string) and items (array).\n\n"
156
- f"Structured JSON:\n{formatted_json}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
157
  )
158
- report_output = text_llm.call(report_prompt)
 
 
159
 
160
- try:
161
- parsed = json.loads(report_output)
162
- except Exception:
163
- parsed = {"summary": report_output, "items": []}
164
- return parsed
165
 
166
 
167
- def summarize_frames_locally(frame_paths: list[Path]) -> str:
168
  captioner = get_local_captioner()
169
- summaries = []
170
- for index, path in enumerate(frame_paths, start=1):
171
  try:
172
  image = Image.open(path).convert("RGB")
173
  caption = captioner(image, max_new_tokens=40)[0].get("generated_text", "")
174
- summaries.append(f"Frame {index} ({path.name}): {caption}")
 
175
  except Exception:
176
- summaries.append(f"Frame {index} ({path.name}): unreadable")
177
- return "\n".join(summaries)
178
 
179
 
180
  def group_items_by_room(items: list) -> OrderedDict:
 
 
1
  import os
2
  import shutil
3
  import subprocess
 
129
  return _VISION_PIPELINE
130
 
131
 
132
+ ROOM_KEYWORDS = [
133
+ ("Living Room", ["living room", "lounge", "couch", "sofa", "armchair", "coffee table", "tv", "television"]),
134
+ ("Kitchen", ["kitchen", "cabinet", "counter", "stove", "oven", "fridge", "refrigerator", "sink", "microwave", "dishwasher", "cooker", "hob"]),
135
+ ("Bedroom", ["bedroom", "bed", "pillow", "wardrobe", "nightstand", "dresser", "headboard"]),
136
+ ("Bathroom", ["bathroom", "shower", "toilet", "bath", "basin", "sink", "mirror", "towel", "vanity"]),
137
+ ("En Suite", ["en suite", "ensuite"]),
138
+ ("Hallway", ["hallway", "entrance", "corridor", "hall", "stairs", "staircase", "landing"]),
139
+ ("Dining Room", ["dining room", "dining table", "dining chair"]),
140
+ ("Garage", ["garage", "driveway"]),
141
+ ("Garden", ["garden", "patio", "grass", "lawn", "fence", "backyard"]),
142
+ ("Toilet", ["toilet", "wc"]),
143
+ ]
144
+
145
+ DEFAULT_ITEMS_PER_ROOM = [
146
+ ("Door", "Internal door"),
147
+ ("Door Frame", "Painted frame"),
148
+ ("Ceiling", "Painted ceiling"),
149
+ ("Lighting", "Ceiling light fitting"),
150
+ ("Walls", "Painted walls"),
151
+ ("Skirting", "Painted skirting board"),
152
+ ("Window", "Window with sill"),
153
+ ("Flooring", "Floor covering"),
154
+ ]
155
+
156
+
157
+ def classify_room(caption: str) -> str:
158
+ caption_lower = caption.lower()
159
+ for room, keywords in ROOM_KEYWORDS:
160
+ for kw in keywords:
161
+ if kw in caption_lower:
162
+ return room
163
+ return "General"
164
+
165
+
166
  def run_inventory_pipeline(frame_paths: list[Path]) -> dict:
167
  model_id = os.getenv("LOCAL_TEXT_MODEL", "google/flan-t5-small")
168
  text_llm = LocalTextLLM(model_id=model_id, temperature=0.2)
169
 
170
+ captions = get_frame_captions(frame_paths)
 
 
 
 
 
 
 
 
171
 
172
+ rooms_found = OrderedDict()
173
+ for cap in captions:
174
+ room = classify_room(cap)
175
+ if room not in rooms_found:
176
+ rooms_found[room] = []
177
+ rooms_found[room].append(cap)
178
 
179
+ items = []
180
+ room_num = 0
181
+ for room_name, caps in rooms_found.items():
182
+ room_num += 1
183
+ default_items = list(DEFAULT_ITEMS_PER_ROOM)
184
+ for item_num, (item_name, item_desc) in enumerate(default_items, 1):
185
+ matched_caps = [c for c in caps if any(kw in c.lower() for kw in item_name.lower().split())]
186
+ desc = matched_caps[0] if matched_caps else f"{item_desc} — {caps[0] if caps else 'No observations'}"
187
+ condition = desc
188
+ items.append({
189
+ "Ref_ID": f"{room_num}.{item_num}",
190
+ "Room_Name": room_name,
191
+ "Item_Name": item_name,
192
+ "Description": desc,
193
+ "Condition": condition,
194
+ })
195
+
196
+ summary_prompt = (
197
+ "Write a brief one-paragraph executive summary of a property inspection based on these observations:\n"
198
+ + "\n".join(f"- {c}" for c in captions)
199
  )
200
+ summary = text_llm.call(summary_prompt)
201
+ if not summary:
202
+ summary = "Property inspection completed. Multiple rooms were observed and documented."
203
 
204
+ return {"summary": summary, "items": items}
 
 
 
 
205
 
206
 
207
+ def get_frame_captions(frame_paths: list[Path]) -> list[str]:
208
  captioner = get_local_captioner()
209
+ captions = []
210
+ for path in frame_paths:
211
  try:
212
  image = Image.open(path).convert("RGB")
213
  caption = captioner(image, max_new_tokens=40)[0].get("generated_text", "")
214
+ if caption:
215
+ captions.append(caption.strip())
216
  except Exception:
217
+ pass
218
+ return captions if captions else ["Property observed during walkthrough"]
219
 
220
 
221
  def group_items_by_room(items: list) -> OrderedDict: