Update app.py to use VLM model moondream

app.py

@@ -1,7 +1,8 @@
 import gradio as gr
 import pandas as pd
 from pathlib import Path
-from transformers import pipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from PIL import Image
 
 # =====================================================================
 # 💾 STATE PERSISTENCE ENGINE (Local File System Checkpoints)
@@ -39,71 +40,47 @@ def save_state_to_disk(dataframe_rows):
     except Exception as e:
         print(f"⚠️ Critical storage write failure: {e}")
 
-# =====================================================================
-# 🪶 LIGHTWEIGHT COMPUTATIONAL ENGINE (1B Model CPU Fallback)
-# =====================================================================
-# This initializes a lightweight TinyLlama-1.1B pipeline.
-# It runs lightning fast directly on Hugging Face's FREE CPU tier.
-print("🤖 Initializing lightweight 1B Edge-native Telemetry Model...")
-generator = pipeline("text-generation", model="TinyLlama/TinyLlama-1.1B-Chat-v1.0", device=-1) # device=-1 forces CPU execution
+print("🤖 Initializing Edge-native Vision-Telemetry Model...")
+model_id = "vikhyatk/moondream2"
+revision = "2024-05-20" # Use a stable release
+model = AutoModelForCausalLM.from_pretrained(model_id, revision=revision, trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision)
 
 def process_kitchen_operations(input_image, budget, days):
-    """
-    Fixed Version: Combines valid kitchen data mapping with the TinyLlama
-    narrative engine so the dataframe state does not reset to placeholders.
-    """
-    # 1. Capture the image telemetry log safely
-    if input_image is not None:
-        width, height = input_image.size
-        telemetry_log = f"Survey Scan Resolution: {width}x{height} pixels"
-    else:
-        telemetry_log = "Using baseline telemetry profile"
+    if input_image is None:
+        # Fallback to your baseline if no image is uploaded
+        df_rows = load_persisted_state()
+        blueprint_md = "No telemetry image provided."
+        return df_rows, blueprint_md
 
-    # 2. KEEP YOUR REAL INGREDIENTS ACTIVE (Prevents resetting to milk/butter/sugar)
-    detected_assets = [
-        {"ingredient": "Greek Yogurt", "qty": "0.5 Liters", "status": "Fresh", "est_cost_usd": 2.50},
-        {"ingredient": "Baby Spinach", "qty": "1 Bag", "status": "Wilting (Critical)", "est_cost_usd": 3.00},
-        {"ingredient": "Chicken Breast", "qty": "400 Grams", "status": "Fresh", "est_cost_usd": 6.50},
-        {"ingredient": "Yellow Onion", "qty": "1 Whole", "status": "Fresh", "est_cost_usd": 0.80}
-    ]
+    # Ensure image is in RGB format
+    image = input_image.convert("RGB")
     
-    df_rows = [
-        [item['ingredient'], item['qty'], item['status'], f"${item['est_cost_usd']:.2f}"] 
-        for item in detected_assets
-    ]
+    # 1. Ask the Vision Model to identify the items
+    prompt = "List the main fresh ingredients visible in this refrigerator with approximate quantities, separated by commas."
+    detected_text = model.answer_question(image, prompt, tokenizer)
     
-    # Commit your clean ingredients permanently to disk!
+    # Example raw output: "Salmon fillets (2 pieces), Broccoli (1 head), Mushrooms, Lettuce"
+    print(f"Telemetried Inventory: {detected_text}")
+    
+    # 2. Parse the text dynamically into DataFrame Rows
+    # (You can split the string and map defaults for status and cost)
+    df_rows = []
+    items = detected_text.split(",")
+    for item in items:
+        clean_item = item.strip()
+        if clean_item:
+            # Dynamically push parsed edge data into the twin matrix
+            df_rows.append([clean_item, "1 Unit", "Nominal", "$3.50"])
+            
+    # Save the real detected state to disk
     save_state_to_disk(df_rows)
     
-    # 3. Calculate financial data
+    # 3. Pass this data to your recipe generation string...
     daily_budget_allowance = budget / days
+    blueprint_md = f"Plan optimized for identified assets: {detected_text}."
     
-    # 4. Prompt TinyLlama to generate the custom culinary routing plan
-    prompt = (
-        f"<|system|>\nYou are the Parallel Plate routing assistant. Based on the scanned fridge items, "
-        f"generate a recipe routing plan. Keep it practical, professional, and limited to 2 short sentences.\n"
-        f"<|user|>\nGenerate a plan for {days} days with a budget of ${daily_budget_allowance:.2f}/day. "
-        f"Telemetry: {telemetry_log}.\n<|assistant|>\n"
-    )
-    
-    try:
-        model_output = generator(prompt, max_new_tokens=120, do_sample=True, temperature=0.6, top_k=50)
-        raw_text = model_output[0]['generated_text']
-        blueprint_md = raw_text.split("<|assistant|>\n")[-1].strip()
-    except Exception as e:
-        # High-quality fallback text just in case the CPU pipeline takes too long during the live demo
-        blueprint_md = (
-            f"**Day 1 (Asset Recovery):** Creamy Chicken & Greens Skillet. *Mitigates waste risk with $0.00 added operational cost.*\n"
-            f"**Day 2-3 (Scarcity Allocation):** Marinated chicken bites with caramelized onions. *Hyper-low resource allocation mode engaged.*"
-        )
-
-    # Add the final formatting wrapper
-    final_output_md = f"### 🛠️ Logistics Routing Strategy Generated via TinyLlama-1.1B\n\n"
-    final_output_md += f"**Operational Horizon:** {days} Days | **Daily Financial Runway Ceiling:** ${daily_budget_allowance:.2f}/day\n\n"
-    final_output_md += f"📋 **Local Cook Execution Pathway:**\n{blueprint_md}\n\n"
-    final_output_md += f"💾 *Status: Data persistence synchronized via {telemetry_log}.*"
-    
-    return df_rows, final_output_md
+    return df_rows, blueprint_md
 
 # =====================================================================
 # 🎨 GRADIO DASHBOARD INTERFACE