nutrilens / app.py
vicarioush's picture
Initial NutriLens submission
8e54894
Raw
History Blame Contribute Delete
21.8 kB
"""
NutriLens - Food Health Impact Analyzer
Gradio Build Small Hackathon (June 2026)
"""
import os
import json
import re
import base64
import io
import time
import gradio as gr
from PIL import Image
from dotenv import load_dotenv
from huggingface_hub import InferenceClient
load_dotenv()
from src.nutrition import lookup_ingredients
from src.literature import lookup_literature, format_citation
from src.prompts import IDENTIFY_PROMPT, build_analysis_prompt, HEALTH_GOALS, AUDIENCES
# ---- Configuration ----
MODEL_ID = os.environ.get("MODEL_ID", "Qwen/Qwen3.6-27B")
API_BASE = os.environ.get("API_BASE", None)
HF_TOKEN = os.environ.get("HF_TOKEN", None)
client = InferenceClient(
model=API_BASE or MODEL_ID,
token=HF_TOKEN,
timeout=300,
)
# ---- Custom CSS ----
CUSTOM_CSS = """
.nutrilens-report h2 {
color: #2d8659;
border-bottom: 2px solid #2d8659;
padding-bottom: 6px;
margin-top: 24px;
}
.nutrilens-report h3 {
color: #5b6abf;
margin-top: 20px;
}
.summary-card {
background: linear-gradient(135deg, #e8f5e9 0%, #e3f2fd 100%);
border-left: 4px solid #2d8659;
border-radius: 8px;
padding: 16px 20px;
margin: 12px 0;
color: #1a3a2a;
}
.dark .summary-card {
background: linear-gradient(135deg, #1b3a2a 0%, #1a2a3a 100%);
color: #c8e6c9;
}
.tip-card {
background: #fff8e1;
border-left: 4px solid #f9a825;
border-radius: 8px;
padding: 16px 20px;
margin: 12px 0;
color: #4a3800;
}
.dark .tip-card {
background: #2a2510;
color: #ffe082;
}
.watch-out-label {
color: #c62828;
}
.dark .watch-out-label {
color: #ff6b6b;
}
.disclaimer-box {
background: #fce4ec;
border-left: 4px solid #e53935;
border-radius: 8px;
padding: 12px 16px;
margin: 16px 0;
color: #4a0e0e;
font-size: 0.9em;
}
.dark .disclaimer-box {
background: #2a1010;
color: #ef9a9a;
}
"""
def call_model(messages: list, max_tokens: int = 1024, retries: int = 2,
extract_answer: bool = True, markers: tuple = None) -> str:
"""Call model with timeout handling and retry.
Set extract_answer=False for ingredient ID (has its own parser).
`markers`, if given, is a (start, end) pair of sentinel lines the
prompt asked the model to wrap its final answer in - tried before
any heuristic extraction since it's deterministic."""
for attempt in range(retries + 1):
try:
response = client.chat_completion(
model=MODEL_ID if not API_BASE else None,
messages=messages,
max_tokens=max_tokens,
temperature=0.3,
)
msg = response.choices[0].message
content = msg.content
reasoning = getattr(msg, "reasoning", None) or ""
# The model may put its final answer in `content`, in
# `reasoning`, or wrap it with the sentinel markers in either -
# whichever field actually has text is the one to look at first.
text = content if content is not None else reasoning
if not text:
return "The model returned an empty response. Please try again."
between = _extract_between_markers(text, *markers) if markers else None
if between is not None:
content = between
elif content is None:
content = _extract_answer_from_reasoning(reasoning) if extract_answer else reasoning
# else: keep msg.content as-is - it's real content with no markers needed
content = re.sub(r"<think>.*?</think>", "", content, flags=re.DOTALL).strip()
return content if content else "Model returned empty content."
except Exception as e:
error_str = str(e)
if ("504" in error_str or "timeout" in error_str.lower()) and attempt < retries:
wait = 3 * (attempt + 1)
print(f"Timeout (attempt {attempt+1}/{retries+1}), retrying in {wait}s...")
time.sleep(wait)
continue
print(f"Model call error: {e}")
if "504" in error_str:
return ("The model server timed out. This usually happens with long "
"ingredient lists. Try with fewer ingredients (5-8 at a time).")
return f"Error calling model: {e}"
return "All retries failed. Please try again later."
def _extract_between_markers(text: str, start: str, end: str) -> str | None:
"""Return the text between two sentinel marker lines, or None if no
substantial match is found. The prompt asks the model to wrap its
final answer in these markers - but while thinking, the model often
also *mentions* the marker format (e.g. "wrap the answer between
@@@REPORT_START@@@ and @@@REPORT_END@@@"), which produces a tiny,
bogus match. The real final answer is always much longer than any
incidental mention, so among all matches we take the longest one
that clears a minimum length."""
pattern = re.escape(start) + r"\s*(.*?)\s*" + re.escape(end)
matches = [m.strip() for m in re.findall(pattern, text, re.DOTALL)]
matches = [m for m in matches if len(m) > 60]
if matches:
return max(matches, key=len)
# The model can also get cut off mid-answer (hits max_tokens before
# emitting the end marker). In that case there's no complete pair, but
# the last start-marker occurrence is still where the real answer
# begins - take everything after it rather than leaking the raw
# "@@@REPORT_START@@@" line to the user.
starts = [m.end() for m in re.finditer(re.escape(start), text)]
if starts:
tail = text[starts[-1]:].strip()
if len(tail) > 200:
return tail
return None
def _extract_answer_from_reasoning(reasoning: str) -> str:
"""
When Qwen3.6 thinks, the reasoning field contains both the
internal chain-of-thought AND the final formatted answer.
This function extracts just the answer.
"""
# Look for markdown headings at the START of a line (not inline mentions).
# The model's self-checks mention headings inline like:
# 'Is "## What's on your plate" present? Yes.'
# But the actual answer has them at line start:
# '\n## What's on your plate\n'
markers = [
r"\n## What.s on your plate",
r"\n## What.s on Your Plate",
r"\n## Overall Meal",
r"\n## Overall Assessment",
r"\n## Summary",
]
for pattern in markers:
matches = list(re.finditer(pattern, reasoning, re.IGNORECASE))
if matches:
# Use the LAST match that's at a line start
idx = matches[-1].start() + 1 # +1 to skip the \n
answer = reasoning[idx:]
# Trim trailing thinking artifacts
for end_marker in ["✅", "[Done]", "[Output Generation]",
"Self-Correction", "Output matches"]:
end_idx = answer.rfind(end_marker)
if end_idx > 0 and end_idx > len(answer) * 0.6:
answer = answer[:end_idx].strip()
if len(answer) > 50:
return answer.strip()
# Fallback: look for the last block of markdown-formatted text
# by finding consecutive lines starting with ## or ### or - or *
lines = reasoning.split('\n')
best_start = None
for i, line in enumerate(lines):
if line.strip().startswith('## ') and not '`' in line and '?' not in line:
# This looks like a real heading, not a self-check
if best_start is None:
best_start = i
if best_start is not None:
answer = '\n'.join(lines[best_start:])
if len(answer) > 50:
return answer.strip()
# For ingredient identification: try to find JSON array
# Look for the largest JSON array (not tiny ones like [1])
json_matches = re.findall(r'\[("[^"]+?"(?:\s*,\s*"[^"]+?")*)\]', reasoning)
if json_matches:
longest = max(json_matches, key=len)
return f'[{longest}]'
# Last resort: return the last 30% of reasoning
cutoff = int(len(reasoning) * 0.7)
return reasoning[cutoff:].strip()
def image_to_data_url(img: Image.Image) -> str:
buf = io.BytesIO()
if img.mode == "RGBA":
img = img.convert("RGB")
max_dim = 1024
if max(img.size) > max_dim:
img.thumbnail((max_dim, max_dim), Image.LANCZOS)
img.save(buf, format="JPEG", quality=80)
b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
return f"data:image/jpeg;base64,{b64}"
def extract_ingredients_from_text(text: str) -> list[str]:
NOISE = {
"zutaten", "ingredients", "ingredienten", "ingrédients", "sastojci",
"contains", "kann auch", "may contain", "enthält", "contient",
"allergens", "allergenen", "nutrition", "nährwerte",
"analyze", "image", "ingredient", "label", "user", "step", "json",
"the", "and", "oder", "und", "et", "i", "a", "an",
}
# Phrases that only show up when the model is restating its own
# instructions (while thinking) rather than naming a food - reject
# any item that contains one of these, regardless of language/case.
INSTRUCTION_PHRASES = (
"actual food", "list only", "json array", "final answer",
"marker", "format", "translate", "do not include", "do not repeat",
)
def is_food(item: str) -> bool:
item_lower = item.lower().strip()
if len(item_lower) < 2 or len(item_lower) > 80:
return False
if item_lower in NOISE:
return False
if any(item_lower.startswith(n) for n in ["zutaten", "may contain", "kann auch"]):
return False
if any(p in item_lower for p in INSTRUCTION_PHRASES):
return False
return True
# Try every bracketed array in the text (the model may mention an
# example array while thinking before producing the real, final one)
# and keep whichever yields the most valid food items - the genuine
# final list is reliably the longest, most complete one.
best = []
for candidate in re.findall(r'\[.*?\]', text, re.DOTALL):
try:
items = json.loads(candidate)
except json.JSONDecodeError:
continue
if not isinstance(items, list):
continue
foods = [str(i).strip() for i in items if is_food(str(i))]
if len(foods) > len(best):
best = foods
if best:
return list(dict.fromkeys(best))
quoted = re.findall(r'"([^"]+)"', text)
if len(quoted) >= 2:
foods = [q for q in quoted if is_food(q)]
if foods:
return list(dict.fromkeys(foods))
arrow_matches = re.findall(r'->\s*([a-zA-Z][a-zA-Z\s,]+?)(?:\n|$)', text)
if arrow_matches:
foods = [m.strip().rstrip(',').strip() for m in arrow_matches if is_food(m.strip())]
if foods:
return list(dict.fromkeys(foods))
parts = [p.strip() for p in text.split(',') if p.strip()]
foods = [p for p in parts if is_food(p)]
if foods:
return list(dict.fromkeys(foods))[:20]
return [text[:100]]
def format_report(raw_report: str, nutrition_fails: int, lit_fails: int,
total_ingredients: int) -> str:
"""Post-process the model's markdown into styled HTML."""
report = raw_report
# The model doesn't always put a line break before "**Watch out:**" -
# it sometimes lands mid-sentence on the same line as the last "Good
# stuff" bullet, which makes Markdown swallow it into that list item.
# Force it onto its own paragraph and color it so it stands out.
report = re.sub(
r"\s*\*\*Watch out:\*\*",
'\n\n<strong class="watch-out-label">Watch out:</strong>',
report,
)
# Cut the tips section out first (wherever the model placed it) so we
# can re-insert it right before the ingredient breakdown instead of at
# the end - the user wants tips to appear up front, near the summary.
tips_match = re.search(
r"##\s*Tips?\s*\n(.*?)(?=\n##|\Z)",
report, re.DOTALL | re.IGNORECASE
)
tips_html = ""
if tips_match:
tips_text = tips_match.group(1).strip()
tips_html = (f"## Tips\n\n"
f'<div class="tip-card">\n\n{tips_text}\n\n</div>\n\n')
report = report[:tips_match.start()] + report[tips_match.end():]
# Wrap the summary section in a card
summary_match = re.search(
r"(##\s*What.s on your plate\s*\n)(.*?)(?=\n##|\n###|\Z)",
report, re.DOTALL | re.IGNORECASE
)
if summary_match:
summary_text = summary_match.group(2).strip()
styled = (f"## What's on your plate\n\n"
f'<div class="summary-card">\n\n{summary_text}\n\n</div>\n\n')
report = report[:summary_match.start()] + styled + report[summary_match.end():]
# Re-insert tips right before the first ingredient heading (### ...),
# which immediately follows the summary card.
if tips_html:
first_heading = re.search(r"\n###\s", report)
if first_heading:
insert_at = first_heading.start() + 1
report = report[:insert_at] + tips_html + "\n" + report[insert_at:]
else:
report += "\n\n" + tips_html
# Add disclaimer card
disclaimer = (
'<div class="disclaimer-box">'
'⚠️ <strong>This is not medical advice.</strong> '
'Always talk to a doctor or nutritionist before changing your diet, '
'especially if you have health conditions, allergies, or take medication.'
'</div>'
)
# Source info
source = "\n\n---\n*Data: "
if nutrition_fails == 0:
source += "USDA FoodData Central"
elif nutrition_fails < total_ingredients:
source += "USDA (partial)"
else:
source += "Model knowledge"
source += " + PubMed"
if lit_fails > 0:
source += " (partial)"
source += f" | Model: {MODEL_ID}*"
return report + "\n\n" + disclaimer + source
def identify_ingredients(image, text_input):
if image is not None:
gr.Info("Reading image with AI... this can take 15-30 seconds.")
data_url = image_to_data_url(image)
messages = [{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{"type": "text", "text": IDENTIFY_PROMPT},
],
}]
# Generous budget: with thinking mode on, the model works through
# the label (translating, deduplicating) before writing the final
# marker-wrapped array - too small a cap truncates it mid-thought,
# leaving only messy draft arrays (mixed German/English) behind.
raw = call_model(
messages, max_tokens=3000, extract_answer=False,
markers=("@@@INGREDIENTS_START@@@", "@@@INGREDIENTS_END@@@"),
)
ingredients = extract_ingredients_from_text(raw)
gr.Info(f"Found {len(ingredients)} ingredients. Review and edit if needed.")
return ", ".join(ingredients)
elif text_input and text_input.strip():
items = [i.strip() for i in text_input.replace("\n", ",").split(",") if i.strip()]
return ", ".join(items)
return ""
def run_analysis(ingredients_text, health_goal, audience, progress=gr.Progress()):
if not ingredients_text or not ingredients_text.strip():
return "Please identify ingredients first.", ""
ingredients = [i.strip() for i in ingredients_text.split(",") if i.strip()]
if not ingredients:
return "No ingredients to analyze.", ""
# Cap tokens based on ingredient count to avoid timeouts.
# Thinking + a sentinel-wrapped final answer use more tokens than a
# bare answer, so the budget is a bit larger than before.
# The model spends a fairly fixed chunk of its budget on thinking
# regardless of list length, so short lists need a floor too - without
# it, thinking alone can exhaust the budget and truncate the report.
max_tok = min(12000, max(6000, 2000 + len(ingredients) * 600))
try:
progress(0.05, desc="Looking up nutritional data...")
nutrition_data, nutrition_fails = lookup_ingredients(ingredients)
progress(0.35, desc="Searching scientific literature...")
goal_key = health_goal if health_goal in HEALTH_GOALS else "General"
lit_data, lit_fails = lookup_literature(
ingredients, health_goal=goal_key, papers_per=2
)
progress(0.6, desc="Generating health report... this can take 30-90 seconds.")
prompt = build_analysis_prompt(
nutrition_data, lit_data, health_goal=goal_key,
audience=audience,
nutrition_failures=nutrition_fails,
literature_failures=lit_fails,
)
raw_report = call_model(
[{"role": "user", "content": prompt}],
max_tokens=max_tok,
markers=("@@@REPORT_START@@@", "@@@REPORT_END@@@"),
)
progress(0.95, desc="Formatting report...")
report = format_report(raw_report, nutrition_fails, lit_fails, len(ingredients))
# Citations
all_citations = []
for papers in lit_data.values():
for p in papers:
c = format_citation(p)
if c not in all_citations:
all_citations.append(c)
citations = ""
if all_citations:
citations = "**References:**\n\n"
for i, c in enumerate(all_citations, 1):
citations += f"{i}. {c}\n\n"
return report, citations
except Exception as e:
import traceback
traceback.print_exc()
return f"Something went wrong: {e}", ""
def _start_identify_loading():
return gr.update(value="⏳ Reading...", interactive=False)
def _stop_identify_loading():
return gr.update(value="1. Identify ingredients", interactive=True)
def _start_analyze_loading():
placeholder = (
"_Generating your health report - looking up nutrition data, "
"searching scientific literature, and writing up the analysis. "
"This can take 30-90 seconds..._"
)
return (
gr.update(value="⏳ Analyzing... (30-90s)", interactive=False),
placeholder,
"",
)
def _stop_analyze_loading():
return gr.update(value="2. Analyze health impact", interactive=True)
# ---- Gradio UI ----
with gr.Blocks(
title="NutriLens",
theme=gr.themes.Soft(
primary_hue="green",
secondary_hue="blue",
),
css=CUSTOM_CSS,
) as demo:
gr.Markdown("""
# 🔬 NutriLens
**Upload a food photo or type ingredients, then get a clear health
breakdown backed by real data and scientific research.**
Works with food labels in any language.
""")
with gr.Row():
with gr.Column(scale=1):
image_input = gr.Image(
label="Food photo or ingredient label",
type="pil",
sources=["upload", "webcam", "clipboard"],
)
text_input = gr.Textbox(
label="Or type ingredients (comma-separated)",
placeholder="chicken breast, brown rice, broccoli, olive oil",
lines=2,
)
identify_btn = gr.Button(
"1. Identify ingredients", variant="secondary", size="lg",
)
with gr.Column(scale=2):
ingredients_box = gr.Textbox(
label="Identified ingredients (review and edit before analyzing)",
placeholder="Ingredients will appear here. Edit them if needed, then click Analyze.",
lines=2,
interactive=True,
)
with gr.Row():
health_goal = gr.Dropdown(
label="Health focus",
choices=list(HEALTH_GOALS.keys()),
value="General",
scale=2,
)
audience = gr.Radio(
label="Explanation level",
choices=list(AUDIENCES.keys()),
value="Everyone",
scale=2,
)
analyze_btn = gr.Button(
"2. Analyze health impact", variant="primary", size="lg",
)
report_out = gr.Markdown(
label="Health report",
elem_classes=["nutrilens-report"],
)
citations_out = gr.Markdown(label="References")
identify_btn.click(
fn=_start_identify_loading,
outputs=[identify_btn],
).then(
fn=identify_ingredients,
inputs=[image_input, text_input],
outputs=[ingredients_box],
).then(
fn=_stop_identify_loading,
outputs=[identify_btn],
)
analyze_btn.click(
fn=_start_analyze_loading,
outputs=[analyze_btn, report_out, citations_out],
).then(
fn=run_analysis,
inputs=[ingredients_box, health_goal, audience],
outputs=[report_out, citations_out],
).then(
fn=_stop_analyze_loading,
outputs=[analyze_btn],
)
gr.Markdown("""
---
⚠️ **NutriLens is not a substitute for professional medical advice.**
Always consult a doctor or registered nutritionist before making dietary
changes, especially if you have health conditions, allergies, or take
medication.
*Data: USDA FoodData Central + PubMed | Model: ≤32B params |
Built for the [Build Small Hackathon](https://huggingface.co/build-small-hackathon)*
""")
if __name__ == "__main__":
demo.launch()