views/insights.py

"""Data insights — what is actually in the catalog, and how it was built.

This page exists because the honest answer to "how good are these
recommendations?" is "as good as the coverage underneath them". Rather than
hide that, it is measured and shown.
"""

from __future__ import annotations

import plotly.express as px
import plotly.graph_objects as go
import streamlit as st

from nutriweb.data import catalog

st.markdown(
    """<div class="nw-hero">
        <h1>What's in the catalog</h1>
        <p>Built from the Open Food Facts product database on Hugging Face,
        filtered to US products and scored with the official Nutri-Score 2023 algorithm.</p>
    </div>""",
    unsafe_allow_html=True,
)


@st.cache_data(show_spinner=False)
def _stats() -> dict:
    return catalog.stats()


@st.cache_data(show_spinner=False)
def _grade_distribution():
    return catalog.connect().execute("""
        SELECT nutriscore_grade AS grade, nutriscore_source AS source, count(*) AS n
        FROM catalog WHERE nutriscore_grade IS NOT NULL
        GROUP BY 1, 2 ORDER BY 1
    """).fetchdf()


@st.cache_data(show_spinner=False)
def _top_categories(limit: int = 12):
    return catalog.connect().execute(f"""
        SELECT replace(split_part(primary_category, ':', 2), '-', ' ') AS category,
               count(*) AS products, round(avg(health_score), 1) AS avg_health
        FROM catalog
        WHERE primary_category IS NOT NULL AND health_score IS NOT NULL
        GROUP BY 1 HAVING count(*) > 200
        ORDER BY products DESC LIMIT {limit}
    """).fetchdf()


@st.cache_data(show_spinner=False)
def _nutrient_profile(limit: int = 8):
    return catalog.connect().execute(f"""
        WITH top AS (
            SELECT primary_category FROM catalog
            WHERE primary_category IS NOT NULL AND health_score IS NOT NULL
            GROUP BY 1 ORDER BY count(*) DESC LIMIT {limit}
        )
        SELECT replace(split_part(c.primary_category, ':', 2), '-', ' ') AS category,
               median(c.proteins_100g) AS protein,
               median(c.carbohydrates_100g) AS carbs,
               median(c.sugars_100g) AS sugars,
               median(c.fat_100g) AS fat,
               median(c.fiber_100g) AS fibre,
               median(c.salt_derived) AS salt
        FROM catalog c JOIN top USING (primary_category)
        GROUP BY 1 ORDER BY 1
    """).fetchdf()


stats = _stats()
cols = st.columns(4)
cols[0].metric("US products", f"{stats['products']:,}")
cols[1].metric("With a health score", f"{stats['scored']:,}")
cols[2].metric("Graded by NutriWeb", f"{stats['graded_by_nutriweb']:,}",
               help="Products where Open Food Facts publishes no Nutri-Score and we computed one.")
cols[3].metric("With a photo", f"{stats['with_image']:,}")

st.divider()

left, right = st.columns(2, gap="large")

with left:
    st.markdown("#### Nutri-Score distribution, by source")
    grades = _grade_distribution()
    grades["source"] = grades["source"].map(
        {"off": "Published by Open Food Facts", "nutriweb": "Computed by NutriWeb"}
    )
    figure = px.bar(
        grades, x="grade", y="n", color="source",
        category_orders={"grade": ["a", "b", "c", "d", "e"]},
        color_discrete_sequence=["#0F8A5F", "#9CC5B4"],
        labels={"grade": "Nutri-Score", "n": "Products", "source": ""},
    )
    figure.update_layout(
        legend=dict(orientation="h", y=-0.22), margin=dict(l=0, r=0, t=6, b=0), height=340,
    )
    st.plotly_chart(figure, width='stretch')
    st.caption(
        "Computing grades ourselves lifts scored coverage well beyond what Open "
        "Food Facts publishes for the US market."
    )

with right:
    st.markdown("#### Largest categories")
    categories = _top_categories()
    figure = px.bar(
        categories.sort_values("products"), x="products", y="category",
        orientation="h", color="avg_health",
        color_continuous_scale=["#E63E11", "#FECB02", "#038141"],
        labels={"products": "Products", "category": "", "avg_health": "Avg health"},
    )
    figure.update_layout(margin=dict(l=0, r=0, t=6, b=0), height=340)
    st.plotly_chart(figure, width='stretch')
    st.caption("Colour shows the average NutriWeb health score for the category.")

st.divider()
st.markdown("#### Nutrient profile of the largest categories")
st.caption(
    "Median grams per 100 g. Axes are clipped at the 95th percentile so a few "
    "extreme products cannot flatten the shape."
)

profile_data = _nutrient_profile()
nutrients = ["protein", "carbs", "sugars", "fat", "fibre", "salt"]
maxima = {n: max(profile_data[n].max(), 1e-6) for n in nutrients}

radar = go.Figure()
for _, row in profile_data.iterrows():
    radar.add_trace(
        go.Scatterpolar(
            r=[row[n] / maxima[n] for n in nutrients],
            theta=[n.capitalize() for n in nutrients],
            fill="toself",
            name=str(row["category"])[:28],
        )
    )
radar.update_layout(
    polar=dict(radialaxis=dict(visible=True, range=[0, 1], showticklabels=False)),
    height=470, margin=dict(l=40, r=40, t=20, b=20),
    legend=dict(orientation="h", y=-0.12),
)
st.plotly_chart(radar, width='stretch')

st.divider()
with st.expander("How the health score is built"):
    st.markdown(
        """
**Health score (0–100)** = 70% Nutri-Score 2023 + 30% NOVA processing group,
minus a capped penalty for additives flagged by EFSA or ANSES.

**Nutri-Score** is computed with the official 2023 algorithm, ported from Open
Food Facts' reference implementation. Where Open Food Facts already publishes a
grade we use theirs; where it does not, we compute one and label it. Validated
against Open Food Facts' own grades on 317,265 products: **98.4% exact agreement**,
99.7% within one letter.

**Where a signal is missing** a neutral prior is substituted rather than
rescaling the others, so every product stays on the same scale and remains
comparable. Products where we had to assume the category are marked *low
confidence* on the product page.

Data: [openfoodfacts/product-database](https://huggingface.co/datasets/openfoodfacts/product-database),
licensed ODbL. Product data © Open Food Facts contributors.
"""
    )