app.py

import os

import pandas as pd
import streamlit as st

from pipeline import run

st.set_page_config(page_title="Concord Data Agents | Legacy ETL", page_icon="🧬", layout="wide")

# ---------------------------------------------------------------------------
# Theme: teal / deep blue-green, clinical, calm -- healthcare/insurance feel
# ---------------------------------------------------------------------------
st.markdown(
    """
    <style>
    :root {
        --cd-teal: #0B4F4A;
        --cd-bg: #0D2B27;
        --cd-panel: #123B36;
        --cd-mint: #4FD1C5;
        --cd-grey: #9FC7C0;
    }
    .stApp { background-color: var(--cd-bg); color: #E6F5F2; }
    section[data-testid="stSidebar"] { background-color: var(--cd-panel); }
    h1, h2, h3 { font-family: -apple-system, 'Helvetica Neue', sans-serif; letter-spacing: -0.01em; }
    .cd-hero { font-size: 2.0rem; font-weight: 700; margin-bottom: 0; color: var(--cd-mint); }
    .cd-sub { color: var(--cd-grey); font-size: 0.95rem; margin-top: 0.2rem; }
    .cd-card {
        background: var(--cd-panel); border: 1px solid #1D544D; border-radius: 12px;
        padding: 14px 16px; margin-bottom: 10px;
    }
    div[data-testid="stMetricValue"] { color: var(--cd-mint); }
    </style>
    """,
    unsafe_allow_html=True,
)

st.markdown('<div class="cd-hero">Concord Data Agents</div>', unsafe_allow_html=True)
st.markdown(
    '<div class="cd-sub">A two-agent pipeline that scans undocumented legacy tables, maps them to a '
    'clean schema, and auto-generates the cleaning rules — no manual data dictionary required.</div>',
    unsafe_allow_html=True,
)
st.write("")

with st.sidebar:
    st.markdown("### 🧬 Pipeline")
    st.caption(
        "Agent 1 scans the undocumented legacy table and infers a schema. "
        "Agent 2 generates and executes per-column cleaning rules. A quality "
        "dashboard scores the data before and after."
    )
    csv_path = os.path.join(os.path.dirname(__file__), "data", "legacy_claims_raw.csv")
    st.caption(f"Dataset: `data/legacy_claims_raw.csv` (synthetic, {sum(1 for _ in open(csv_path)) - 1} rows)")
    run_btn = st.button("▶ Run pipeline", type="primary")

if "pipeline_out" not in st.session_state:
    st.session_state.pipeline_out = None

if run_btn:
    st.session_state.pipeline_out = run(csv_path)

out = st.session_state.pipeline_out

if out is None:
    st.info("Click **Run pipeline** in the sidebar to scan and clean the synthetic legacy dataset.")
    st.markdown("#### Raw legacy data preview")
    st.dataframe(pd.read_csv(csv_path, dtype=str, keep_default_na=False).head(15), use_container_width=True)
else:
    tab1, tab2, tab3, tab4 = st.tabs([
        "1. Agent 1 — Schema Scan", "2. Agent 2 — Cleaning Log", "3. Quality Dashboard", "4. Before / After Data"
    ])

    with tab1:
        st.markdown("Agent 1 scanned each undocumented column and inferred its semantic meaning, type, and issues.")
        rows = []
        for p in out.profiles:
            rows.append({
                "Raw column": p.raw_name,
                "Inferred field": p.canonical_name,
                "Type": p.inferred_type,
                "Distinct raw values": p.n_distinct,
                "Missing/sentinel count": p.n_missing_like,
                "Issues detected": "; ".join(p.issues) if p.issues else "none",
            })
        st.dataframe(pd.DataFrame(rows), use_container_width=True, hide_index=True)

    with tab2:
        st.markdown("Agent 2 generated and executed a cleaning rule per column based on Agent 1's schema map.")
        for entry in out.cleaning_log:
            st.markdown(
                f'<div class="cd-card"><b>{entry.canonical_name}</b> '
                f'<span style="color:var(--cd-grey); font-size:0.85rem;">(from {entry.column})</span><br>'
                f'{entry.action}</div>',
                unsafe_allow_html=True,
            )

    with tab3:
        st.markdown("#### Completeness: before vs. after")
        before_map = {q.column: q.completeness_pct for q in out.raw_quality}
        after_by_canonical = {q.column: q.completeness_pct for q in out.cleaned_quality}

        comp_rows = []
        for p in out.profiles:
            comp_rows.append({
                "Field": p.canonical_name,
                "Completeness before (%)": before_map.get(p.raw_name, None),
                "Completeness after (%)": after_by_canonical.get(p.canonical_name, None),
            })
        comp_df = pd.DataFrame(comp_rows).drop_duplicates(subset="Field")
        st.dataframe(comp_df, use_container_width=True, hide_index=True)
        st.bar_chart(comp_df.set_index("Field")[["Completeness before (%)", "Completeness after (%)"]])

        m1, m2, m3 = st.columns(3)
        m1.metric("Rows before", len(out.raw_df))
        m2.metric("Rows after (deduped)", len(out.cleaned_df))
        m3.metric("Duplicate rows removed", len(out.raw_df) - len(out.cleaned_df))

    with tab4:
        c1, c2 = st.columns(2)
        with c1:
            st.markdown("**Raw (legacy) data**")
            st.dataframe(out.raw_df.head(20), use_container_width=True, hide_index=True)
        with c2:
            st.markdown("**Cleaned data**")
            st.dataframe(out.cleaned_df.head(20), use_container_width=True, hide_index=True)