app.py

# app_simple_plus.py — ASAP Explorer (no table, log-scale map, English-filtered word cloud, top-15 composers)
import re
import pandas as pd
import numpy as np
import plotly.express as px
import plotly.graph_objects as go
from wordcloud import WordCloud, STOPWORDS
from dateutil import parser as dateparser
import gradio as gr
from datasets import load_dataset
import os

COMPOSITION_PATH = "ASAPcomposition.csv"
PEOPLE_PATH = "ASAPdata.csv"
PLOT_TEMPLATE = "plotly_white"
HF_TOKEN = os.environ.get("HF_TOKEN")

# ===== Word cloud English filter =====
MIN_ASCII_RATIO = 0.7  # keep titles whose ASCII-char ratio >= this

def _ascii_ratio(s: str) -> float:
    if not s:
        return 0.0
    total = len(s)
    ascii_count = sum(1 for ch in s if ord(ch) < 128)
    return ascii_count / total if total else 0.0

# =========================
# Robust Year Parsing
# =========================
def _norm_year_text(s: str) -> str:
    s = s.strip()
    s = s.replace("–", "-").replace("—", "-").replace(" to ", "-")
    sl = s.lower()
    sl = re.sub(r"\b(ca|c\.|circa|approx(?:imate(?:ly)?)?|about|around)\b\.?", "", sl)
    sl = sl.replace("?", " ").replace("[", " ").replace("]", " ")
    sl = re.sub(r"\s+", " ", sl).strip()
    return sl

def _parse_year_robust(y):
    if pd.isna(y):
        return (np.nan, np.nan)
    s = _norm_year_text(str(y))
    m = re.search(r"\b(1[5-9]\d{2}|20\d{2})\s*-\s*(\d{2,4})\b", s)
    if m:
        start = int(m.group(1))
        end_str = m.group(2)
        if len(end_str) == 2:
            end = (start // 100) * 100 + int(end_str)
            if end % 100 < start % 100:
                end += 100
        else:
            end = int(end_str)
        return (start, end)
    years = [int(z) for z in re.findall(r"(?<!\d)(1[5-9]\d{2}|20\d{2})(?!\d)", s)]
    if years:
        return (min(years), max(years) if len(years) > 1 else np.nan)
    m = re.search(r"\b(early|mid|late)?\s*(\d{3})0s\b", s)
    if m:
        when = m.group(1) or ""
        base = int(m.group(2)) * 10
        offset = {"early": 0, "mid": 5, "late": 8}.get(when, 0)
        return (base + offset, np.nan)
    m = re.search(r"\b(early|mid|late)?\s*(\d{1,2})(?:st|nd|rd|th)?\s*(?:century|c\.?)\b", s)
    if m:
        when = m.group(1) or "mid"
        cent = int(m.group(2))
        base = (cent - 1) * 100
        offset = {"early": 0, "mid": 50, "late": 80}[when]
        return (base + offset, np.nan)
    try:
        dt = dateparser.parse(s, fuzzy=True)
        if dt:
            return (int(dt.year), np.nan)
    except Exception:
        pass
    return (np.nan, np.nan)

def _parse_duration(d):
    if pd.isna(d): return np.nan
    s = str(d).strip().lower()
    if ":" in s:
        parts = [p.strip() for p in s.split(":")]
        try:
            parts = [int(p) for p in parts]
            if len(parts) == 2:
                m, sec = parts
                return m + sec / 60
            if len(parts) == 3:
                h, m, sec = parts
                return h * 60 + m + sec / 60
        except Exception:
            return np.nan
    try:
        return float(s)
    except Exception:
        return np.nan

# =========================
# Loaders (compositions)
# =========================
def load_compositions():
    ds = load_dataset("csv", data_files="hf://datasets/zliang/ASAP/ASAPcomposition.csv", token=HF_TOKEN)
    df = ds["train"].to_pandas()
    df = df.rename(columns={
        "Name": "Composer",
        "Composition Title": "Title",
        "Duration": "Duration",
        "Year": "Year",
        "Level": "Level"
    })
    starts, ends = [], []
    for raw in df.get("Year", pd.Series([np.nan] * len(df))):
        y0, y1 = _parse_year_robust(raw)
        starts.append(y0); ends.append(y1)
    df["YearStart"] = starts
    df["YearEnd"] = ends
    df["YearParsed"] = df["YearStart"]
    df["DurationMin"] = df.get("Duration", pd.Series([np.nan]*len(df))).map(_parse_duration)
    df["LevelStd"] = df.get("Level", pd.Series(["Unknown"]*len(df))).fillna("Unknown")
    return df

# =========================
# Demonyms & country normalization (people)
# =========================
_COUNTRY_ALIASES = {
    "czech republic": "Czechia",
    "viet nam": "Vietnam",
    "russian federation": "Russia",
    "syrian arab republic": "Syria",
    "lao people's democratic republic": "Laos",
    "bolivia, plurinational state of": "Bolivia",
    "venezuela, bolivarian republic of": "Venezuela",
    "tanzania, united republic of": "Tanzania",
    "moldova, republic of": "Moldova",
    "iran, islamic republic of": "Iran",
    "korea, republic of": "South Korea",
    "korea, democratic people's republic of": "North Korea",
    "congo, the democratic republic of the": "Democratic Republic of the Congo",
    "congo (kinshasa)": "Democratic Republic of the Congo",
    "congo (brazzaville)": "Congo",
    "eswatini": "Eswatini",
    "macedonia": "North Macedonia",
    "myanmar (burma)": "Myanmar",
    "cote d'ivoire": "Côte d’Ivoire",
    "ivory coast": "Côte d’Ivoire",
}

_DEMONYM_TO_COUNTRY = {
    "american": "United States", "u.s.": "United States", "u.s.a.": "United States", "us": "United States",
    "canadian": "Canada", "mexican": "Mexico", "argentinian": "Argentina", "argentine": "Argentina",
    "brazilian": "Brazil", "chilean": "Chile", "peruvian": "Peru", "colombian": "Colombia",
    "venezuelan": "Venezuela", "cuban": "Cuba", "puerto rican": "Puerto Rico",
    "dominican": "Dominican Republic", "haitian": "Haiti", "jamaican": "Jamaica",
    "barbadian": "Barbados", "bahamian": "Bahamas", "trinidadian": "Trinidad and Tobago", "tobagonian": "Trinidad and Tobago",
    "british": "United Kingdom", "english": "United Kingdom", "scottish": "United Kingdom", "welsh": "United Kingdom",
    "irish": "Ireland", "french": "France", "german": "Germany", "austrian": "Austria", "swiss": "Switzerland",
    "italian": "Italy", "spanish": "Spain", "spaniard": "Spain", "portuguese": "Portugal", "dutch": "Netherlands",
    "belgian": "Belgium", "danish": "Denmark", "norwegian": "Norway", "swedish": "Sweden", "finnish": "Finland",
    "estonian": "Estonia", "latvian": "Latvia", "lithuanian": "Lithuania", "polish": "Poland", "czech": "Czechia",
    "slovak": "Slovakia", "hungarian": "Hungary", "romanian": "Romania", "bulgarian": "Bulgaria", "greek": "Greece",
    "russian": "Russia", "ukrainian": "Ukraine", "belarusian": "Belarus", "georgian": "Georgia", "armenian": "Armenia",
    "azerbaijani": "Azerbaijan", "serbian": "Serbia", "croatian": "Croatia", "bosnian": "Bosnia and Herzegovina",
    "montenegrin": "Montenegro", "slovenian": "Slovenia", "macedonian": "North Macedonia", "albanian": "Albania",
    "turkish": "Turkey", "cypriot": "Cyprus", "israeli": "Israel", "palestinian": "Palestine", "lebanese": "Lebanon",
    "jordanian": "Jordan", "syrian": "Syria", "iraqi": "Iraq", "iranian": "Iran", "saudi": "Saudi Arabia",
    "emirati": "United Arab Emirates", "qatari": "Qatar", "kuwaiti": "Kuwait", "bahraini": "Bahrain", "omani": "Oman", "yemeni": "Yemen",
    "egyptian": "Egypt", "moroccan": "Morocco", "algerian": "Algeria", "tunisian": "Tunisia", "libyan": "Libya",
    "ethiopian": "Ethiopia", "eritrean": "Eritrea", "somali": "Somalia", "kenyan": "Kenya", "tanzanian": "Tanzania",
    "ugandan": "Uganda", "rwandan": "Rwanda", "burundian": "Burundi", "congolese": "Democratic Republic of the Congo",
    "angolan": "Angola", "zambian": "Zambia", "zimbabwean": "Zimbabwe", "botswanan": "Botswana", "namibian": "Namibia",
    "south african": "South Africa", "mozambican": "Mozambique", "ghanaian": "Ghana", "nigerian": "Nigeria", "cameroonian": "Cameroon",
    "ivorian": "Côte d’Ivoire", "senegalese": "Senegal", "malian": "Mali",
    "chinese": "China", "taiwanese": "Taiwan", "hong konger": "Hong Kong", "japanese": "Japan", "korean": "South Korea",
    "north korean": "North Korea", "indian": "India", "pakistani": "Pakistan", "bangladeshi": "Bangladesh", "sri lankan": "Sri Lanka",
    "nepalese": "Nepal", "bhutanese": "Bhutan", "burmese": "Myanmar", "myanmarese": "Myanmar", "thai": "Thailand",
    "cambodian": "Cambodia", "laotian": "Laos", "vietnamese": "Vietnam", "malaysian": "Malaysia", "singaporean": "Singapore",
    "indonesian": "Indonesia", "filipino": "Philippines",
    "australian": "Australia", "new zealander": "New Zealand", "fijian": "Fiji", "samoan": "Samoa", "tongan": "Tonga",
}

def _col(df, *candidates):
    cols = {c.lower().strip(): c for c in df.columns}
    for cand in candidates:
        k = cand.lower().strip()
        if k in cols: return cols[k]
    return None

def _basic_clean_nat(s: str) -> str:
    s = str(s)
    s = s.replace("(", " ").replace(")", " ")
    s = re.sub(r"[.\u200b]", " ", s)
    s = re.sub(r"\s+", " ", s)
    return s.strip()

def _normalize_country_or_demonym(token: str):
    if token is None or (isinstance(token, float) and np.isnan(token)):
        return np.nan
    t = _basic_clean_nat(token).lower()
    t = re.sub(r"\b(citizen|national|born|of|the|composer)\b", " ", t)
    t = re.sub(r"\s+", " ", t).strip()
    if t in _DEMONYM_TO_COUNTRY: return _DEMONYM_TO_COUNTRY[t]
    if t in _COUNTRY_ALIASES: return _COUNTRY_ALIASES[t]
    short = {
        "usa": "United States", "u.s.a": "United States", "u.s": "United States", "us": "United States",
        "uk": "United Kingdom", "england": "United Kingdom", "scotland": "United Kingdom", "wales": "United Kingdom",
        "korea": "South Korea", "russia": "Russia",
    }
    if t in short: return short[t]
    if re.search(r"[a-z]", t):  # likely a country already
        return t.title()
    return np.nan

def _split_and_normalize_nationalities(value):
    if pd.isna(value): return []
    s = _basic_clean_nat(value)
    s = re.sub(r"\s*(/|;|&|\band\b|,)\s*", ",", s, flags=re.I)
    parts = [p for p in (x.strip() for x in s.split(",")) if p]
    out = []
    for p in parts:
        norm = _normalize_country_or_demonym(p)
        if isinstance(norm, str) and norm:
            out.append(norm)
    return out

def load_people():
    ds = load_dataset("csv", data_files="hf://datasets/zliang/ASAP/ASAPdata.csv", token=HF_TOKEN)
    df = ds["train"].to_pandas()
    col_gender = _col(df, "Gender", "gender")
    col_nat = _col(df, "Nationality", "Country", "nationality", "country")
    col_eth = _col(df, "Ethnicity", "ethnicity")
    col_era = _col(df, "Music Era", "Era", "Period", "music era", "era", "period")
    if col_gender: df = df.rename(columns={col_gender: "Gender"})
    else: df["Gender"] = np.nan
    if col_nat: df = df.rename(columns={col_nat: "Nationality"})
    else: df["Nationality"] = np.nan
    if col_eth: df = df.rename(columns={col_eth: "Ethnicity"})
    else: df["Ethnicity"] = np.nan
    if col_era: df = df.rename(columns={col_era: "Music Era"})
    else: df["Music Era"] = np.nan
    df["NationalityList"] = df["Nationality"].apply(_split_and_normalize_nationalities)
    df_exp = df.explode("NationalityList").rename(columns={"NationalityList": "NationalityNorm"})
    df_exp["NationalityNorm"] = df_exp["NationalityNorm"].replace({"": np.nan})
    return df, df_exp

# =========================
# Visuals — Compositions
# =========================
def area_timeline(df):
    d = df.dropna(subset=["YearParsed"])
    if d.empty: return go.Figure()
    counts = d.groupby(["YearParsed", "LevelStd"])["Title"].count().reset_index(name="Count")
    counts = counts.sort_values("YearParsed")
    counts["Smoothed"] = counts.groupby("LevelStd")["Count"].transform(lambda s: s.rolling(3, min_periods=1).mean())
    fig = px.area(counts, x="YearParsed", y="Smoothed", color="LevelStd",
                  title="Compositions per Year by Level", template=PLOT_TEMPLATE)
    fig.update_layout(margin=dict(l=10, r=10, t=50, b=10), legend_title="Level")
    fig.update_xaxes(title="Year"); fig.update_yaxes(title="Smoothed count")
    return fig

def bar_top_composers(df, top_n=15):
    if df.empty:
        return go.Figure()

    # group + sort
    d = (
        df.groupby("Composer")["Title"].count()
          .sort_values(ascending=False)
          .head(top_n)
          .reset_index()
          .rename(columns={"Title": "Count"})
    )

    # order composers by descending count
    composer_order = d["Composer"].tolist()

    fig = px.bar(
        d,
        x="Count",
        y="Composer",
        orientation="h",
        title=f"Top {top_n} Composers by Number of Compositions",
        template=PLOT_TEMPLATE
    )

    # force y-axis order: most at top → least at bottom
    fig.update_layout(
        yaxis=dict(categoryorder="array", categoryarray=composer_order[::-1]),
        margin=dict(l=10, r=10, t=50, b=10)
    )

    return fig


def treemap_composer_level(df):
    if df.empty: return go.Figure()
    d = df.groupby(["Composer", "LevelStd"])["Title"].count().reset_index(name="Count")
    fig = px.treemap(d, path=["Composer", "LevelStd"], values="Count",
                     title="Catalog Structure: Composer → Level", template=PLOT_TEMPLATE)
    fig.update_layout(margin=dict(l=10, r=10, t=50, b=10))
    return fig

def make_wordcloud(df):
    titles = df["Title"].dropna().astype(str)
    # English filter: keep titles with sufficient ASCII ratio
    titles = [t for t in titles if _ascii_ratio(t) >= MIN_ASCII_RATIO]
    if not titles:
        return None
    text = " ".join(titles)
    stopwords = set(STOPWORDS)
    stopwords.update({"Piano", "II", "IV","V","III","Piece","Pieces","Op","VII","IX","VIII"})  # 👈 add more as needed
    wc = WordCloud(width=1200, height=500, background_color="white",
                   stopwords=stopwords, collocations=True)
    return wc.generate(text).to_image()

# =========================
# Visuals — People (pies + LOG map)
# =========================
def pie(df_people_exp_or_raw, column, title, max_slices=12):
    if column not in df_people_exp_or_raw.columns:
        return go.Figure()
    s = df_people_exp_or_raw[column].dropna()
    if s.empty: return go.Figure()
    counts = s.value_counts(dropna=False)
    if len(counts) > max_slices:
        head = counts.iloc[:max_slices-1]
        other = pd.Series({"Other": counts.iloc[max_slices-1:].sum()})
        counts = pd.concat([head, other])
    d = counts.reset_index()
    d.columns = [column, "Count"]
    fig = px.pie(d, names=column, values="Count", title=title, template=PLOT_TEMPLATE, hole=0.35)
    fig.update_traces(textposition="inside", textinfo="percent+label")
    fig.update_layout(margin=dict(l=10, r=10, t=50, b=10))
    return fig

def world_map_nationality(df_people_exp):
    # Exploded + normalized nationalities expected here
    s = df_people_exp["NationalityNorm"].dropna()
    if s.empty:
        return go.Figure()

    counts = s.value_counts().reset_index()
    counts.columns = ["country", "count"]

    # ---- log transform in the data (avoid coloraxis.type) ----
    counts["count_log10"] = np.log10(counts["count"] + 1.0)

    fig = px.choropleth(
        counts,
        locations="country",
        locationmode="country names",
        color="count_log10",
        color_continuous_scale="Blues",
        title="Global Distribution by Nationality (log scale)",
        template=PLOT_TEMPLATE,
        hover_data={"count": True, "count_log10": False, "country": False},
    )

    # Colorbar: show a few human-friendly ticks with labels in raw counts
    if not counts.empty:
        vmax = int(counts["count"].max())
        # choose nice ticks (1, 3, 10, 30, 100, ...)
        raw_ticks = []
        step = 1
        base = [1, 3]
        while step <= vmax:
            for b in base:
                val = b * step
                if val <= vmax:
                    raw_ticks.append(val)
            step *= 10
        raw_ticks = sorted(set([1] + raw_ticks + [vmax]))

        tickvals = np.log10(np.array(raw_ticks, dtype=float) + 1.0)
        ticktext = [str(v) for v in raw_ticks]

        fig.update_layout(
            margin=dict(l=10, r=10, t=50, b=10),
            coloraxis_colorbar=dict(title="Count", tickvals=tickvals, ticktext=ticktext),
        )
    else:
        fig.update_layout(margin=dict(l=10, r=10, t=50, b=10), coloraxis_colorbar_title="Count")

    return fig

# =========================
# Pipeline
# =========================
def compute_all():
    comp = load_compositions()
    ppl_raw, ppl_exp = load_people()
    return (
        # compositions
        area_timeline(comp),
        bar_top_composers(comp),
        treemap_composer_level(comp),
        make_wordcloud(comp),
        # people pies + log map
        pie(ppl_raw, "Gender", "Gender"),
        pie(ppl_exp, "NationalityNorm", "Nationality"),
        pie(ppl_raw, "Ethnicity", "Ethnicity"),
        pie(ppl_raw, "Music Era", "Music Era"),
        world_map_nationality(ppl_exp)
    )

# =========================
# UI (simple, no uploads, no filters, no table)
# =========================
theme = gr.themes.Soft(primary_hue="blue", neutral_hue="slate")

with gr.Blocks(title="A Seat At The Piano", theme=theme) as demo:
    gr.Markdown(
        f"## 🎼 ASAP Explorer\n"
        f"A Seat at the Piano was founded in the summer of 2020 in the midst of social and racial reckoning around the world. We are a team of classically trained pianists with varying backgrounds and experiences, who strive to raise the voices of those who are less heard and to inspire more thoughtful, inclusive programming within the performing and pedagogical spheres.\n"
    )

    with gr.Tabs():
        with gr.Tab("Composer Demographics"):
            with gr.Row():
                pie_gender = gr.Plot()
                pie_nat = gr.Plot()
            with gr.Row():
                pie_eth = gr.Plot()
                pie_era = gr.Plot()
            world_map = gr.Plot()
        with gr.Tab("Compositions Overview"):
            timeline_plot = gr.Plot()
            top_plot = gr.Plot()
        with gr.Tab("Compositions Catogories"):
            tree_plot = gr.Plot()
            wc_img = gr.Image(type="pil")


    demo.load(
        compute_all,
        inputs=None,
        outputs=[
            timeline_plot, top_plot, tree_plot, wc_img,
            pie_gender, pie_nat, pie_eth, pie_era, world_map
        ],
    )

if __name__ == "__main__":
    demo.launch()