adding local app - to be integrated with public app

app.py

@@ -1,150 +1,7 @@
 import evaluate
 import numpy as np
-import pandas as pd
-import ast
-import json
-import gradio as gr
-from evaluate.utils import launch_gradio_widget
-from ece import ECE
-
-import matplotlib.pyplot as plt
-import seaborn as sns
-sns.set_style('white')
-sns.set_context("paper", font_scale=1)  # 2
-# plt.rcParams['figure.figsize'] = [10, 7]
-plt.rcParams['figure.dpi'] = 300
-plt.switch_backend('agg') #; https://stackoverflow.com/questions/14694408/runtimeerror-main-thread-is-not-in-main-loop
-
-sliders = [
-    gr.Slider(0, 100, value=10, label="n_bins"),
-    gr.Slider(0, 100, value=None, label="bin_range", visible=False), #DEV: need to have a double slider
-    gr.Dropdown(choices=["equal-range", "equal-mass"], value="equal-range", label="scheme"),
-    gr.Dropdown(choices=["upper-edge", "center"], value="upper-edge", label="proxy"),
-    gr.Dropdown(choices=[1, 2, np.inf], value=1, label="p"),
-]
-
-slider_defaults = [slider.value for slider in sliders]
-
-# example data
-df = dict()
-df["predictions"] = [[0.6, 0.2, 0.2], [0, 0.95, 0.05], [0.7, 0.1, 0.2]]
-df["references"] = [0, 1, 2]
-
-component = gr.inputs.Dataframe(
-    headers=["predictions", "references"], col_count=2, datatype="number", type="pandas"
-)
-
-component.value = [
-    [[0.6, 0.2, 0.2], 0],
-    [[0.7, 0.1, 0.2], 2],
-    [[0, 0.95, 0.05], 1],
-]
-sample_data = [[component] + slider_defaults]  ##json.dumps(df)
-
-
-metric = ECE()
-# module = evaluate.load("jordyvl/ece")
-# launch_gradio_widget(module)
-
-"""
-Switch inputs and compute_fn
-"""
-
-def reliability_plot(results):
-    fig = plt.figure()
-    ax1 = plt.subplot2grid((3, 1), (0, 0), rowspan=2)
-    ax2 = plt.subplot2grid((3, 1), (2, 0))
-
-    n_bins = len(results["y_bar"])
-    bin_range = [
-        results["y_bar"][0] - results["y_bar"][0],
-        results["y_bar"][-1],
-    ]  # np.linspace(0, 1, n_bins)
-    # if upper edge then minus binsize; same for center [but half]
-
-    ranged = np.linspace(bin_range[0], bin_range[1], n_bins)
-    ax1.plot(
-        ranged,
-        ranged,
-        color="darkgreen",
-        ls="dotted",
-        label="Perfect",
-    )
-    # ax1.plot(results["y_bar"], results["y_bar"], color="darkblue", label="Perfect")
-
-    anindices = np.where(~np.isnan(results["p_bar"][:-1]))[0]
-    bin_freqs = np.zeros(n_bins)
-    bin_freqs[anindices] = results["bin_freq"]
-    ax2.hist(results["y_bar"], results["y_bar"], weights=bin_freqs)
 
-    #widths = np.diff(results["y_bar"])
-    for j, bin in enumerate(results["y_bar"]):
-        perfect = results["y_bar"][j]
-        empirical = results["p_bar"][j]
 
-        if np.isnan(empirical):
-            continue
-
-        ax1.bar([perfect], height=[empirical], width=-ranged[j], align="edge", color="lightblue")
-
-        if perfect == empirical:
-            continue
-
-    acc_plt = ax2.axvline(
-        x=results["accuracy"], ls="solid", lw=3, c="black", label="Accuracy"
-    )
-    conf_plt = ax2.axvline(
-        x=results["p_bar_cont"], ls="dotted", lw=3, c="#444", label="Avg. confidence"
-    )
-    ax2.legend(handles=[acc_plt, conf_plt])
-
-    #Bin differences
-    ax1.set_ylabel("Conditional Expectation")
-    ax1.set_ylim([-0.05, 1.05]) #respective to bin range
-    ax1.legend(loc="lower right")
-    ax1.set_title("Reliability Diagram")
-
-    #Bin frequencies
-    ax2.set_xlabel("Confidence")
-    ax2.set_ylabel("Count")
-    ax2.legend(loc="upper left")#, ncol=2
-    plt.tight_layout()
-    return fig
-
-def compute_and_plot(data, n_bins, bin_range, scheme, proxy, p):
-    # DEV: check on invalid datatypes with better warnings
-
-    if isinstance(data, pd.DataFrame):
-        data.dropna(inplace=True)
-
-    predictions = [
-        ast.literal_eval(prediction) if not isinstance(prediction, list) else prediction
-        for prediction in data["predictions"]
-    ]
-    references = [reference for reference in data["references"]]
-
-    results = metric._compute(
-        predictions,
-        references,
-        n_bins=n_bins,
-        # bin_range=None,#not needed
-        scheme=scheme,
-        proxy=proxy,
-        p=p,
-        detail=True,
-    )
-
-    plot = reliability_plot(results)
-    return results["ECE"], plot #plt.gcf()
-
-
-outputs = [gr.outputs.Textbox(label="ECE"), gr.Plot(label="Reliability diagram")]
-
-iface = gr.Interface(
-    fn=compute_and_plot,
-    inputs=[component] + sliders,
-    outputs=outputs,
-    description=metric.info.description,
-    article=metric.info.citation,
-    # examples=sample_data; # ValueError: Examples argument must either be a directory or a nested list, where each sublist represents a set of inputs.
-).launch()
+from evaluate.utils import launch_gradio_widget
+module = evaluate.load("jordyvl/ece")
+launch_gradio_widget(module)

local_app.py

@@ -0,0 +1,150 @@
+import evaluate
+import numpy as np
+import pandas as pd
+import ast
+import json
+import gradio as gr
+from evaluate.utils import launch_gradio_widget
+from ece import ECE
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+sns.set_style('white')
+sns.set_context("paper", font_scale=1)  # 2
+# plt.rcParams['figure.figsize'] = [10, 7]
+plt.rcParams['figure.dpi'] = 300
+plt.switch_backend('agg') #; https://stackoverflow.com/questions/14694408/runtimeerror-main-thread-is-not-in-main-loop
+
+sliders = [
+    gr.Slider(0, 100, value=10, label="n_bins"),
+    gr.Slider(0, 100, value=None, label="bin_range", visible=False), #DEV: need to have a double slider
+    gr.Dropdown(choices=["equal-range", "equal-mass"], value="equal-range", label="scheme"),
+    gr.Dropdown(choices=["upper-edge", "center"], value="upper-edge", label="proxy"),
+    gr.Dropdown(choices=[1, 2, np.inf], value=1, label="p"),
+]
+
+slider_defaults = [slider.value for slider in sliders]
+
+# example data
+df = dict()
+df["predictions"] = [[0.6, 0.2, 0.2], [0, 0.95, 0.05], [0.7, 0.1, 0.2]]
+df["references"] = [0, 1, 2]
+
+component = gr.inputs.Dataframe(
+    headers=["predictions", "references"], col_count=2, datatype="number", type="pandas"
+)
+
+component.value = [
+    [[0.6, 0.2, 0.2], 0],
+    [[0.7, 0.1, 0.2], 2],
+    [[0, 0.95, 0.05], 1],
+]
+sample_data = [[component] + slider_defaults]  ##json.dumps(df)
+
+
+metric = ECE()
+# module = evaluate.load("jordyvl/ece")
+# launch_gradio_widget(module)
+
+"""
+Switch inputs and compute_fn
+"""
+
+def reliability_plot(results):
+    fig = plt.figure()
+    ax1 = plt.subplot2grid((3, 1), (0, 0), rowspan=2)
+    ax2 = plt.subplot2grid((3, 1), (2, 0))
+
+    n_bins = len(results["y_bar"])
+    bin_range = [
+        results["y_bar"][0] - results["y_bar"][0],
+        results["y_bar"][-1],
+    ]  # np.linspace(0, 1, n_bins)
+    # if upper edge then minus binsize; same for center [but half]
+
+    ranged = np.linspace(bin_range[0], bin_range[1], n_bins)
+    ax1.plot(
+        ranged,
+        ranged,
+        color="darkgreen",
+        ls="dotted",
+        label="Perfect",
+    )
+    # ax1.plot(results["y_bar"], results["y_bar"], color="darkblue", label="Perfect")
+
+    anindices = np.where(~np.isnan(results["p_bar"][:-1]))[0]
+    bin_freqs = np.zeros(n_bins)
+    bin_freqs[anindices] = results["bin_freq"]
+    ax2.hist(results["y_bar"], results["y_bar"], weights=bin_freqs)
+
+    #widths = np.diff(results["y_bar"])
+    for j, bin in enumerate(results["y_bar"]):
+        perfect = results["y_bar"][j]
+        empirical = results["p_bar"][j]
+
+        if np.isnan(empirical):
+            continue
+
+        ax1.bar([perfect], height=[empirical], width=-ranged[j], align="edge", color="lightblue")
+
+        if perfect == empirical:
+            continue
+
+    acc_plt = ax2.axvline(
+        x=results["accuracy"], ls="solid", lw=3, c="black", label="Accuracy"
+    )
+    conf_plt = ax2.axvline(
+        x=results["p_bar_cont"], ls="dotted", lw=3, c="#444", label="Avg. confidence"
+    )
+    ax2.legend(handles=[acc_plt, conf_plt])
+
+    #Bin differences
+    ax1.set_ylabel("Conditional Expectation")
+    ax1.set_ylim([-0.05, 1.05]) #respective to bin range
+    ax1.legend(loc="lower right")
+    ax1.set_title("Reliability Diagram")
+
+    #Bin frequencies
+    ax2.set_xlabel("Confidence")
+    ax2.set_ylabel("Count")
+    ax2.legend(loc="upper left")#, ncol=2
+    plt.tight_layout()
+    return fig
+
+def compute_and_plot(data, n_bins, bin_range, scheme, proxy, p):
+    # DEV: check on invalid datatypes with better warnings
+
+    if isinstance(data, pd.DataFrame):
+        data.dropna(inplace=True)
+
+    predictions = [
+        ast.literal_eval(prediction) if not isinstance(prediction, list) else prediction
+        for prediction in data["predictions"]
+    ]
+    references = [reference for reference in data["references"]]
+
+    results = metric._compute(
+        predictions,
+        references,
+        n_bins=n_bins,
+        # bin_range=None,#not needed
+        scheme=scheme,
+        proxy=proxy,
+        p=p,
+        detail=True,
+    )
+
+    plot = reliability_plot(results)
+    return results["ECE"], plot #plt.gcf()
+
+
+outputs = [gr.outputs.Textbox(label="ECE"), gr.Plot(label="Reliability diagram")]
+
+iface = gr.Interface(
+    fn=compute_and_plot,
+    inputs=[component] + sliders,
+    outputs=outputs,
+    description=metric.info.description,
+    article=metric.info.citation,
+    # examples=sample_data; # ValueError: Examples argument must either be a directory or a nested list, where each sublist represents a set of inputs.
+).launch()