Upload 3 files

main.py

@@ -0,0 +1,24 @@
+import gradio as gr
+import matplotlib
+import models
+import utils
+
+def infer(gt: str, data: str) -> matplotlib.figure:
+    nli_res = models.compute_metric(gt, data)
+    tone_res = models.compare_tone(gt, data)
+    res_text = ""
+    if (nli_res["label"] == "neutral"):
+        res_text += "Model's response is unrelated to the Ground Truth"
+    if (nli_res["label"] == "contradiction"):
+        res_text += "Model's response contradicts the Ground Truth"
+    if (nli_res["label"] == "entailment"):
+        res_text += "Model's response is consistant with the Ground Truth"
+    return res_text, utils.create_pie_chart_nli(nli_res), utils.plot_tones(tone_res)
+
+examples = [["Cross-encoders are better than bi-encoders for analyzing the relationship betwen texts", "Bi-encoders are superior to cross-encoders"],
+            ["Cross-encoders are better than bi-encoders for analyzing the relationship betwen texts", "The cosine similarity function can be used to compare the outputs of a bi-encoder"],
+            ["Cross-encoders are better than bi-encoders for analyzing the relationship betwen texts", "Bi-encoders are outperformed by cross-encoders in the task of relationship analysis"],
+            ["Birds can fly. There are fish in the sea.", "Fish inhabit the ocean. Birds can aviate."],
+            ["Birds can fly. There are fish in the sea.", "Fish inhabit the ocean. Birds can not aviate."]]
+app = gr.Interface(fn=infer, inputs=[gr.Textbox(label="Ground Truth"), gr.Textbox(label="Model Response")], examples=examples, outputs=[gr.Textbox(label="Result"), gr.Plot(label="Comparison with GT"), gr.Plot(label="Difference in Tone")])
+app.launch()

models.py

@@ -0,0 +1,52 @@
+from sentence_transformers import CrossEncoder
+from transformers import AutoModelForSequenceClassification
+from transformers import AutoTokenizer, AutoConfig
+import numpy as np
+
+def softmax(x):
+    e_x = np.exp(x - np.max(x))
+    return e_x / e_x.sum(axis=0)
+
+# 90.04% accuracy on MNLI mismatched set
+nli_model = CrossEncoder('cross-encoder/nli-deberta-v3-base')
+
+def compute_metric(ground_truth: str, inference: str) -> dict:
+    scores = nli_model.predict([ground_truth, inference], apply_softmax=True)
+    label = ['contradiction', 'entailment', 'neutral'][scores.argmax()]
+    return {
+        'label': label,
+        'contradiction': scores[0],
+        'entailment': scores[1],
+        'neutral': scores[2],
+    }
+
+def _compare_tone(text: str) -> dict:
+    # Trained on ~124M Tweets for sentiment analysis
+    model_name = r"cardiffnlp/twitter-roberta-base-sentiment-latest"
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    config = AutoConfig.from_pretrained(model_name)
+    model = AutoModelForSequenceClassification.from_pretrained(model_name)
+
+    encoded_input = tokenizer(text, return_tensors='pt')
+    output = model(**encoded_input)
+    scores = output[0][0].detach().numpy()
+    scores = softmax(scores)
+    ranking = np.argsort(scores)
+    ranking = ranking[::-1]
+    result = {}
+    for i in range(scores.shape[0]):
+        l = config.id2label[ranking[i]]
+        s = scores[ranking[i]]
+        result[l] = np.round(float(s), 4)
+
+    return result
+
+def compare_tone(ground_truth: str, inference: str) -> dict:
+    gt = _compare_tone(ground_truth)
+    model_res = _compare_tone(inference)
+    return {"gt": gt, "model": model_res}
+    
+if __name__ == "__main__":
+    print(compute_metric("Foxes are closer to dogs than they are to cats. Therefore, foxes are not cats.", "Foxes are not cats."))
+    print(compute_metric("Foxes are closer to dogs than they are to cats. Therefore, foxes are not cats.", "Foxes are cats."))
+    print(compare_tone("This is neutural", "Wtf"))

utils.py

@@ -0,0 +1,28 @@
+import matplotlib
+import matplotlib.pyplot as plt
+
+def create_pie_chart_nli(data: dict) -> matplotlib.figure:
+    labels = ["neutral", "contradiction", "entailment"]
+    sizes = [data[label] for label in labels]
+    colors = ["gray", "red", "green"]
+    
+    fig, ax = plt.subplots()
+
+    ax.set_title("Comparison with GT")
+    ax.pie(sizes, labels=labels, colors=colors, autopct='%1.1f%%')
+
+    ax.axis('equal')
+    
+    return fig
+
+def plot_tones(data: dict) -> matplotlib.figure:
+    keys = data["gt"].keys()
+
+    fig, ax = plt.subplots()
+    ax.set_title("Tone")
+    ax.bar(x=keys, height=[data["gt"][key] for key in keys], color="b", label="Ground Truth", width=0.7)
+    ax.bar(x=keys, height=[data["model"][key] for key in keys], color="r", alpha=0.5, label="Model response", width=0.5)
+
+    fig.legend()
+
+    return fig