Compute total entropy too.

app.py

@@ -5,8 +5,8 @@ import torch
 import gradio as gr
 from transformers import AutoTokenizer, AutoModelForCausalLM
 
-#MODEL_ID = os.getenv("MODEL_ID", "Qwen/Qwen2.5-Coder-1.5B")
-MODEL_ID = os.getenv("MODEL_ID", "bigcode/starcoder2-3b")
+MODEL_ID = os.getenv("MODEL_ID", "Qwen/Qwen2.5-Coder-1.5B")
+#MODEL_ID = os.getenv("MODEL_ID", "bigcode/starcoder2-3b")
 
 
 def load_model():
@@ -73,10 +73,16 @@ def compute_entropy(code: str):
 
     avg_nll = sum(nll_list) / len(nll_list)
     avg_bits = avg_nll / math.log(2)
+    # total entropy in bits is the sum of per-token NLL (bits)
+    total_bits = sum(nll_list) / math.log(2)
+    # total entropy in nats is simply the sum of per-token NLL (nats)
+    total_nats = sum(nll_list)
     summary = (
         f"Tokens evaluated: {len(nll_list)}\n"
         f"Average NLL (nats): {avg_nll:.4f}\n"
-        f"Average NLL (bits): {avg_bits:.4f}"
+        f"Average NLL (bits): {avg_bits:.4f}\n"
+        f"Total entropy (nats): {total_nats:.4f}\n"
+        f"Total entropy (bits): {total_bits:.4f}"
     )
 
     return summary, rows