Refactor

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,12 +1,10 @@
-from math import e
-import pandas as pd
 import gradio as gr
 import matplotlib.pyplot as plt
 import matplotlib
-import io
-import base64
 
 from data import ModelBenchmarkData
+from bar_plot import create_matplotlib_bar_plot
+
 
 # Configure matplotlib for better performance
 matplotlib.use('Agg')
@@ -16,12 +14,6 @@ plt.ioff()
 DATA = ModelBenchmarkData("data.json")
 
 
-def refresh_plot_data():
-    data = DATA.get_ttft_tpot_data(estimator="median", use_cuda_time=False)
-    print(data)
-    return pd.DataFrame(data)
-
-
 def load_css():
     """Load CSS styling."""
     try:
@@ -30,132 +22,11 @@ def load_css():
     except FileNotFoundError:
         return "body { background: #000; color: #fff; }"
 
-def create_matplotlib_bar_charts():
-    """Create side-by-side matplotlib bar charts for TTFT and TPOT data."""
-    data = DATA.get_ttft_tpot_data(estimator="median", use_cuda_time=False)
-
-    # Create figure with dark theme - larger for more screen space
-    plt.style.use('dark_background')
-    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 16))
-    fig.patch.set_facecolor('#000000')
-
-    # Prepare data
-    labels = data['label']
-    ttft_values = data['ttft']
-    tpot_values = data['tpot']
-
-    # Calculate error bars (standard deviation) for each configuration
-    import numpy as np
-    ttft_errors = []
-    tpot_errors = []
-
-    # Get raw data to calculate standard deviations
-    raw_data = DATA.data
-    for label in labels:
-        if label in raw_data:
-            # Calculate std dev for TTFT
-            ttft_raw = [d['wall_time'] for d in raw_data[label]['ttft']]
-            ttft_errors.append(float(np.std(ttft_raw)))
-
-            # Calculate std dev for TPOT
-            tpot_raw = [d['wall_time'] for d in raw_data[label]['tpot']]
-            tpot_errors.append(float(np.std(tpot_raw)))
-        else:
-            ttft_errors.append(0)
-            tpot_errors.append(0)
-
-    # Define color mapping based on configuration keywords
-    def get_color_for_config(label):
-        is_eager = 'eager' in label.lower()
-        is_sdpa = 'sdpa' in label.lower()
-        is_compiled = '_compiled' in label.lower()
-        if is_eager:
-            if is_compiled:
-                return '#FF4444'  # Red for eager compiled
-            else:
-                return '#FF6B6B'  # Light red for eager uncompiled
-        elif is_sdpa:
-            if is_compiled:
-                return '#4A90E2'  # Blue for SDPA compiled
-            else:
-                return '#7BB3F0'  # Light blue for SDPA uncompiled
-        else:
-            return '#FFD700'  # Yellow for others
-
-    # Get colors for each bar
-    colors = [get_color_for_config(label) for label in labels]
-
-    # TTFT Plot (left)
-    ax1.set_facecolor('#000000')
-    _ = ax1.bar(range(len(labels)), ttft_values,
-                    color=colors, width=1.0, edgecolor='white', linewidth=1,
-                    label=[label[:15] + '...' if len(label) > 15 else label for label in labels])
-
-    # Add error bars for TTFT
-    ax1.errorbar(
-        range(len(labels)), ttft_values, yerr=ttft_errors,
-        fmt='none', ecolor='white', alpha=0.8,
-        elinewidth=1.5, capthick=1.5, capsize=4,
-    )
-
-    ax1.set_ylabel('TTFT (seconds)', color='white', fontsize=14)
-    ax1.set_title('Time to first token (lower is better)', color='white', fontsize=16, pad=20)
-
-    ax1.set_xticks([])
-    ax1.tick_params(colors='white')
-    ax1.grid(True, alpha=0.3, color='white')
-
-    # TPOT Plot (right)
-    ax2.set_facecolor('#000000')
-    _ = ax2.bar(range(len(labels)), tpot_values,
-                    color=colors, width=1.0, edgecolor='white', linewidth=1)
-
-    # Add error bars for TPOT
-    ax2.errorbar(
-        range(len(labels)), tpot_values, yerr=tpot_errors,
-        fmt='none', ecolor='white', alpha=0.8,
-        elinewidth=1.5, capthick=1.5, capsize=4,
-    )
-
-    ax2.set_ylabel('TPOT (seconds)', color='white', fontsize=14)
-    ax2.set_title('Time per output token (lower is better)', color='white', fontsize=16, pad=20)
-
-    ax2.set_xticks([])
-    ax2.tick_params(colors='white')
-    ax2.grid(True, alpha=0.3, color='white')
-
-    # Add common legend with full text
-    legend_labels = labels  # Use full labels without truncation
-    legend_handles = [plt.Rectangle((0,0),1,1, color=color, edgecolor='white') for color in colors]
-    fig.legend(legend_handles, legend_labels, loc='lower center', ncol=1,
-               bbox_to_anchor=(0.5, -0.05), facecolor='black', edgecolor='white',
-               labelcolor='white', fontsize=12)
-
-    # Tight layout with spacing between subplots and extra bottom space for legend
-    plt.tight_layout()
-    plt.subplots_adjust(wspace=0.3, bottom=0.075)
-
-    # Save plot to bytes with high DPI for crisp text
-    buffer = io.BytesIO()
-    plt.savefig(buffer, format='png', facecolor='#000000',
-                bbox_inches='tight', dpi=130)
-    buffer.seek(0)
-
-    # Convert to base64 for HTML embedding
-    img_data = base64.b64encode(buffer.getvalue()).decode()
-    plt.close(fig)
 
-    # Return HTML with embedded image - full height
-    html = f"""
-    <div style="width: 100%; height: 100vh; background: #000; display: flex; justify-content: center; align-items: center;">
-        <img src="data:image/png;base64,{img_data}" style="width: 100%; height: 100%; object-fit: contain;" />
-    </div>
-    """
-    return html
 
 def refresh_plot():
     """Generate new matplotlib charts and update description."""
-    return create_matplotlib_bar_charts(), "**Transformer CI Dashboard**<br>-<br>**AMD runs on MI325**<br>**NVIDIA runs on A10**<br><br>*This dashboard only tracks important models*<br>*(Data refreshed)*"
+    return create_matplotlib_bar_plot(DATA.get_bar_plot_data()), "**Transformer CI Dashboard**<br>-<br>**AMD runs on MI325**<br>**NVIDIA runs on A10**<br><br>*This dashboard only tracks important models*<br>*(Data refreshed)*"
 
 # Create Gradio interface
 with gr.Blocks(title="Random Data Dashboard", css=load_css(), fill_height=True, fill_width=True) as demo:
@@ -169,7 +40,7 @@ with gr.Blocks(title="Random Data Dashboard", css=load_css(), fill_height=True,
         # Main plot area
         with gr.Column(elem_classes=["main-content"]):
             plot = gr.HTML(
-                create_matplotlib_bar_charts(),
+                create_matplotlib_bar_plot(DATA.get_bar_plot_data()),
                 elem_classes=["plot-container"],
             )
 

bar_plot.py

@@ -0,0 +1,134 @@
+import matplotlib.pyplot as plt
+import io
+import numpy as np
+import base64
+
+
+# Color manipulation functions
+def hex_to_rgb(hex_color):
+    hex_color = hex_color.lstrip('#')
+    r, g, b = int(hex_color[0:2], 16), int(hex_color[2:4], 16), int(hex_color[4:6], 16)
+    return r, g, b
+
+def increase_brightness(r, g, b, factor):
+    return tuple(map(lambda x: int(x + (255 - x) * factor), (r, g, b)))
+
+def increase_saturation(r, g, b, factor) -> tuple[int, int, int]:
+    gray = 0.299 * r + 0.587 * g + 0.114 * b
+    return tuple(map(lambda x: int(gray + (x - gray) * factor), (r, g, b)))
+
+def rgb_to_hex(r, g, b):
+    r, g, b = map(lambda x: min(max(x, 0), 255), (r, g, b))
+    return f"#{r:02x}{g:02x}{b:02x}"
+
+# Color assignment function
+def get_color_for_config(config):
+    
+    # Determine the main hue for the attention implementation
+    attn_implementation, sdpa_backend = config["attn_implementation"], config["sdpa_backend"]
+    if attn_implementation == "eager":
+        main_hue = "#FF6B6B"
+    elif attn_implementation == "sdpa":
+        main_hue = {
+            None: "#4A90E2",
+            "math": "#408DDBFF",
+            "flash_attention": "#28767EFF",
+            "efficient_attention": "#605895FF",
+            "cudnn_attention": "#774AE2FF",
+        }[sdpa_backend]
+    elif attn_implementation == "flash_attention_2":
+        main_hue = "#FFD700"
+    else:
+        raise ValueError(f"Unknown attention implementation: {attn_implementation}")
+
+    # Apply color modifications for compilation and kernelization
+    r, g, b = hex_to_rgb(main_hue)
+    if config["compilation"]:
+        r, g, b = increase_brightness(r, g, b, 0.3)
+    if config["kernelize"]:
+        r, g, b = increase_saturation(r, g, b, 0.8)
+    
+    # Return the color as a hex string
+    return rgb_to_hex(r, g, b)
+
+
+def make_bar_kwargs(per_scenario_data: dict, key: str) -> tuple[dict, list]:
+    bar_kwargs = {"x": [], "height": [], "color": [], "label": []}
+    errors = []
+    for i, (name, data) in enumerate(per_scenario_data.items()):    
+        bar_kwargs["x"].append(i)
+        bar_kwargs["height"].append(np.median(data[key]))
+        bar_kwargs["color"].append(get_color_for_config(data["config"]))
+        bar_kwargs["label"].append(name)
+        errors.append(np.std(data[key]))
+    return bar_kwargs, errors
+
+def draw_bar_plot(ax: plt.Axes, bar_kwargs: dict, errors: list, title: str, ylabel: str):
+    ax.set_facecolor('#000000')
+    # Draw bars
+    _ = ax.bar(**bar_kwargs, width=1.0, edgecolor='white', linewidth=1)
+    # Add error bars
+    ax.errorbar(
+        bar_kwargs["x"], bar_kwargs["height"], yerr=errors, 
+        fmt='none', ecolor='white', alpha=0.8, elinewidth=1.5, capthick=1.5, capsize=4,
+    )
+    # Set labels and title
+    ax.set_ylabel(ylabel, color='white', fontsize=14)
+    ax.set_title(title, color='white', fontsize=16, pad=20)
+    # Set ticks and grid
+    ax.set_xticks([])
+    ax.tick_params(colors='white')
+    ax.grid(True, alpha=0.3, color='white')
+    # Truncate axis to better fit the bars
+    # new_ymin, new_ymax = 1e9, -1e9
+    # for h, e in zip(bar_kwargs["height"], errors):
+    #     new_ymin = min(new_ymin, 0.98 * (h - e))
+    #     new_ymax = max(new_ymax, 1.02 * (h + e))
+    # ymin, ymax = ax.get_ylim()
+    # ax.set_ylim(max(ymin, new_ymin), min(ymax, new_ymax))
+
+
+def create_matplotlib_bar_plot(per_scenario_data: dict):
+    """Create side-by-side matplotlib bar charts for TTFT and TPOT data."""
+
+    # Create figure with dark theme - larger for more screen space
+    plt.style.use('dark_background')
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(18, 12))
+    fig.patch.set_facecolor('#000000')
+
+    # TTFT Plot (left)
+    ttft_bars, ttft_errors = make_bar_kwargs(per_scenario_data, "ttft")
+    draw_bar_plot(ax1, ttft_bars, ttft_errors, "Time to first token (lower is better)", "TTFT (seconds)")
+
+    # TPOT Plot (right)
+    itl_bars, itl_errors = make_bar_kwargs(per_scenario_data, "itl")
+    draw_bar_plot(ax2, itl_bars, itl_errors, "Time per output token (lower is better)", "ITL (seconds)")
+
+    # Add common legend with full text
+    legend_labels = ttft_bars["label"]  # Use full labels without truncation
+    legend_handles = [plt.Rectangle((0,0),1,1, color=color) for color in ttft_bars["color"]]
+    fig.legend(legend_handles, legend_labels, loc='lower center', ncol=1,
+               bbox_to_anchor=(0.5, -0.05), facecolor='black', edgecolor='white',
+               labelcolor='white', fontsize=12)
+
+    # Tight layout with spacing between subplots and extra bottom space for legend
+    # plt.subplots_adjust(wspace=0.3, bottom=0.075)
+
+    # Save plot to bytes with high DPI for crisp text
+    buffer = io.BytesIO()
+    plt.savefig(buffer, format='png', facecolor='#000000',
+                bbox_inches='tight', dpi=150)
+    buffer.seek(0)
+
+    # Convert to base64 for HTML embedding
+    img_data = base64.b64encode(buffer.getvalue()).decode()
+    plt.close(fig)
+
+
+    # Return HTML with embedded image - full height
+    html = f"""
+    <div style="width: 100%; height: 100vh; background: #000; display: flex; justify-content: center; align-items: center;">
+        <img src="data:image/png;base64,{img_data}" style="width: 100%; height: 100%; object-fit: contain;" />
+    </div>
+    """
+    return html

data.py

@@ -1,13 +1,6 @@
 import json
 import numpy as np
-
-
-def estimate_from_measures(measures: list[float], estimator: str) -> float:
-    if estimator == "median":
-        return float(np.median(measures))
-    elif estimator == "mean":
-        return float(np.mean(measures))
-    raise ValueError(f"Invalid estimator: {estimator}")
+from typing import Optional
 
 
 class ModelBenchmarkData:
@@ -16,16 +9,25 @@ class ModelBenchmarkData:
         with open(json_path, "r") as f:
             self.data = json.load(f)
 
-    def get_ttft_tpot_data(self, estimator: str = "median", use_cuda_time: bool = False) -> dict:
-        aggregated_data = {"ttft": [], "tpot": [], "label": [], "position": []}
-        time_key = "cuda_time" if use_cuda_time else "wall_time"
-        position = 0
-        for cfg_name, data in self.data.items():
-            x_measures = [d[time_key] for d in data["ttft"]]
-            y_measures = [d[time_key] for d in data["tpot"]]
-            aggregated_data["ttft"].append(estimate_from_measures(x_measures, estimator))
-            aggregated_data["tpot"].append(estimate_from_measures(y_measures, estimator))
-            aggregated_data["label"].append(cfg_name)
-            aggregated_data["position"].append(position)
-            position += 1
-        return aggregated_data
+    def compute_e2e_latency(self, measures: dict) -> tuple[float, Optional[float]]:
+        return measures["e2e_latency"]
+
+    def compute_ttft(self, measures: dict) -> float:
+        return measures["t_tokens"][0] - measures["wall_time_start"]
+
+    def compute_itl(self, measures: dict) -> Optional[float]:
+        if len(measures["t_tokens"]) < 2:
+            return None
+        delta_t = measures["t_tokens"][-1] - measures["t_tokens"][0]
+        num_tokens = len(measures["t_tokens"]) - 1
+        return delta_t / num_tokens
+
+    def get_bar_plot_data(self) -> dict:
+        per_scenario_data = {}
+        for i, (cfg_name, data) in enumerate(self.data.items()):
+            per_scenario_data[cfg_name] = {
+                "ttft": [self.compute_ttft(d) for d in data["measures"]],
+                "itl": [self.compute_itl(d) for d in data["measures"]],
+                "config": data["metadata"]["config"],
+            }
+        return per_scenario_data