init

.gitignore

@@ -0,0 +1,4 @@
+.vscode/
+.env/
+debug/
+__pycache__/

app.py

@@ -0,0 +1,222 @@
+import json
+from dataclasses import dataclass, field, fields
+from functools import cached_property
+from pathlib import Path
+from typing import Literal
+from functools import partial
+from content import *
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+import pandas as pd
+
+# This dataframe must contain the following columns:
+# - model: the name of the model
+# - language: the language of the model
+# - dataset: the dataset used to evaluate the model
+# - score: the score of the model on the (language, dataset) pair
+# - model_type: the type of the model (e.g. "Chat Model", "Base Model")
+df = pd.read_csv("data/raw_scores.csv")
+choices_language = list(df["language"].unique())
+choices_dataset = list(df["dataset"].unique())
+choices_model_type = list(df["model_type"].unique())
+
+# Utility functions for data processing
+reduce_functions = {
+    "Mean": lambda x: np.mean(x),
+    "Median": lambda x: np.median(x),
+    "Max": lambda x: np.max(x),
+    "Min": lambda x: np.min(x),
+}
+map_functions = {
+    "Raw": lambda x: x,
+    "Rank": partial(pd.Series.rank, ascending=False, method="dense"),
+    "Normalize": lambda x: (x - np.min(x)) / (np.max(x) - np.min(x)),
+}
+score_ascending = {
+    "Raw": False,
+    "Rank": True,
+    "Normalize": False,
+}
+
+
+def prepare_dataframe(
+    df: pd.DataFrame,
+    filters: dict[str, list[str]],
+    group_by: Literal["language", "dataset"],
+    map_function: str,
+    reduce_function: str,
+) -> pd.DataFrame:
+    # Filters contains a value subset for each column
+    language = filters["language"]
+    dataset = filters["dataset"]
+
+    # Columns are divided into two groups: other_columns and group_by_columns, apart from `model` which should be the index
+    other_columns = list(set(df.columns) - set(["language", "dataset", "score"]))
+    group_by_columns = filters[group_by]
+
+    # Step 1: Filter the dataframe based on the selected language and dataset
+    for k, v in filters.items():
+        df = df[df[k].isin(v)]
+    # If dataframe is empty, return an empty dataframe
+    if len(df) == 0:
+        gr.Warning(
+            f"No scores remain after the filter application. Please verify the checkboxes."
+        )
+        return pd.DataFrame(columns=other_columns)
+    # Sanity check: All score exists for each (language,dataset) pair
+    score_count = (
+        df.drop_duplicates(subset=["model", "language", "dataset"])
+        .groupby(["model"])["score"]
+        .count()
+    )
+    invalid_models = score_count[
+        score_count < len(language) * len(dataset)
+    ].index.tolist()
+    df = df[~df["model"].isin(invalid_models)]
+    # Send a warning message if there are any invalid models
+    for model in invalid_models:
+        gr.Warning(
+            f"<strong>{model}</strong> is lacking some scores thus hidden. Please report to the maintainers."
+        )
+
+    # Step 2: Process Scores
+    # Step 2.0: Map the scores along each (language, dataset) pair
+    df["score"] = df.groupby(["language", "dataset"])["score"].transform(
+        map_functions[map_function]
+    )
+    # Step 2.1: Reduce the scores along the column other than `group_by`
+    df = (
+        df.groupby(other_columns + [group_by])
+        .agg({"score": reduce_functions[reduce_function]})
+        .reset_index()
+    )
+    # Step 2.2: Reduce the scores along `group_by` to get the overall score of each model
+    reduced_col = df.groupby(other_columns).agg(
+        {"score": reduce_functions[reduce_function]}
+    )["score"]
+    # Step 2.3: Pivot the dataframe, then concat the overall score
+    df = df.pivot(index=other_columns, columns=group_by, values=["score"]).droplevel(
+        0, 1
+    )
+    df["Overall Score"] = reduced_col
+
+    # Step 3: Styling for display
+    # - Sort the dataframe by the reduced score
+    # - Sort the columns for better readability
+    # - Highlight the maximum value in each column
+    # - Format the score to 2 decimal places if it is a float
+    other_columns.remove("model")
+    df = (
+        df.reset_index()[
+            ["model"] + other_columns + ["Overall Score"] + group_by_columns
+        ]
+        .sort_values(by="Overall Score", ascending=score_ascending[map_function])
+        .style.format(precision=2)
+    )
+    if score_ascending[map_function]:
+        df = df.highlight_min(
+            axis=0, color="#18864B", subset=["Overall Score"] + group_by_columns
+        )
+    else:
+        df = df.highlight_max(
+            axis=0, color="#18864B", subset=["Overall Score"] + group_by_columns
+        )
+    return df
+
+
+with gr.Blocks(theme=gr.themes.Base()) as demo:
+
+    gr.Markdown(
+        MARKDOWN_HEADER
+    )
+
+    with gr.Column():
+        # UI definition
+        checkbox_language = gr.CheckboxGroup(
+            choices=choices_language,
+            value=choices_language,
+            label="Language(s)",
+            interactive=True,
+        )
+        checkbox_dataset = gr.CheckboxGroup(
+            choices=choices_dataset,
+            value=choices_dataset,
+            label="Dataset(s)",
+            interactive=True,
+        )
+        checkbox_model_type = gr.CheckboxGroup(
+            choices=choices_model_type,
+            value=choices_model_type,
+            label="Model Type(s)",
+            interactive=True,
+        )
+        dropdown_map_function = gr.Dropdown(
+            choices=map_functions.keys(),
+            value="Raw",
+            label="Map Function",
+            interactive=True,
+            info=MARKDOWN_MAP_FUNCTION
+        )
+        dropdown_reduce_function = gr.Dropdown(
+            choices=reduce_functions.keys(),
+            value="Mean",
+            label="Reduce Function",
+            interactive=True,
+            info=MARKDOWN_REDUCE_FUNCTION
+        )
+        ratio_group_by = gr.Radio(
+            choices=["language", "dataset"],
+            value="language",
+            label="Group by",
+            interactive=True,
+        )
+
+        dataframe = gr.DataFrame(
+            prepare_dataframe(
+                df=df,
+                filters={
+                    "language": choices_language,
+                    "dataset": choices_dataset,
+                },
+                group_by="language",
+                map_function="Raw",
+                reduce_function="Mean",
+            )
+        )
+        # Event listeners
+        gr.on(
+            triggers=[
+                checkbox_model_type.change,
+                checkbox_language.change,
+                checkbox_dataset.change,
+                ratio_group_by.change,
+                dropdown_reduce_function.change,
+                dropdown_map_function.change,
+            ],
+            fn=lambda model_type, language, dataset, group_by, map_function, reduce_function: prepare_dataframe(
+                df=df,
+                filters={
+                    "language": language,
+                    "dataset": dataset,
+                    "model_type": model_type,
+                },
+                group_by=group_by,
+                map_function=map_function,
+                reduce_function=reduce_function,
+            ),
+            inputs=[
+                checkbox_model_type,
+                checkbox_language,
+                checkbox_dataset,
+                ratio_group_by,
+                dropdown_map_function,
+                dropdown_reduce_function,
+            ],
+            outputs=[dataframe],
+        )
+
+if __name__ == "__main__":
+    demo.launch(debug=True, server_port=7899)

content.py

@@ -0,0 +1,20 @@
+MARKDOWN_HEADER = (
+    "# M-YASS\n<u>M</u>ultilingual - <u>Y</u>et <u>A</u>nother <u>S</u>ynergistic <u>S</u>core\n"
+    "1. Filter: a subset of Language, Dataset and Model Type;\n"
+    "2. Map Function: Each score will be mapped within each `<language,dataset>` pair. Can be used to get rank or normalize score;\n"
+    "3. Reduce Function: An array of scores will be reduced to one. Can be used to get the average score;\n"
+    "4. Group By: Language or Dataset. If `language` is selected, the score of all tasks within this language will be reduced by `Reduce Function`\n"
+)
+
+MARKDOWN_MAP_FUNCTION = (
+    "Raw: `lambda x: x`\n"
+    "Rank: `partial(pd.Series.rank, ascending=False, method='dense')`\n"
+    "Normalize: `lambda x: (x - np.min(x)) / (np.max(x) - np.min(x))`\n"
+)
+
+MARKDOWN_REDUCE_FUNCTION = (
+    "Mean: `lambda x: np.mean(x)`\n"
+    "Median: `lambda x: np.median(x)`\n"
+    "Max: `lambda x: np.max(x)`\n"
+    "Min: `lambda x: np.min(x)`\n"
+)

data/raw_scores.csv

@@ -0,0 +1,721 @@
+model,model_type,language,dataset,score
+Qwen2.5-Coder-32B-Instruct,Chat Model,English,Interpreter,83.2
+Qwen2.5-Coder-32B-Instruct,Chat Model,Chinese,Interpreter,76.05
+Qwen2.5-Coder-32B-Instruct,Chat Model,Japanese,Interpreter,73.59
+Qwen2.5-Coder-32B-Instruct,Chat Model,French,Interpreter,61.54
+Qwen2.5-Coder-32B-Instruct,Chat Model,English,Script,58.15
+Qwen2.5-Coder-32B-Instruct,Chat Model,Chinese,Script,26.98
+Qwen2.5-Coder-32B-Instruct,Chat Model,Japanese,Script,46.04
+Qwen2.5-Coder-32B-Instruct,Chat Model,French,Script,52.73
+Qwen2.5-Coder-32B-Instruct,Chat Model,English,Compile,58.12
+Qwen2.5-Coder-32B-Instruct,Chat Model,Chinese,Compile,73.69
+Qwen2.5-Coder-32B-Instruct,Chat Model,Japanese,Compile,81.95
+Qwen2.5-Coder-32B-Instruct,Chat Model,French,Compile,68.53
+Qwen2.5-Coder-32B,Base Model,English,Interpreter,57.1
+Qwen2.5-Coder-32B,Base Model,Chinese,Interpreter,65.07
+Qwen2.5-Coder-32B,Base Model,Japanese,Interpreter,62.7
+Qwen2.5-Coder-32B,Base Model,French,Interpreter,47.93
+Qwen2.5-Coder-32B,Base Model,English,Script,51.28
+Qwen2.5-Coder-32B,Base Model,Chinese,Script,32.3
+Qwen2.5-Coder-32B,Base Model,Japanese,Script,57.25
+Qwen2.5-Coder-32B,Base Model,French,Script,31.2
+Qwen2.5-Coder-32B,Base Model,English,Compile,60.21
+Qwen2.5-Coder-32B,Base Model,Chinese,Compile,65.49
+Qwen2.5-Coder-32B,Base Model,Japanese,Compile,64.35
+Qwen2.5-Coder-32B,Base Model,French,Compile,65.94
+OpenCodeInterpreter-DS-33B,Chat Model,English,Interpreter,75.23
+OpenCodeInterpreter-DS-33B,Chat Model,Chinese,Interpreter,69.06
+OpenCodeInterpreter-DS-33B,Chat Model,Japanese,Interpreter,59.32
+OpenCodeInterpreter-DS-33B,Chat Model,French,Interpreter,34.55
+OpenCodeInterpreter-DS-33B,Chat Model,English,Script,46.58
+OpenCodeInterpreter-DS-33B,Chat Model,Chinese,Script,22.31
+OpenCodeInterpreter-DS-33B,Chat Model,Japanese,Script,57.76
+OpenCodeInterpreter-DS-33B,Chat Model,French,Script,45.94
+OpenCodeInterpreter-DS-33B,Chat Model,English,Compile,51.85
+OpenCodeInterpreter-DS-33B,Chat Model,Chinese,Compile,54.8
+OpenCodeInterpreter-DS-33B,Chat Model,Japanese,Compile,64.47
+OpenCodeInterpreter-DS-33B,Chat Model,French,Compile,57.95
+Nxcode-CQ-7B-orpo,Chat Model,English,Interpreter,87.23
+Nxcode-CQ-7B-orpo,Chat Model,Chinese,Interpreter,71.69
+Nxcode-CQ-7B-orpo,Chat Model,Japanese,Interpreter,67.98
+Nxcode-CQ-7B-orpo,Chat Model,French,Interpreter,32.26
+Nxcode-CQ-7B-orpo,Chat Model,English,Script,50.54
+Nxcode-CQ-7B-orpo,Chat Model,Chinese,Script,23.22
+Nxcode-CQ-7B-orpo,Chat Model,Japanese,Script,37.24
+Nxcode-CQ-7B-orpo,Chat Model,French,Script,39.73
+Nxcode-CQ-7B-orpo,Chat Model,English,Compile,58.51
+Nxcode-CQ-7B-orpo,Chat Model,Chinese,Compile,60.91
+Nxcode-CQ-7B-orpo,Chat Model,Japanese,Compile,68.04
+Nxcode-CQ-7B-orpo,Chat Model,French,Compile,65.53
+CodeQwen1.5-7B-Chat,Chat Model,English,Interpreter,87.2
+CodeQwen1.5-7B-Chat,Chat Model,Chinese,Interpreter,70.31
+CodeQwen1.5-7B-Chat,Chat Model,Japanese,Interpreter,69.22
+CodeQwen1.5-7B-Chat,Chat Model,French,Interpreter,29.38
+CodeQwen1.5-7B-Chat,Chat Model,English,Script,51.89
+CodeQwen1.5-7B-Chat,Chat Model,Chinese,Script,22.71
+CodeQwen1.5-7B-Chat,Chat Model,Japanese,Script,32.89
+CodeQwen1.5-7B-Chat,Chat Model,French,Script,39.86
+CodeQwen1.5-7B-Chat,Chat Model,English,Compile,59.59
+CodeQwen1.5-7B-Chat,Chat Model,Chinese,Compile,61.04
+CodeQwen1.5-7B-Chat,Chat Model,Japanese,Compile,67.85
+CodeQwen1.5-7B-Chat,Chat Model,French,Compile,67.49
+CodeFuse-DeepSeek-33b,Chat Model,English,Interpreter,76.83
+CodeFuse-DeepSeek-33b,Chat Model,Chinese,Interpreter,66.46
+CodeFuse-DeepSeek-33b,Chat Model,Japanese,Interpreter,57.76
+CodeFuse-DeepSeek-33b,Chat Model,French,Interpreter,34.16
+CodeFuse-DeepSeek-33b,Chat Model,English,Script,38.36
+CodeFuse-DeepSeek-33b,Chat Model,Chinese,Script,24.36
+CodeFuse-DeepSeek-33b,Chat Model,Japanese,Script,52.8
+CodeFuse-DeepSeek-33b,Chat Model,French,Script,40.37
+CodeFuse-DeepSeek-33b,Chat Model,English,Compile,49.37
+CodeFuse-DeepSeek-33b,Chat Model,Chinese,Compile,60.76
+CodeFuse-DeepSeek-33b,Chat Model,Japanese,Compile,65.22
+CodeFuse-DeepSeek-33b,Chat Model,French,Compile,53.85
+DeepSeek-Coder-33b-instruct,Chat Model,English,Interpreter,80.02
+DeepSeek-Coder-33b-instruct,Chat Model,Chinese,Interpreter,65.13
+DeepSeek-Coder-33b-instruct,Chat Model,Japanese,Interpreter,52.5
+DeepSeek-Coder-33b-instruct,Chat Model,French,Interpreter,31.69
+DeepSeek-Coder-33b-instruct,Chat Model,English,Script,42.92
+DeepSeek-Coder-33b-instruct,Chat Model,Chinese,Script,17.85
+DeepSeek-Coder-33b-instruct,Chat Model,Japanese,Script,50.92
+DeepSeek-Coder-33b-instruct,Chat Model,French,Script,39.43
+DeepSeek-Coder-33b-instruct,Chat Model,English,Compile,49.42
+DeepSeek-Coder-33b-instruct,Chat Model,Chinese,Compile,52.03
+DeepSeek-Coder-33b-instruct,Chat Model,Japanese,Compile,62.36
+DeepSeek-Coder-33b-instruct,Chat Model,French,Compile,55.56
+Artigenz-Coder-DS-6.7B,Chat Model,English,Interpreter,70.89
+Artigenz-Coder-DS-6.7B,Chat Model,Chinese,Interpreter,66.16
+Artigenz-Coder-DS-6.7B,Chat Model,Japanese,Interpreter,55.24
+Artigenz-Coder-DS-6.7B,Chat Model,French,Interpreter,30.15
+Artigenz-Coder-DS-6.7B,Chat Model,English,Script,41.94
+Artigenz-Coder-DS-6.7B,Chat Model,Chinese,Script,20.09
+Artigenz-Coder-DS-6.7B,Chat Model,Japanese,Script,44.56
+Artigenz-Coder-DS-6.7B,Chat Model,French,Script,39.94
+Artigenz-Coder-DS-6.7B,Chat Model,English,Compile,43.91
+Artigenz-Coder-DS-6.7B,Chat Model,Chinese,Compile,56.84
+Artigenz-Coder-DS-6.7B,Chat Model,Japanese,Compile,59.75
+Artigenz-Coder-DS-6.7B,Chat Model,French,Compile,48.77
+DeepSeek-Coder-7b-instruct,Chat Model,English,Interpreter,80.22
+DeepSeek-Coder-7b-instruct,Chat Model,Chinese,Interpreter,65.8
+DeepSeek-Coder-7b-instruct,Chat Model,Japanese,Interpreter,59.4
+DeepSeek-Coder-7b-instruct,Chat Model,French,Interpreter,20.87
+DeepSeek-Coder-7b-instruct,Chat Model,English,Script,38.84
+DeepSeek-Coder-7b-instruct,Chat Model,Chinese,Script,21.59
+DeepSeek-Coder-7b-instruct,Chat Model,Japanese,Script,47.78
+DeepSeek-Coder-7b-instruct,Chat Model,French,Script,38.56
+DeepSeek-Coder-7b-instruct,Chat Model,English,Compile,44.22
+DeepSeek-Coder-7b-instruct,Chat Model,Chinese,Compile,53.34
+DeepSeek-Coder-7b-instruct,Chat Model,Japanese,Compile,59.66
+DeepSeek-Coder-7b-instruct,Chat Model,French,Compile,47.73
+OpenCodeInterpreter-DS-6.7B,Chat Model,English,Interpreter,73.2
+OpenCodeInterpreter-DS-6.7B,Chat Model,Chinese,Interpreter,63.85
+OpenCodeInterpreter-DS-6.7B,Chat Model,Japanese,Interpreter,57.34
+OpenCodeInterpreter-DS-6.7B,Chat Model,French,Interpreter,24.32
+OpenCodeInterpreter-DS-6.7B,Chat Model,English,Script,39.69
+OpenCodeInterpreter-DS-6.7B,Chat Model,Chinese,Script,18.22
+OpenCodeInterpreter-DS-6.7B,Chat Model,Japanese,Script,44.3
+OpenCodeInterpreter-DS-6.7B,Chat Model,French,Script,39.08
+OpenCodeInterpreter-DS-6.7B,Chat Model,English,Compile,45.99
+OpenCodeInterpreter-DS-6.7B,Chat Model,Chinese,Compile,51.41
+OpenCodeInterpreter-DS-6.7B,Chat Model,Japanese,Compile,60.01
+OpenCodeInterpreter-DS-6.7B,Chat Model,French,Compile,48.22
+Phind-CodeLlama-34B-v2,Chat Model,English,Interpreter,71.95
+Phind-CodeLlama-34B-v2,Chat Model,Chinese,Interpreter,65.34
+Phind-CodeLlama-34B-v2,Chat Model,Japanese,Interpreter,56.26
+Phind-CodeLlama-34B-v2,Chat Model,French,Interpreter,28.7
+Phind-CodeLlama-34B-v2,Chat Model,English,Script,45.12
+Phind-CodeLlama-34B-v2,Chat Model,Chinese,Script,14.12
+Phind-CodeLlama-34B-v2,Chat Model,Japanese,Script,44.27
+Phind-CodeLlama-34B-v2,Chat Model,French,Script,37.7
+Phind-CodeLlama-34B-v2,Chat Model,English,Compile,49.63
+Phind-CodeLlama-34B-v2,Chat Model,Chinese,Compile,54.06
+Phind-CodeLlama-34B-v2,Chat Model,Japanese,Compile,59.59
+Phind-CodeLlama-34B-v2,Chat Model,French,Compile,57.67
+Phind-CodeLlama-34B-v1,Chat Model,English,Interpreter,65.85
+Phind-CodeLlama-34B-v1,Chat Model,Chinese,Interpreter,64.45
+Phind-CodeLlama-34B-v1,Chat Model,Japanese,Interpreter,55.53
+Phind-CodeLlama-34B-v1,Chat Model,French,Interpreter,24.89
+Phind-CodeLlama-34B-v1,Chat Model,English,Script,43.23
+Phind-CodeLlama-34B-v1,Chat Model,Chinese,Script,15.5
+Phind-CodeLlama-34B-v1,Chat Model,Japanese,Script,42.05
+Phind-CodeLlama-34B-v1,Chat Model,French,Script,36.71
+Phind-CodeLlama-34B-v1,Chat Model,English,Compile,53.27
+Phind-CodeLlama-34B-v1,Chat Model,Chinese,Compile,49.47
+Phind-CodeLlama-34B-v1,Chat Model,Japanese,Compile,57.81
+Phind-CodeLlama-34B-v1,Chat Model,French,Compile,54.1
+Phind-CodeLlama-34B-Python-v1,Chat Model,English,Interpreter,70.22
+Phind-CodeLlama-34B-Python-v1,Chat Model,Chinese,Interpreter,66.24
+Phind-CodeLlama-34B-Python-v1,Chat Model,Japanese,Interpreter,52.05
+Phind-CodeLlama-34B-Python-v1,Chat Model,French,Interpreter,18.88
+Phind-CodeLlama-34B-Python-v1,Chat Model,English,Script,44.23
+Phind-CodeLlama-34B-Python-v1,Chat Model,Chinese,Script,13.78
+Phind-CodeLlama-34B-Python-v1,Chat Model,Japanese,Script,39.44
+Phind-CodeLlama-34B-Python-v1,Chat Model,French,Script,37.76
+Phind-CodeLlama-34B-Python-v1,Chat Model,English,Compile,47.11
+Phind-CodeLlama-34B-Python-v1,Chat Model,Chinese,Compile,48.72
+Phind-CodeLlama-34B-Python-v1,Chat Model,Japanese,Compile,55.34
+Phind-CodeLlama-34B-Python-v1,Chat Model,French,Compile,49.22
+CodeQwen1.5-7B,Base Model,English,Interpreter,50.79
+CodeQwen1.5-7B,Base Model,Chinese,Interpreter,50.07
+CodeQwen1.5-7B,Base Model,Japanese,Interpreter,50.92
+CodeQwen1.5-7B,Base Model,French,Interpreter,24.89
+CodeQwen1.5-7B,Base Model,English,Script,37.27
+CodeQwen1.5-7B,Base Model,Chinese,Script,16.03
+CodeQwen1.5-7B,Base Model,Japanese,Script,46.29
+CodeQwen1.5-7B,Base Model,French,Script,27.95
+CodeQwen1.5-7B,Base Model,English,Compile,43.33
+CodeQwen1.5-7B,Base Model,Chinese,Compile,42.15
+CodeQwen1.5-7B,Base Model,Japanese,Compile,48.35
+CodeQwen1.5-7B,Base Model,French,Compile,47.56
+CodeLlama-70b-Instruct,Chat Model,English,Interpreter,75.6
+CodeLlama-70b-Instruct,Chat Model,Chinese,Interpreter,57.76
+CodeLlama-70b-Instruct,Chat Model,Japanese,Interpreter,57.14
+CodeLlama-70b-Instruct,Chat Model,French,Interpreter,0.0
+CodeLlama-70b-Instruct,Chat Model,English,Script,42.24
+CodeLlama-70b-Instruct,Chat Model,Chinese,Script,19.88
+CodeLlama-70b-Instruct,Chat Model,Japanese,Script,44.1
+CodeLlama-70b-Instruct,Chat Model,French,Script,29.19
+CodeLlama-70b-Instruct,Chat Model,English,Compile,42.86
+CodeLlama-70b-Instruct,Chat Model,Chinese,Compile,47.2
+CodeLlama-70b-Instruct,Chat Model,Japanese,Compile,48.45
+CodeLlama-70b-Instruct,Chat Model,French,Compile,47.2
+WizardCoder-Python-34B-V1.0,Chat Model,English,Interpreter,70.73
+WizardCoder-Python-34B-V1.0,Chat Model,Chinese,Interpreter,55.28
+WizardCoder-Python-34B-V1.0,Chat Model,Japanese,Interpreter,47.2
+WizardCoder-Python-34B-V1.0,Chat Model,French,Interpreter,18.63
+WizardCoder-Python-34B-V1.0,Chat Model,English,Script,41.51
+WizardCoder-Python-34B-V1.0,Chat Model,Chinese,Script,15.38
+WizardCoder-Python-34B-V1.0,Chat Model,Japanese,Script,32.3
+WizardCoder-Python-34B-V1.0,Chat Model,French,Script,39.75
+WizardCoder-Python-34B-V1.0,Chat Model,English,Compile,44.3
+WizardCoder-Python-34B-V1.0,Chat Model,Chinese,Compile,44.94
+WizardCoder-Python-34B-V1.0,Chat Model,Japanese,Compile,47.2
+WizardCoder-Python-34B-V1.0,Chat Model,French,Compile,46.15
+CodeLlama-70b,Base Model,English,Interpreter,52.44
+CodeLlama-70b,Base Model,Chinese,Interpreter,56.52
+CodeLlama-70b,Base Model,Japanese,Interpreter,46.58
+CodeLlama-70b,Base Model,French,Interpreter,0.0
+CodeLlama-70b,Base Model,English,Script,42.24
+CodeLlama-70b,Base Model,Chinese,Script,24.84
+CodeLlama-70b,Base Model,Japanese,Script,41.61
+CodeLlama-70b,Base Model,French,Script,27.95
+CodeLlama-70b,Base Model,English,Compile,42.86
+CodeLlama-70b,Base Model,Chinese,Compile,44.72
+CodeLlama-70b,Base Model,Japanese,Compile,49.69
+CodeLlama-70b,Base Model,French,Compile,49.69
+DeepSeek-Coder-33b-base,Base Model,English,Interpreter,52.45
+DeepSeek-Coder-33b-base,Base Model,Chinese,Interpreter,51.28
+DeepSeek-Coder-33b-base,Base Model,Japanese,Interpreter,41.76
+DeepSeek-Coder-33b-base,Base Model,French,Interpreter,23.37
+DeepSeek-Coder-33b-base,Base Model,English,Script,32.83
+DeepSeek-Coder-33b-base,Base Model,Chinese,Script,17.41
+DeepSeek-Coder-33b-base,Base Model,Japanese,Script,36.51
+DeepSeek-Coder-33b-base,Base Model,French,Script,26.76
+DeepSeek-Coder-33b-base,Base Model,English,Compile,35.75
+DeepSeek-Coder-33b-base,Base Model,Chinese,Compile,43.77
+DeepSeek-Coder-33b-base,Base Model,Japanese,Compile,51.22
+DeepSeek-Coder-33b-base,Base Model,French,Compile,43.78
+CodeLlama-70b-Python,Base Model,English,Interpreter,55.49
+CodeLlama-70b-Python,Base Model,Chinese,Interpreter,56.52
+CodeLlama-70b-Python,Base Model,Japanese,Interpreter,52.8
+CodeLlama-70b-Python,Base Model,French,Interpreter,0.0
+CodeLlama-70b-Python,Base Model,English,Script,35.4
+CodeLlama-70b-Python,Base Model,Chinese,Script,21.12
+CodeLlama-70b-Python,Base Model,Japanese,Script,44.72
+CodeLlama-70b-Python,Base Model,French,Script,26.09
+CodeLlama-70b-Python,Base Model,English,Compile,39.13
+CodeLlama-70b-Python,Base Model,Chinese,Compile,45.96
+CodeLlama-70b-Python,Base Model,Japanese,Compile,49.69
+CodeLlama-70b-Python,Base Model,French,Compile,48.45
+StarCoder2-15B,Base Model,English,Interpreter,44.15
+StarCoder2-15B,Base Model,Chinese,Interpreter,44.24
+StarCoder2-15B,Base Model,Japanese,Interpreter,39.48
+StarCoder2-15B,Base Model,French,Interpreter,22.41
+StarCoder2-15B,Base Model,English,Script,33.19
+StarCoder2-15B,Base Model,Chinese,Script,23.64
+StarCoder2-15B,Base Model,Japanese,Script,43.75
+StarCoder2-15B,Base Model,French,Script,19.81
+StarCoder2-15B,Base Model,English,Compile,34.18
+StarCoder2-15B,Base Model,Chinese,Compile,33.86
+StarCoder2-15B,Base Model,Japanese,Compile,41.44
+StarCoder2-15B,Base Model,French,Compile,38.03
+CodeLlama-34b-Instruct,Chat Model,English,Interpreter,50.79
+CodeLlama-34b-Instruct,Chat Model,Chinese,Interpreter,45.85
+CodeLlama-34b-Instruct,Chat Model,Japanese,Interpreter,36.98
+CodeLlama-34b-Instruct,Chat Model,French,Interpreter,18.09
+CodeLlama-34b-Instruct,Chat Model,English,Script,32.65
+CodeLlama-34b-Instruct,Chat Model,Chinese,Script,13.63
+CodeLlama-34b-Instruct,Chat Model,Japanese,Script,38.87
+CodeLlama-34b-Instruct,Chat Model,French,Script,24.25
+CodeLlama-34b-Instruct,Chat Model,English,Compile,37.63
+CodeLlama-34b-Instruct,Chat Model,Chinese,Compile,41.53
+CodeLlama-34b-Instruct,Chat Model,Japanese,Compile,41.53
+CodeLlama-34b-Instruct,Chat Model,French,Compile,39.26
+WizardCoder-Python-13B-V1.0,Chat Model,English,Interpreter,62.19
+WizardCoder-Python-13B-V1.0,Chat Model,Chinese,Interpreter,48.45
+WizardCoder-Python-13B-V1.0,Chat Model,Japanese,Interpreter,42.24
+WizardCoder-Python-13B-V1.0,Chat Model,French,Interpreter,16.15
+WizardCoder-Python-13B-V1.0,Chat Model,English,Script,38.99
+WizardCoder-Python-13B-V1.0,Chat Model,Chinese,Script,11.54
+WizardCoder-Python-13B-V1.0,Chat Model,Japanese,Script,32.92
+WizardCoder-Python-13B-V1.0,Chat Model,French,Script,27.33
+WizardCoder-Python-13B-V1.0,Chat Model,English,Compile,32.28
+WizardCoder-Python-13B-V1.0,Chat Model,Chinese,Compile,41.77
+WizardCoder-Python-13B-V1.0,Chat Model,Japanese,Compile,42.86
+WizardCoder-Python-13B-V1.0,Chat Model,French,Compile,34.62
+DeepSeek-Coder-7b-base,Base Model,English,Interpreter,45.83
+DeepSeek-Coder-7b-base,Base Model,Chinese,Interpreter,45.9
+DeepSeek-Coder-7b-base,Base Model,Japanese,Interpreter,36.92
+DeepSeek-Coder-7b-base,Base Model,French,Interpreter,18.73
+DeepSeek-Coder-7b-base,Base Model,English,Script,28.74
+DeepSeek-Coder-7b-base,Base Model,Chinese,Script,19.74
+DeepSeek-Coder-7b-base,Base Model,Japanese,Script,33.89
+DeepSeek-Coder-7b-base,Base Model,French,Script,28.99
+DeepSeek-Coder-7b-base,Base Model,English,Compile,25.8
+DeepSeek-Coder-7b-base,Base Model,Chinese,Compile,37.72
+DeepSeek-Coder-7b-base,Base Model,Japanese,Compile,45.53
+DeepSeek-Coder-7b-base,Base Model,French,Compile,34.67
+CodeLlama-34b,Base Model,English,Interpreter,45.11
+CodeLlama-34b,Base Model,Chinese,Interpreter,41.66
+CodeLlama-34b,Base Model,Japanese,Interpreter,40.43
+CodeLlama-34b,Base Model,French,Interpreter,16.94
+CodeLlama-34b,Base Model,English,Script,31.4
+CodeLlama-34b,Base Model,Chinese,Script,15.27
+CodeLlama-34b,Base Model,Japanese,Script,37.49
+CodeLlama-34b,Base Model,French,Script,22.71
+CodeLlama-34b,Base Model,English,Compile,35.28
+CodeLlama-34b,Base Model,Chinese,Compile,40.19
+CodeLlama-34b,Base Model,Japanese,Compile,41.42
+CodeLlama-34b,Base Model,French,Compile,38.73
+CodeLlama-34b-Python,Base Model,English,Interpreter,53.29
+CodeLlama-34b-Python,Base Model,Chinese,Interpreter,44.72
+CodeLlama-34b-Python,Base Model,Japanese,Interpreter,39.78
+CodeLlama-34b-Python,Base Model,French,Interpreter,13.19
+CodeLlama-34b-Python,Base Model,English,Script,31.37
+CodeLlama-34b-Python,Base Model,Chinese,Script,17.29
+CodeLlama-34b-Python,Base Model,Japanese,Script,31.9
+CodeLlama-34b-Python,Base Model,French,Script,22.35
+CodeLlama-34b-Python,Base Model,English,Compile,34.3
+CodeLlama-34b-Python,Base Model,Chinese,Compile,39.46
+CodeLlama-34b-Python,Base Model,Japanese,Compile,39.09
+CodeLlama-34b-Python,Base Model,French,Compile,39.67
+WizardCoder-15B-V1.0,Chat Model,English,Interpreter,58.12
+WizardCoder-15B-V1.0,Chat Model,Chinese,Interpreter,41.91
+WizardCoder-15B-V1.0,Chat Model,Japanese,Interpreter,39.34
+WizardCoder-15B-V1.0,Chat Model,French,Interpreter,13.39
+WizardCoder-15B-V1.0,Chat Model,English,Script,33.98
+WizardCoder-15B-V1.0,Chat Model,Chinese,Script,12.14
+WizardCoder-15B-V1.0,Chat Model,Japanese,Script,27.85
+WizardCoder-15B-V1.0,Chat Model,French,Script,22.53
+WizardCoder-15B-V1.0,Chat Model,English,Compile,27.06
+WizardCoder-15B-V1.0,Chat Model,Chinese,Compile,35.77
+WizardCoder-15B-V1.0,Chat Model,Japanese,Compile,38.95
+WizardCoder-15B-V1.0,Chat Model,French,Compile,33.74
+CodeLlama-13b-Instruct,Chat Model,English,Interpreter,50.6
+CodeLlama-13b-Instruct,Chat Model,Chinese,Interpreter,40.92
+CodeLlama-13b-Instruct,Chat Model,Japanese,Interpreter,32.07
+CodeLlama-13b-Instruct,Chat Model,French,Interpreter,16.66
+CodeLlama-13b-Instruct,Chat Model,English,Script,32.23
+CodeLlama-13b-Instruct,Chat Model,Chinese,Script,16.29
+CodeLlama-13b-Instruct,Chat Model,Japanese,Script,31.6
+CodeLlama-13b-Instruct,Chat Model,French,Script,20.14
+CodeLlama-13b-Instruct,Chat Model,English,Compile,31.75
+CodeLlama-13b-Instruct,Chat Model,Chinese,Compile,33.99
+CodeLlama-13b-Instruct,Chat Model,Japanese,Compile,36.36
+CodeLlama-13b-Instruct,Chat Model,French,Compile,32.82
+CodeGemma-7B,Base Model,English,Interpreter,40.13
+CodeGemma-7B,Base Model,Chinese,Interpreter,43.06
+CodeGemma-7B,Base Model,Japanese,Interpreter,34.76
+CodeGemma-7B,Base Model,French,Interpreter,13.55
+CodeGemma-7B,Base Model,English,Script,28.91
+CodeGemma-7B,Base Model,Chinese,Script,10.44
+CodeGemma-7B,Base Model,Japanese,Script,32.88
+CodeGemma-7B,Base Model,French,Script,20.96
+CodeGemma-7B,Base Model,English,Compile,32.62
+CodeGemma-7B,Base Model,Chinese,Compile,35.03
+CodeGemma-7B,Base Model,Japanese,Compile,40.34
+CodeGemma-7B,Base Model,French,Compile,34.56
+CodeLlama-13b,Base Model,English,Interpreter,35.07
+CodeLlama-13b,Base Model,Chinese,Interpreter,38.26
+CodeLlama-13b,Base Model,Japanese,Interpreter,32.57
+CodeLlama-13b,Base Model,French,Interpreter,13.63
+CodeLlama-13b,Base Model,English,Script,28.01
+CodeLlama-13b,Base Model,Chinese,Script,15.78
+CodeLlama-13b,Base Model,Japanese,Script,31.26
+CodeLlama-13b,Base Model,French,Script,18.32
+CodeLlama-13b,Base Model,English,Compile,29.54
+CodeLlama-13b,Base Model,Chinese,Compile,32.23
+CodeLlama-13b,Base Model,Japanese,Compile,35.81
+CodeLlama-13b,Base Model,French,Compile,29.72
+CodeLlama-13b-Python,Base Model,English,Interpreter,42.89
+CodeLlama-13b-Python,Base Model,Chinese,Interpreter,40.66
+CodeLlama-13b-Python,Base Model,Japanese,Interpreter,34.55
+CodeLlama-13b-Python,Base Model,French,Interpreter,12.51
+CodeLlama-13b-Python,Base Model,English,Script,30.4
+CodeLlama-13b-Python,Base Model,Chinese,Script,9.82
+CodeLlama-13b-Python,Base Model,Japanese,Script,29.9
+CodeLlama-13b-Python,Base Model,French,Script,18.35
+CodeLlama-13b-Python,Base Model,English,Compile,25.85
+CodeLlama-13b-Python,Base Model,Chinese,Compile,33.56
+CodeLlama-13b-Python,Base Model,Japanese,Compile,36.21
+CodeLlama-13b-Python,Base Model,French,Compile,29.32
+CodeLlama-7b-Instruct,Chat Model,English,Interpreter,45.65
+CodeLlama-7b-Instruct,Chat Model,Chinese,Interpreter,33.11
+CodeLlama-7b-Instruct,Chat Model,Japanese,Interpreter,28.55
+CodeLlama-7b-Instruct,Chat Model,French,Interpreter,11.81
+CodeLlama-7b-Instruct,Chat Model,English,Script,27.58
+CodeLlama-7b-Instruct,Chat Model,Chinese,Script,11.81
+CodeLlama-7b-Instruct,Chat Model,Japanese,Script,30.47
+CodeLlama-7b-Instruct,Chat Model,French,Script,19.7
+CodeLlama-7b-Instruct,Chat Model,English,Compile,26.66
+CodeLlama-7b-Instruct,Chat Model,Chinese,Compile,28.77
+CodeLlama-7b-Instruct,Chat Model,Japanese,Compile,29.03
+CodeLlama-7b-Instruct,Chat Model,French,Compile,24.27
+StarCoder2-7B,Base Model,English,Interpreter,34.09
+StarCoder2-7B,Base Model,Chinese,Interpreter,35.35
+StarCoder2-7B,Base Model,Japanese,Interpreter,30.58
+StarCoder2-7B,Base Model,French,Interpreter,11.58
+StarCoder2-7B,Base Model,English,Script,20.42
+StarCoder2-7B,Base Model,Chinese,Script,15.12
+StarCoder2-7B,Base Model,Japanese,Script,30.67
+StarCoder2-7B,Base Model,French,Script,16.72
+StarCoder2-7B,Base Model,English,Compile,26.06
+StarCoder2-7B,Base Model,Chinese,Compile,29.42
+StarCoder2-7B,Base Model,Japanese,Compile,33.63
+StarCoder2-7B,Base Model,French,Compile,29.62
+CodeGemma-7B-it,Chat Model,English,Interpreter,52.74
+CodeGemma-7B-it,Chat Model,Chinese,Interpreter,47.71
+CodeGemma-7B-it,Chat Model,Japanese,Interpreter,19.18
+CodeGemma-7B-it,Chat Model,French,Interpreter,9.22
+CodeGemma-7B-it,Chat Model,English,Script,19.56
+CodeGemma-7B-it,Chat Model,Chinese,Script,11.83
+CodeGemma-7B-it,Chat Model,Japanese,Script,22.02
+CodeGemma-7B-it,Chat Model,French,Script,21.64
+CodeGemma-7B-it,Chat Model,English,Compile,6.91
+CodeGemma-7B-it,Chat Model,Chinese,Compile,41.42
+CodeGemma-7B-it,Chat Model,Japanese,Compile,42.6
+CodeGemma-7B-it,Chat Model,French,Compile,38.31
+CodeShell-7B,Chat Model,English,Interpreter,34.32
+CodeShell-7B,Chat Model,Chinese,Interpreter,33.17
+CodeShell-7B,Chat Model,Japanese,Interpreter,30.87
+CodeShell-7B,Chat Model,French,Interpreter,17.2
+CodeShell-7B,Chat Model,English,Script,22.08
+CodeShell-7B,Chat Model,Chinese,Script,8.85
+CodeShell-7B,Chat Model,Japanese,Script,22.39
+CodeShell-7B,Chat Model,French,Script,20.52
+CodeShell-7B,Chat Model,English,Compile,24.3
+CodeShell-7B,Chat Model,Chinese,Compile,30.43
+CodeShell-7B,Chat Model,Japanese,Compile,28.21
+CodeShell-7B,Chat Model,French,Compile,24.55
+CodeLlama-7b,Base Model,English,Interpreter,29.98
+CodeLlama-7b,Base Model,Chinese,Interpreter,31.8
+CodeLlama-7b,Base Model,Japanese,Interpreter,25.17
+CodeLlama-7b,Base Model,French,Interpreter,11.94
+CodeLlama-7b,Base Model,English,Script,25.6
+CodeLlama-7b,Base Model,Chinese,Script,11.6
+CodeLlama-7b,Base Model,Japanese,Script,30.36
+CodeLlama-7b,Base Model,French,Script,18.04
+CodeLlama-7b,Base Model,English,Compile,25.52
+CodeLlama-7b,Base Model,Chinese,Compile,29.2
+CodeLlama-7b,Base Model,Japanese,Compile,27.23
+CodeLlama-7b,Base Model,French,Compile,25.82
+OctoCoder-15B,Chat Model,English,Interpreter,45.3
+OctoCoder-15B,Chat Model,Chinese,Interpreter,32.8
+OctoCoder-15B,Chat Model,Japanese,Interpreter,26.76
+OctoCoder-15B,Chat Model,French,Interpreter,10.61
+OctoCoder-15B,Chat Model,English,Script,24.5
+OctoCoder-15B,Chat Model,Chinese,Script,13.35
+OctoCoder-15B,Chat Model,Japanese,Script,22.56
+OctoCoder-15B,Chat Model,French,Script,14.39
+OctoCoder-15B,Chat Model,English,Compile,18.24
+OctoCoder-15B,Chat Model,Chinese,Compile,26.03
+OctoCoder-15B,Chat Model,Japanese,Compile,29.32
+OctoCoder-15B,Chat Model,French,Compile,24.26
+CodeLlama-7b-Python,Base Model,English,Interpreter,40.48
+CodeLlama-7b-Python,Base Model,Chinese,Interpreter,36.34
+CodeLlama-7b-Python,Base Model,Japanese,Interpreter,1.08
+CodeLlama-7b-Python,Base Model,French,Interpreter,9.04
+CodeLlama-7b-Python,Base Model,English,Script,28.53
+CodeLlama-7b-Python,Base Model,Chinese,Script,8.94
+CodeLlama-7b-Python,Base Model,Japanese,Script,26.15
+CodeLlama-7b-Python,Base Model,French,Script,18.25
+CodeLlama-7b-Python,Base Model,English,Compile,26.75
+CodeLlama-7b-Python,Base Model,Chinese,Compile,29.15
+CodeLlama-7b-Python,Base Model,Japanese,Compile,30.34
+CodeLlama-7b-Python,Base Model,French,Compile,26.96
+StarCoder-15B,Base Model,English,Interpreter,33.57
+StarCoder-15B,Base Model,Chinese,Interpreter,30.79
+StarCoder-15B,Base Model,Japanese,Interpreter,26.08
+StarCoder-15B,Base Model,French,Interpreter,0.07
+StarCoder-15B,Base Model,English,Script,23.02
+StarCoder-15B,Base Model,Chinese,Script,13.57
+StarCoder-15B,Base Model,Japanese,Script,23.89
+StarCoder-15B,Base Model,French,Script,15.5
+StarCoder-15B,Base Model,English,Compile,22.74
+StarCoder-15B,Base Model,Chinese,Compile,30.22
+StarCoder-15B,Base Model,Japanese,Compile,31.55
+StarCoder-15B,Base Model,French,Compile,21.84
+Falcon-180B,Base Model,English,Interpreter,35.37
+Falcon-180B,Base Model,Chinese,Interpreter,31.68
+Falcon-180B,Base Model,Japanese,Interpreter,
+Falcon-180B,Base Model,French,Interpreter,10.56
+Falcon-180B,Base Model,English,Script,24.53
+Falcon-180B,Base Model,Chinese,Script,14.1
+Falcon-180B,Base Model,Japanese,Script,26.71
+Falcon-180B,Base Model,French,Script,
+Falcon-180B,Base Model,English,Compile,15.82
+Falcon-180B,Base Model,Chinese,Compile,28.48
+Falcon-180B,Base Model,Japanese,Compile,28.57
+Falcon-180B,Base Model,French,Compile,25.0
+StarCoderBase-15B,Base Model,English,Interpreter,30.35
+StarCoderBase-15B,Base Model,Chinese,Interpreter,31.7
+StarCoderBase-15B,Base Model,Japanese,Interpreter,26.75
+StarCoderBase-15B,Base Model,French,Interpreter,11.77
+StarCoderBase-15B,Base Model,English,Script,21.09
+StarCoderBase-15B,Base Model,Chinese,Script,10.01
+StarCoderBase-15B,Base Model,Japanese,Script,26.61
+StarCoderBase-15B,Base Model,French,Script,10.18
+StarCoderBase-15B,Base Model,English,Compile,16.74
+StarCoderBase-15B,Base Model,Chinese,Compile,28.53
+StarCoderBase-15B,Base Model,Japanese,Compile,30.56
+StarCoderBase-15B,Base Model,French,Compile,24.46
+StarCoder2-3B,Base Model,English,Interpreter,31.44
+StarCoder2-3B,Base Model,Chinese,Interpreter,35.37
+StarCoder2-3B,Base Model,Japanese,Interpreter,27.61
+StarCoder2-3B,Base Model,French,Interpreter,7.8
+StarCoder2-3B,Base Model,English,Script,19.87
+StarCoder2-3B,Base Model,Chinese,Script,12.56
+StarCoder2-3B,Base Model,Japanese,Script,28.01
+StarCoder2-3B,Base Model,French,Script,14.22
+StarCoder2-3B,Base Model,English,Compile,25.09
+StarCoder2-3B,Base Model,Chinese,Compile,27.41
+StarCoder2-3B,Base Model,Japanese,Compile,27.24
+StarCoder2-3B,Base Model,French,Compile,24.52
+CodeGeex2-6B,Base Model,English,Interpreter,33.49
+CodeGeex2-6B,Base Model,Chinese,Interpreter,29.9
+CodeGeex2-6B,Base Model,Japanese,Interpreter,25.27
+CodeGeex2-6B,Base Model,French,Interpreter,11.75
+CodeGeex2-6B,Base Model,English,Script,20.93
+CodeGeex2-6B,Base Model,Chinese,Script,8.44
+CodeGeex2-6B,Base Model,Japanese,Script,15.94
+CodeGeex2-6B,Base Model,French,Script,14.58
+CodeGeex2-6B,Base Model,English,Compile,22.06
+CodeGeex2-6B,Base Model,Chinese,Compile,23.46
+CodeGeex2-6B,Base Model,Japanese,Compile,28.45
+CodeGeex2-6B,Base Model,French,Compile,20.45
+StarCoderBase-7B,Base Model,English,Interpreter,28.37
+StarCoderBase-7B,Base Model,Chinese,Interpreter,27.35
+StarCoderBase-7B,Base Model,Japanese,Interpreter,22.12
+StarCoderBase-7B,Base Model,French,Interpreter,11.08
+StarCoderBase-7B,Base Model,English,Script,21.77
+StarCoderBase-7B,Base Model,Chinese,Script,8.1
+StarCoderBase-7B,Base Model,Japanese,Script,23.35
+StarCoderBase-7B,Base Model,French,Script,14.51
+StarCoderBase-7B,Base Model,English,Compile,15.1
+StarCoderBase-7B,Base Model,Chinese,Compile,24.44
+StarCoderBase-7B,Base Model,Japanese,Compile,23.3
+StarCoderBase-7B,Base Model,French,Compile,22.6
+OctoGeeX-7B,Chat Model,English,Interpreter,42.28
+OctoGeeX-7B,Chat Model,Chinese,Interpreter,28.5
+OctoGeeX-7B,Chat Model,Japanese,Interpreter,25.85
+OctoGeeX-7B,Chat Model,French,Interpreter,12.02
+OctoGeeX-7B,Chat Model,English,Script,22.94
+OctoGeeX-7B,Chat Model,Chinese,Script,9.77
+OctoGeeX-7B,Chat Model,Japanese,Script,16.19
+OctoGeeX-7B,Chat Model,French,Script,13.66
+OctoGeeX-7B,Chat Model,English,Compile,17.03
+OctoGeeX-7B,Chat Model,Chinese,Compile,19.33
+OctoGeeX-7B,Chat Model,Japanese,Compile,23.93
+OctoGeeX-7B,Chat Model,French,Compile,17.94
+WizardCoder-3B-V1.0,Chat Model,English,Interpreter,32.92
+WizardCoder-3B-V1.0,Chat Model,Chinese,Interpreter,26.16
+WizardCoder-3B-V1.0,Chat Model,Japanese,Interpreter,24.83
+WizardCoder-3B-V1.0,Chat Model,French,Interpreter,9.44
+WizardCoder-3B-V1.0,Chat Model,English,Script,19.6
+WizardCoder-3B-V1.0,Chat Model,Chinese,Script,7.91
+WizardCoder-3B-V1.0,Chat Model,Japanese,Script,21.75
+WizardCoder-3B-V1.0,Chat Model,French,Script,13.64
+WizardCoder-3B-V1.0,Chat Model,English,Compile,15.7
+WizardCoder-3B-V1.0,Chat Model,Chinese,Compile,24.34
+WizardCoder-3B-V1.0,Chat Model,Japanese,Compile,24.94
+WizardCoder-3B-V1.0,Chat Model,French,Compile,20.56
+CodeGen25-7B-multi,Base Model,English,Interpreter,28.7
+CodeGen25-7B-multi,Base Model,Chinese,Interpreter,26.27
+CodeGen25-7B-multi,Base Model,Japanese,Interpreter,21.98
+CodeGen25-7B-multi,Base Model,French,Interpreter,10.37
+CodeGen25-7B-multi,Base Model,English,Script,19.11
+CodeGen25-7B-multi,Base Model,Chinese,Script,8.84
+CodeGen25-7B-multi,Base Model,Japanese,Script,23.44
+CodeGen25-7B-multi,Base Model,French,Script,11.59
+CodeGen25-7B-multi,Base Model,English,Compile,16.62
+CodeGen25-7B-multi,Base Model,Chinese,Compile,26.01
+CodeGen25-7B-multi,Base Model,Japanese,Compile,25.75
+CodeGen25-7B-multi,Base Model,French,Compile,21.84
+CodeGemma-2B,Base Model,English,Interpreter,27.28
+CodeGemma-2B,Base Model,Chinese,Interpreter,29.94
+CodeGemma-2B,Base Model,Japanese,Interpreter,28.76
+CodeGemma-2B,Base Model,French,Interpreter,8.77
+CodeGemma-2B,Base Model,English,Script,0.01
+CodeGemma-2B,Base Model,Chinese,Script,9.54
+CodeGemma-2B,Base Model,Japanese,Script,25.23
+CodeGemma-2B,Base Model,French,Script,10.71
+CodeGemma-2B,Base Model,English,Compile,6.3
+CodeGemma-2B,Base Model,Chinese,Compile,24.71
+CodeGemma-2B,Base Model,Japanese,Compile,29.33
+CodeGemma-2B,Base Model,French,Compile,26.81
+Refact-1.6B,Chat Model,English,Interpreter,31.1
+Refact-1.6B,Chat Model,Chinese,Interpreter,22.36
+Refact-1.6B,Chat Model,Japanese,Interpreter,22.36
+Refact-1.6B,Chat Model,French,Interpreter,4.97
+Refact-1.6B,Chat Model,English,Script,13.84
+Refact-1.6B,Chat Model,Chinese,Script,10.26
+Refact-1.6B,Chat Model,Japanese,Script,15.53
+Refact-1.6B,Chat Model,French,Script,13.04
+Refact-1.6B,Chat Model,English,Compile,18.35
+Refact-1.6B,Chat Model,Chinese,Compile,22.78
+Refact-1.6B,Chat Model,Japanese,Compile,21.12
+Refact-1.6B,Chat Model,French,Compile,18.59
+Stable-code-3b,Base Model,English,Interpreter,30.72
+Stable-code-3b,Base Model,Chinese,Interpreter,31.64
+Stable-code-3b,Base Model,Japanese,Interpreter,23.68
+Stable-code-3b,Base Model,French,Interpreter,0.0
+Stable-code-3b,Base Model,English,Script,21.41
+Stable-code-3b,Base Model,Chinese,Script,10.09
+Stable-code-3b,Base Model,Japanese,Script,17.54
+Stable-code-3b,Base Model,French,Script,13.37
+Stable-code-3b,Base Model,English,Compile,0.0
+Stable-code-3b,Base Model,Chinese,Compile,28.75
+Stable-code-3b,Base Model,Japanese,Compile,29.42
+Stable-code-3b,Base Model,French,Compile,22.15
+DeepSeek-Coder-1b-base,Base Model,English,Interpreter,32.13
+DeepSeek-Coder-1b-base,Base Model,Chinese,Interpreter,28.46
+DeepSeek-Coder-1b-base,Base Model,Japanese,Interpreter,22.75
+DeepSeek-Coder-1b-base,Base Model,French,Interpreter,9.58
+DeepSeek-Coder-1b-base,Base Model,English,Script,15.17
+DeepSeek-Coder-1b-base,Base Model,Chinese,Script,9.91
+DeepSeek-Coder-1b-base,Base Model,Japanese,Script,19.44
+DeepSeek-Coder-1b-base,Base Model,French,Script,11.4
+DeepSeek-Coder-1b-base,Base Model,English,Compile,11.39
+DeepSeek-Coder-1b-base,Base Model,Chinese,Compile,27.16
+DeepSeek-Coder-1b-base,Base Model,Japanese,Compile,27.96
+DeepSeek-Coder-1b-base,Base Model,French,Compile,18.13
+StarCoderBase-3B,Base Model,English,Interpreter,21.5
+StarCoderBase-3B,Base Model,Chinese,Interpreter,21.32
+StarCoderBase-3B,Base Model,Japanese,Interpreter,18.55
+StarCoderBase-3B,Base Model,French,Interpreter,7.87
+StarCoderBase-3B,Base Model,English,Script,16.1
+StarCoderBase-3B,Base Model,Chinese,Script,4.97
+StarCoderBase-3B,Base Model,Japanese,Script,18.04
+StarCoderBase-3B,Base Model,French,Script,10.1
+StarCoderBase-3B,Base Model,English,Compile,9.98
+StarCoderBase-3B,Base Model,Chinese,Compile,19.25
+StarCoderBase-3B,Base Model,Japanese,Compile,19.43
+StarCoderBase-3B,Base Model,French,Compile,16.32
+WizardCoder-1B-V1.0,Chat Model,English,Interpreter,23.17
+WizardCoder-1B-V1.0,Chat Model,Chinese,Interpreter,19.13
+WizardCoder-1B-V1.0,Chat Model,Japanese,Interpreter,14.71
+WizardCoder-1B-V1.0,Chat Model,French,Interpreter,6.51
+WizardCoder-1B-V1.0,Chat Model,English,Script,13.85
+WizardCoder-1B-V1.0,Chat Model,Chinese,Script,4.64
+WizardCoder-1B-V1.0,Chat Model,Japanese,Script,15.52
+WizardCoder-1B-V1.0,Chat Model,French,Script,10.01
+WizardCoder-1B-V1.0,Chat Model,English,Compile,9.59
+WizardCoder-1B-V1.0,Chat Model,Chinese,Compile,19.68
+WizardCoder-1B-V1.0,Chat Model,Japanese,Compile,15.94
+WizardCoder-1B-V1.0,Chat Model,French,Compile,13.91
+Replit-2.7B,Base Model,English,Interpreter,20.12
+Replit-2.7B,Base Model,Chinese,Interpreter,20.18
+Replit-2.7B,Base Model,Japanese,Interpreter,16.14
+Replit-2.7B,Base Model,French,Interpreter,3.22
+Replit-2.7B,Base Model,English,Script,1.24
+Replit-2.7B,Base Model,Chinese,Script,6.41
+Replit-2.7B,Base Model,Japanese,Script,2.11
+Replit-2.7B,Base Model,French,Script,7.2
+Replit-2.7B,Base Model,English,Compile,5.88
+Replit-2.7B,Base Model,Chinese,Compile,21.39
+Replit-2.7B,Base Model,Japanese,Compile,20.37
+Replit-2.7B,Base Model,French,Compile,15.19
+CodeGen25-7B-mono,Base Model,English,Interpreter,33.08
+CodeGen25-7B-mono,Base Model,Chinese,Interpreter,23.22
+CodeGen25-7B-mono,Base Model,Japanese,Interpreter,16.75
+CodeGen25-7B-mono,Base Model,French,Interpreter,4.07
+CodeGen25-7B-mono,Base Model,English,Script,4.65
+CodeGen25-7B-mono,Base Model,Chinese,Script,4.32
+CodeGen25-7B-mono,Base Model,Japanese,Script,6.75
+CodeGen25-7B-mono,Base Model,French,Script,4.41
+CodeGen25-7B-mono,Base Model,English,Compile,1.71
+CodeGen25-7B-mono,Base Model,Chinese,Compile,19.75
+CodeGen25-7B-mono,Base Model,Japanese,Compile,18.62
+CodeGen25-7B-mono,Base Model,French,Compile,7.83
+StarCoderBase-1.1B,Base Model,English,Interpreter,15.17
+StarCoderBase-1.1B,Base Model,Chinese,Interpreter,13.38
+StarCoderBase-1.1B,Base Model,Japanese,Interpreter,9.94
+StarCoderBase-1.1B,Base Model,French,Interpreter,5.03
+StarCoderBase-1.1B,Base Model,English,Script,11.31
+StarCoderBase-1.1B,Base Model,Chinese,Script,4.65
+StarCoderBase-1.1B,Base Model,Japanese,Script,12.52
+StarCoderBase-1.1B,Base Model,French,Script,5.73
+StarCoderBase-1.1B,Base Model,English,Compile,3.92
+StarCoderBase-1.1B,Base Model,Chinese,Compile,14.2
+StarCoderBase-1.1B,Base Model,Japanese,Compile,11.68
+StarCoderBase-1.1B,Base Model,French,Compile,10.24
+CodeGen-16B-Multi,Base Model,English,Interpreter,19.26
+CodeGen-16B-Multi,Base Model,Chinese,Interpreter,19.15
+CodeGen-16B-Multi,Base Model,Japanese,Interpreter,8.37
+CodeGen-16B-Multi,Base Model,French,Interpreter,0.66
+CodeGen-16B-Multi,Base Model,English,Script,0.0
+CodeGen-16B-Multi,Base Model,Chinese,Script,7.68
+CodeGen-16B-Multi,Base Model,Japanese,Script,8.5
+CodeGen-16B-Multi,Base Model,French,Script,6.45
+CodeGen-16B-Multi,Base Model,English,Compile,1.25
+CodeGen-16B-Multi,Base Model,Chinese,Compile,22.2
+CodeGen-16B-Multi,Base Model,Japanese,Compile,21.0
+CodeGen-16B-Multi,Base Model,French,Compile,4.21
+StableCode-3B-alpha,Base Model,English,Interpreter,20.2
+StableCode-3B-alpha,Base Model,Chinese,Interpreter,18.98
+StableCode-3B-alpha,Base Model,Japanese,Interpreter,3.95
+StableCode-3B-alpha,Base Model,French,Interpreter,0.008
+StableCode-3B-alpha,Base Model,English,Script,0.0
+StableCode-3B-alpha,Base Model,Chinese,Script,4.77
+StableCode-3B-alpha,Base Model,Japanese,Script,5.14
+StableCode-3B-alpha,Base Model,French,Script,0.8
+StableCode-3B-alpha,Base Model,English,Compile,0.98
+StableCode-3B-alpha,Base Model,Chinese,Compile,19.54
+StableCode-3B-alpha,Base Model,Japanese,Compile,20.77
+StableCode-3B-alpha,Base Model,French,Compile,2.03
+DeciCoder-1B,Base Model,English,Interpreter,19.32
+DeciCoder-1B,Base Model,Chinese,Interpreter,17.85
+DeciCoder-1B,Base Model,Japanese,Interpreter,2.01
+DeciCoder-1B,Base Model,French,Interpreter,0.47
+DeciCoder-1B,Base Model,English,Script,0.0
+DeciCoder-1B,Base Model,Chinese,Script,6.08
+DeciCoder-1B,Base Model,Japanese,Script,0.0
+DeciCoder-1B,Base Model,French,Script,0.1
+DeciCoder-1B,Base Model,English,Compile,0.63
+DeciCoder-1B,Base Model,Chinese,Compile,15.3
+DeciCoder-1B,Base Model,Japanese,Compile,6.87
+DeciCoder-1B,Base Model,French,Compile,1.72
+Phi-1,Base Model,English,Interpreter,51.22
+Phi-1,Base Model,Chinese,Interpreter,19.25
+Phi-1,Base Model,Japanese,Interpreter,12.42
+Phi-1,Base Model,French,Interpreter,3.11
+Phi-1,Base Model,English,Script,0.63
+Phi-1,Base Model,Chinese,Script,7.05
+Phi-1,Base Model,Japanese,Script,6.21
+Phi-1,Base Model,French,Script,6.21
+Phi-1,Base Model,English,Compile,10.13
+Phi-1,Base Model,Chinese,Compile,10.76
+Phi-1,Base Model,Japanese,Compile,14.29
+Phi-1,Base Model,French,Compile,4.49
+SantaCoder-1.1B,Base Model,English,Interpreter,18.12
+SantaCoder-1.1B,Base Model,Chinese,Interpreter,15.47
+SantaCoder-1.1B,Base Model,Japanese,Interpreter,1.5
+SantaCoder-1.1B,Base Model,French,Interpreter,0.0
+SantaCoder-1.1B,Base Model,English,Script,0.0
+SantaCoder-1.1B,Base Model,Chinese,Script,0.0
+SantaCoder-1.1B,Base Model,Japanese,Script,0.1
+SantaCoder-1.1B,Base Model,French,Script,0.0
+SantaCoder-1.1B,Base Model,English,Compile,0.7
+SantaCoder-1.1B,Base Model,Chinese,Compile,15.0
+SantaCoder-1.1B,Base Model,Japanese,Compile,6.2
+SantaCoder-1.1B,Base Model,French,Compile,2.0