Spaces:
Running
Running
Martin Jurkovic
commited on
Commit
·
a41edef
1
Parent(s):
5909269
Update about section
Browse files- app.py +0 -87
- src/about.py +12 -31
app.py
CHANGED
|
@@ -155,93 +155,6 @@ with demo:
|
|
| 155 |
with gr.TabItem("📝 About", elem_id="syntherela-benchmark-tab-table", id=3):
|
| 156 |
gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
|
| 157 |
|
| 158 |
-
with gr.TabItem("🚀 Submit here! ", elem_id="syntherela-benchmark-tab-table", id=4):
|
| 159 |
-
with gr.Column():
|
| 160 |
-
with gr.Row():
|
| 161 |
-
gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
|
| 162 |
-
|
| 163 |
-
with gr.Column():
|
| 164 |
-
with gr.Accordion(
|
| 165 |
-
f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
|
| 166 |
-
open=False,
|
| 167 |
-
):
|
| 168 |
-
with gr.Row():
|
| 169 |
-
finished_eval_table = gr.components.Dataframe(
|
| 170 |
-
value=finished_eval_queue_df,
|
| 171 |
-
headers=EVAL_COLS,
|
| 172 |
-
datatype=EVAL_TYPES,
|
| 173 |
-
row_count=5,
|
| 174 |
-
)
|
| 175 |
-
with gr.Accordion(
|
| 176 |
-
f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
|
| 177 |
-
open=False,
|
| 178 |
-
):
|
| 179 |
-
with gr.Row():
|
| 180 |
-
running_eval_table = gr.components.Dataframe(
|
| 181 |
-
value=running_eval_queue_df,
|
| 182 |
-
headers=EVAL_COLS,
|
| 183 |
-
datatype=EVAL_TYPES,
|
| 184 |
-
row_count=5,
|
| 185 |
-
)
|
| 186 |
-
|
| 187 |
-
with gr.Accordion(
|
| 188 |
-
f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
|
| 189 |
-
open=False,
|
| 190 |
-
):
|
| 191 |
-
with gr.Row():
|
| 192 |
-
pending_eval_table = gr.components.Dataframe(
|
| 193 |
-
value=pending_eval_queue_df,
|
| 194 |
-
headers=EVAL_COLS,
|
| 195 |
-
datatype=EVAL_TYPES,
|
| 196 |
-
row_count=5,
|
| 197 |
-
)
|
| 198 |
-
with gr.Row():
|
| 199 |
-
gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
|
| 200 |
-
|
| 201 |
-
with gr.Row():
|
| 202 |
-
with gr.Column():
|
| 203 |
-
model_name_textbox = gr.Textbox(label="Model name")
|
| 204 |
-
revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
|
| 205 |
-
model_type = gr.Dropdown(
|
| 206 |
-
choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
|
| 207 |
-
label="Model type",
|
| 208 |
-
multiselect=False,
|
| 209 |
-
value=None,
|
| 210 |
-
interactive=True,
|
| 211 |
-
)
|
| 212 |
-
|
| 213 |
-
# with gr.Column():
|
| 214 |
-
# precision = gr.Dropdown(
|
| 215 |
-
# choices=[i.value.name for i in Precision if i != Precision.Unknown],
|
| 216 |
-
# label="Precision",
|
| 217 |
-
# multiselect=False,
|
| 218 |
-
# value="float16",
|
| 219 |
-
# interactive=True,
|
| 220 |
-
# )
|
| 221 |
-
# weight_type = gr.Dropdown(
|
| 222 |
-
# choices=[i.value.name for i in WeightType],
|
| 223 |
-
# label="Weights type",
|
| 224 |
-
# multiselect=False,
|
| 225 |
-
# value="Original",
|
| 226 |
-
# interactive=True,
|
| 227 |
-
# )
|
| 228 |
-
base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
|
| 229 |
-
|
| 230 |
-
submit_button = gr.Button("Submit Eval")
|
| 231 |
-
submission_result = gr.Markdown()
|
| 232 |
-
submit_button.click(
|
| 233 |
-
add_new_eval,
|
| 234 |
-
[
|
| 235 |
-
model_name_textbox,
|
| 236 |
-
base_model_name_textbox,
|
| 237 |
-
revision_name_textbox,
|
| 238 |
-
# precision,
|
| 239 |
-
# weight_type,
|
| 240 |
-
model_type,
|
| 241 |
-
],
|
| 242 |
-
submission_result,
|
| 243 |
-
)
|
| 244 |
-
|
| 245 |
with gr.Row():
|
| 246 |
with gr.Accordion("📙 Citation", open=False):
|
| 247 |
citation_button = gr.Textbox(
|
|
|
|
| 155 |
with gr.TabItem("📝 About", elem_id="syntherela-benchmark-tab-table", id=3):
|
| 156 |
gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
|
| 157 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
with gr.Row():
|
| 159 |
with gr.Accordion("📙 Citation", open=False):
|
| 160 |
citation_button = gr.Textbox(
|
src/about.py
CHANGED
|
@@ -60,47 +60,28 @@ Intro text
|
|
| 60 |
|
| 61 |
# Which evaluations are you running? how can people reproduce what you have?
|
| 62 |
LLM_BENCHMARKS_TEXT = f"""
|
| 63 |
-
|
|
|
|
| 64 |
|
| 65 |
-
|
| 66 |
-
To reproduce our results, here is the commands you can run:
|
| 67 |
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
### 1) Make sure you can load your model and tokenizer using AutoClasses:
|
| 74 |
-
```python
|
| 75 |
-
from transformers import AutoConfig, AutoModel, AutoTokenizer
|
| 76 |
-
config = AutoConfig.from_pretrained("your model name", revision=revision)
|
| 77 |
-
model = AutoModel.from_pretrained("your model name", revision=revision)
|
| 78 |
-
tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
|
| 79 |
-
```
|
| 80 |
-
If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
|
| 81 |
|
| 82 |
-
|
| 83 |
-
Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
|
| 84 |
|
| 85 |
-
|
| 86 |
-
It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
|
| 87 |
-
|
| 88 |
-
### 3) Make sure your model has an open license!
|
| 89 |
-
This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
|
| 90 |
-
|
| 91 |
-
### 4) Fill up your model card
|
| 92 |
-
When we add extra information about models to the leaderboard, it will be automatically taken from the model card
|
| 93 |
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
Make sure you have followed the above steps first.
|
| 97 |
-
If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
|
| 98 |
"""
|
| 99 |
|
| 100 |
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
|
| 101 |
CITATION_BUTTON_TEXT = r"""
|
| 102 |
@inproceedings{
|
| 103 |
-
|
| 104 |
title={SyntheRela: A Benchmark For Synthetic Relational Database Generation},
|
| 105 |
author={Martin Jurkovic and Valter Hudovernik and Erik {\v{S}}trumbelj},
|
| 106 |
booktitle={Will Synthetic Data Finally Solve the Data Access Problem?},
|
|
|
|
| 60 |
|
| 61 |
# Which evaluations are you running? how can people reproduce what you have?
|
| 62 |
LLM_BENCHMARKS_TEXT = f"""
|
| 63 |
+
# About
|
| 64 |
+
The **SyntheRela leaderboard** provides a public evaluation of relational database synthesis methods using the **SyntheRela benchmark** ([github](https://github.com/martinjurkovic/syntherela)). This benchmark incorporates best practices, a novel robust detection metric, and a relational deep learning utility approach that leverages graph neural networks. It enables a comprehensive comparison of methods across multiple real-world databases.
|
| 65 |
|
| 66 |
+
To add a model to the leaderboard, run the **SyntheRela benchmark** on your generated data and then open a pull request on the [SyntheRela repository](https://github.com/martinjurkovic/syntherela).
|
|
|
|
| 67 |
|
| 68 |
+
## Authors
|
| 69 |
+
- **Martin Jurkovič**
|
| 70 |
+
- **Valter Hudovernik**
|
| 71 |
+
- **Erik Štrumbelj**
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
|
| 73 |
+
If you use the results from this leaderboard in your research, please **cite our paper** (citation below).
|
|
|
|
| 74 |
|
| 75 |
+
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
|
| 77 |
+
EVALUATION_QUEUE_TEXT = """
|
| 78 |
+
To add a model to the leaderboard, run the **SyntheRela benchmark** on your generated data and then open a pull request on the [SyntheRela repository](https://github.com/martinjurkovic/syntherela).
|
|
|
|
|
|
|
| 79 |
"""
|
| 80 |
|
| 81 |
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
|
| 82 |
CITATION_BUTTON_TEXT = r"""
|
| 83 |
@inproceedings{
|
| 84 |
+
iclrsyntheticdata2025syntherela,
|
| 85 |
title={SyntheRela: A Benchmark For Synthetic Relational Database Generation},
|
| 86 |
author={Martin Jurkovic and Valter Hudovernik and Erik {\v{S}}trumbelj},
|
| 87 |
booktitle={Will Synthetic Data Finally Solve the Data Access Problem?},
|