description
Browse files
main.py
CHANGED
|
@@ -87,6 +87,40 @@ with gr.Blocks(title="Impermanent Leaderboard") as app:
|
|
| 87 |
"on data they could not have seen during training."
|
| 88 |
)
|
| 89 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
with gr.Tab("Leaderboard π"):
|
| 91 |
lb = compute_leaderboard(df)
|
| 92 |
gr.Dataframe(
|
|
|
|
| 87 |
"on data they could not have seen during training."
|
| 88 |
)
|
| 89 |
|
| 90 |
+
cutoff_dates = sorted(df["cutoff"].unique())
|
| 91 |
+
n_dates = len(cutoff_dates)
|
| 92 |
+
date_min, date_max = cutoff_dates[0], cutoff_dates[-1]
|
| 93 |
+
statistical_models = ["zero_model", "seasonal_naive", "auto_arima", "auto_ets", "auto_lgbm"]
|
| 94 |
+
foundation_models = ["chronos", "moirai", "timesfm"]
|
| 95 |
+
all_model_names = statistical_models + foundation_models
|
| 96 |
+
|
| 97 |
+
gr.Markdown(f"""\
|
| 98 |
+
## Datasets
|
| 99 |
+
|
| 100 |
+
GitHub repositories are selected across several **buckets based on their number of stars**,
|
| 101 |
+
yielding a mix of both intermittent (low-activity) and high-volume time series.
|
| 102 |
+
For each bucket, an automated pipeline fetches four signals:
|
| 103 |
+
|
| 104 |
+
- **Open issues** β number of issues opened
|
| 105 |
+
- **Opened PRs** β number of pull requests opened
|
| 106 |
+
- **Pushes** β number of push events
|
| 107 |
+
- **Stars** β number of new stars
|
| 108 |
+
|
| 109 |
+
Each signal is collected at both **daily** and **weekly** granularity.
|
| 110 |
+
|
| 111 |
+
## Models
|
| 112 |
+
|
| 113 |
+
The benchmark evaluates two families of forecasting methods:
|
| 114 |
+
|
| 115 |
+
- **Statistical / ML models:** {", ".join(f"`{m}`" for m in statistical_models)}
|
| 116 |
+
- **Foundation models:** {", ".join(f"`{m}`" for m in foundation_models)}
|
| 117 |
+
|
| 118 |
+
## Evaluation dates
|
| 119 |
+
|
| 120 |
+
Forecast methods are evaluated **every week** using rolling forecast evaluations.
|
| 121 |
+
Currently **{n_dates} evaluations** are available, from **{date_min}** to **{date_max}**.
|
| 122 |
+
""")
|
| 123 |
+
|
| 124 |
with gr.Tab("Leaderboard π"):
|
| 125 |
lb = compute_leaderboard(df)
|
| 126 |
gr.Dataframe(
|