diff --git a/.dockerignore b/.dockerignore index 4494b288f673b751f5f50da46a69cc8a7d12340d..e6905a23959a7bf2e81c2c63e6e6097a1a13f93e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,44 +1 @@ -# Python -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# IDE -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# OS -.DS_Store -Thumbs.db - -# Git -.git/ -.gitignore - -# Documentation -*.md -!README.md - -# Tests -tests/ -test_*.py -*_test.py - -# Logs -*.log -logs/ - -# Temporary files -*.tmp -*.temp -.cache/ +.env* \ No newline at end of file diff --git a/.gitignore b/.gitignore index 60a6fb645203f7433d186d540033f87d0800978d..5ef6a520780202a1d6addd833d800ccb1ecac0bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,41 @@ -# Build local -docker-compose.yml -venv-ts-frontend/ -.streamlit/ -# Secrets -.env - -# Other +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc .DS_Store -*/__pycache__/ \ No newline at end of file +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# env files (can opt-in for committing if needed) +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/Dockerfile b/Dockerfile index 7bd11b1e5f26f701fc0bdc56cdc229ced49f0946..2e3341018e62a1dda3e83a9ef4efbb6e4ca532fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,66 @@ -FROM python:3.12-slim +# syntax=docker.io/docker/dockerfile:1 +FROM node:20-alpine AS base + +# Install dependencies only when needed +FROM base AS deps +# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed. +RUN apk add --no-cache libc6-compat WORKDIR /app -RUN apt-get update && apt-get install -y \ - gcc \ - && rm -rf /var/lib/apt/lists/* +# Install dependencies based on the preferred package manager +COPY package.json yarn.lock* package-lock.json* pnpm-lock.yaml* .npmrc* ./ +RUN \ + if [ -f yarn.lock ]; then yarn --frozen-lockfile; \ + elif [ -f package-lock.json ]; then npm ci; \ + elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm i --frozen-lockfile; \ + else echo "Lockfile not found." && exit 1; \ + fi -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +# Rebuild the source code only when needed +FROM base AS builder +WORKDIR /app +COPY --from=deps /app/node_modules ./node_modules COPY . . -EXPOSE 8501 +# Next.js collects completely anonymous telemetry data about general usage. +# Learn more here: https://nextjs.org/telemetry +# Uncomment the following line in case you want to disable telemetry during the build. +# ENV NEXT_TELEMETRY_DISABLED=1 + +RUN \ + if [ -f yarn.lock ]; then yarn run build; \ + elif [ -f package-lock.json ]; then npm run build; \ + elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm run build; \ + else echo "Lockfile not found." && exit 1; \ + fi + +# Production image, copy all the files and run next +FROM base AS runner +WORKDIR /app + +ENV NODE_ENV=production +# Uncomment the following line in case you want to disable telemetry during runtime. +# ENV NEXT_TELEMETRY_DISABLED=1 + +RUN addgroup --system --gid 1001 nodejs +RUN adduser --system --uid 1001 nextjs + +COPY --from=builder /app/public ./public + +# Automatically leverage output traces to reduce image size +# https://nextjs.org/docs/advanced-features/output-file-tracing +COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ +COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static + +USER nextjs + +EXPOSE 3000 -HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health +ENV PORT=3000 -ENTRYPOINT ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"] \ No newline at end of file +# server.js is created by next build from the standalone output +# https://nextjs.org/docs/pages/api-reference/config/next-config-js/output +ENV HOSTNAME="0.0.0.0" +CMD ["node", "server.js"] \ No newline at end of file diff --git a/README.md b/README.md index 2eeecdc5823eed6af3f2da9a343c461fe51af0d0..4ff0587c3b696b868cdcd864c1529a17ad44a571 100644 --- a/README.md +++ b/README.md @@ -1,74 +1,19 @@ --- -title: TS Arena +title: TS-Arena emoji: π colorFrom: red colorTo: red sdk: docker -app_port: 8501 -tags: -- streamlit -models: -- google/timesfm-2.0-500m-pytorch -- Maple728/TimeMoE-50M -- Salesforce/moirai-1.1-R-large -- thuml/sundial-base-128m - pinned: false +app_port: 3000 short_description: Time Series Forecasting Arena --- +# TS-Arena Frontend -# TS-Arena Streamlit Dashboard - -This is a Streamlit version of the TS-Arena Challenge Dashboard. - -## Installation - -1. Install dependencies: -```bash -pip install -r requirements.txt -``` - -2. Set up environment variables: -```bash -export X_API_URL="your_api_url_here" -export X_API_KEY="your_api_key_here" -``` - -Or create a `.env` file with: -``` -X_API_URL=your_api_url_here -X_API_KEY=your_api_key_here -``` - -## Running the App - -```bash -streamlit run app.py -``` - -The app will be available at `http://localhost:8501` - -## Features - -- **Model Ranking**: View and filter model rankings based on performance metrics (Tab 1) -- **Challenge Visualization**: Select and visualize active and completed challenges (Tab 2) -- **Time Series Selection**: Choose specific time series to analyze within a challenge -- **Interactive Plots**: Plotly charts with zoom, pan, and hover features -- **Upcoming Challenges**: View upcoming challenges in the sidebar -- **Add Model**: Information on how to participate and register your models (Tab 3) - -## Differences from Gradio Version - -Streamlit has some differences in behavior compared to Gradio: +## Concept -1. **State Management**: Streamlit uses session state and reruns the entire script on interaction -2. **Layout**: Streamlit uses `st.sidebar` for the sidebar and columns for layout -3. **Interactivity**: Dropdowns and multiselect widgets automatically trigger reruns -4. **Tabs**: Uses `st.tabs()` instead of Gradio's `gr.Tab()` +TS-Arena is a platform designed for **pre-registered forecasts into the real future**. This approach ensures rigorous, information leakage-free evaluations by comparing model predictions against data that did not exist at the time of the forecast. -## File Structure +This repository contains the backend infrastructure that powers the TS-Arena platform. It is designed to be self-hostable, allowing you to run your own instance of the benchmarking environment. -- `app.py` - Main Streamlit application -- `utils/api_client.py` - API client for fetching challenge data -- `utils/utils.py` - Utility functions for parsing durations -- `requirements.txt` - Python dependencies +For more information visit our [Git repository](https://github.com/DAG-UPB/ts-arena-backend). diff --git a/app.py b/app.py deleted file mode 100644 index 85fc5bf2cd8c1d4804fb87f54926515f205f3cee..0000000000000000000000000000000000000000 --- a/app.py +++ /dev/null @@ -1,185 +0,0 @@ -from components.challenges_tab_component import render_challenges_tab_component -from components.filter_component import render_filter_component, set_rankings_session_state -import streamlit as st -import pandas as pd -import sys -import os - -from components.challenge_list_component import render_challenge_list_component - -# Add src to path for imports -sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) -from utils.api_client import ChallengeStatus, DashboardAPIClient - -# Get API URL from environment variable -api_url = os.getenv('X_API_URL', '') -if api_url: - os.environ['DASHBOARD_API_URL'] = api_url - print(f"Loaded API URL from X_API_URL: {api_url}", file=sys.stderr) -else: - print("Warning: X_API_URL environment variable not set, using default", file=sys.stderr) - -# Get API key from environment variable -api_key = os.getenv('X_API_KEY', '') -if api_key: - os.environ['DASHBOARD_API_KEY'] = api_key - print(f"Loaded API key from X_API_KEY environment variable", file=sys.stderr) -else: - print("Warning: X_API_KEY environment variable not set", file=sys.stderr) - -# Initialize API client -api_client = DashboardAPIClient() - -# Page config -st.set_page_config( - page_title="TS-Arena Challenge Dashboard", - page_icon="ποΈ", - layout="wide", - initial_sidebar_state="expanded" -) - -# Custom CSS to make sidebar wider -st.markdown(""" - -""", unsafe_allow_html=True) - - -# Main app -st.title("ποΈ TS-Arena β The Live-Forecasting Benchmark (Prototype)") - -# Sidebar for challenge list -with st.sidebar: - st.header("ποΈ Upcoming") - upcoming_challenges = api_client.list_upcoming_challenges() - # Show only newest 5 challenges to reduce loading time - show_first = 5 - challenges_display = upcoming_challenges[:show_first] - render_challenge_list_component(challenges=challenges_display, api_client=api_client, challange_list_type="upcoming", show_first=5) - -# Create tabs -tab1, tab2, tab3 = st.tabs(["Model Ranking", "Challenges", "Add Model"]) - -with tab1: - with st.spinner("Loading model ranking..."): - render_filter_component(api_client=api_client, filter_type="model_ranking") - st.markdown("---") - if st.session_state.get('filtered_rankings') is None: - set_rankings_session_state(api_client=api_client) - st.rerun() - # Main content area - st.header("π Model Ranking") - - # Display rankings table from session state if available - filtered_rankings = st.session_state.get('filtered_rankings', None) - if filtered_rankings: - df_rankings = pd.concat([pd.DataFrame(df) for df in filtered_rankings], ignore_index=True) - tab_headers = list(df_rankings['time_ranges'].unique()) - tabs = st.tabs(tab_headers) - for unique_timerange, tab in zip(tab_headers, tabs): - filtered_by_time = df_rankings[df_rankings['time_ranges'] == unique_timerange].reset_index(drop=True) - with tab: - st.dataframe(filtered_by_time, use_container_width=True) - else: - st.info("No models match the selected filters") - -with tab2: - render_challenges_tab_component(api_client=api_client) - -with tab3: - st.header("π Join the TS-Arena Benchmark!") - - st.markdown(""" - Ready to test your forecasting models against the best? Participate actively in our benchmark challenges! - Simply send us an email and we'll provide you with an API key to get started. - - ### How it works: - - Email us with your organization name and we'll send you an API key. - With this key, you can register your own models and officially participate in active competitions - once your model is registered. - """) - - col1, col2 = st.columns(2) - - with col1: - st.markdown(""" - - π§ Request API Key - - """, unsafe_allow_html=True) - - with col2: - st.markdown(""" - - β Ask Questions - - """, unsafe_allow_html=True) - - st.markdown("---") - - st.markdown(""" - ### Bring the benchmark to life with your models! - - - π **Personal API Key** for model registration and submission - - π **Access to benchmark datasets** and challenge specifications - - π **Rankings and leaderboards** showing your model's performance - - π **Detailed evaluation metrics** across multiple time series - - π€ **Community support** from fellow forecasting researchers - - ### Requirements: - - - Valid organization or affiliation - - Commitment to fair participation - - Adherence to benchmark guidelines - - For more information, please contact us via email! - - ### Help Us Validate Your Model Implementation: - Transparency is at the heart of our live benchmarking project. We have integrated a wide range of state-of-the-art time series forecasting models to provide the community with real-time performance insights. - To ensure that every model is evaluated under optimal conditions, **we invite the original authors and maintainers to review our implementations.** If you are a developer of one of the featured models, we would value your feedback on: - - - Model configuration and hyperparameters. - - Data preprocessing steps. - - Implementation-specific nuances. - - Our goal is to represent your work as accurately as possible. Please visit our [GitHub Repository](https://github.com/DAG-UPB/ts-arena) to review the code or open an issue for any suggested improvements. - """) - # TODO: Change link to GitHub Repository diff --git a/components/challange_card_component.py b/components/challange_card_component.py deleted file mode 100644 index f2765ff3dc38a47ec7fb9efc434b90347e787c2f..0000000000000000000000000000000000000000 --- a/components/challange_card_component.py +++ /dev/null @@ -1,196 +0,0 @@ -import sys -import streamlit as st -import time -from datetime import datetime, timezone - -from utils.api_client import ChallengeStatus -from utils.utils import duration_to_max_unit, parse_iso8601_duration, to_local - -@st.fragment(run_every="1s") -def timer_fragment(): - now = datetime.now(timezone.utc) - - st.markdown( - f""" -
+ Test your forecasting models against the best in real-time benchmark challenges +
++ Ready to test your forecasting models against the best? Participate actively in our benchmark challenges! + Simply send us an email and we'll provide you with an API key to get started. Find detailed participation instructions in our{' '} + + Git repository + . +
+ + ++ Email us with your organization name and we'll send you an API key. +
++ Use your API key to register models for challenges in the registration phase. +
++ Obtain time-series context via our API and submit your forecasts before registration closes. +
++ During the active phase, your model is evaluated on live data. Monitor performance on leaderboards and explore forecasts through interactive plots. +
+For model registration and submission
+Access to datasets and challenge specifications
+Real-time performance tracking
+Comprehensive evaluation across time series
+Connect with forecasting researchers
++ Transparency is at the heart of our live benchmarking project. We have integrated a wide range of + state-of-the-art time series forecasting models to provide the community with real-time performance insights. +
++ To ensure that every model is evaluated under optimal conditions, we invite the original authors + and maintainers to review our implementations. If you are a developer of one of the featured models, + we would value your feedback on: +
++ Our goal is to represent your work as accurately as possible. Please visit our GitHub repository + to review the code or open an issue for any suggested improvements. +
+ + +Error
+{error || 'Round not found'}
+{round.description}
+ )} ++ Registration is open β submit your forecasts by{' '} + + {round.registration_end + ? new Date(round.registration_end).toLocaleDateString('en-US', { + weekday: 'long', + year: 'numeric', + month: 'long', + day: 'numeric', + hour: '2-digit', + minute: '2-digit' + }) + : 'the registration deadline'} + +
+Model rankings per series based on MASE score
+No leaderboard data available yet
+The leaderboard will be available once the round begins.
+| + Model + | ++ Avg Rank + | + {seriesIds.map(seriesId => ( +
+
+ {(seriesNameMap.get(seriesId) || `Series ${seriesId}`).replace(/[_-]/g, ' ')}
+
+ |
+ ))}
+ |
|---|---|---|---|
|
+
+ {model.model_name}
+
+ {model.readable_id} + |
+ + {model.avgRank.toFixed(2)} + | + {seriesIds.map(seriesId => { + const rankData = model.seriesRanks[seriesId]; + if (!rankData) { + return ( ++ - + | + ); + } + return ( ++ + {rankData.rank} + + | + ); + })} +
Series data not yet available
+The time series will be revealed once the round starts.
+Error
+{error || 'Challenge definition not found'}
+ID: {definition.id}
+{definition.description}
++ All rounds associated with this challenge definition, grouped by status +
+| + Name + | ++ Description + | ++ Registration + | ++ Start Time + | ++ End Time + | ++ Context Length + | ++ Frequency + | ++ Horizon + | ++ Domains + | ++ Categories + | ++ Subcategories + | +
|---|---|---|---|---|---|---|---|---|---|---|
| + + {round.name} + + | ++ {round.description} + | +
+ {new Date(round.registration_start).toLocaleDateString('en-US', {
+ year: 'numeric',
+ month: 'short',
+ day: 'numeric',
+ hour: '2-digit',
+ minute: '2-digit',
+ })}
+ to
+ {new Date(round.registration_end).toLocaleDateString('en-US', {
+ year: 'numeric',
+ month: 'short',
+ day: 'numeric',
+ hour: '2-digit',
+ minute: '2-digit',
+ })}
+ |
+ + {new Date(round.start_time).toLocaleDateString('en-US', { + year: 'numeric', + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + })} + | ++ {new Date(round.end_time).toLocaleDateString('en-US', { + year: 'numeric', + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + })} + | ++ {round.context_length} + | ++ {round.frequency} + | ++ {round.horizon} + | +
+
+ {round.domains && round.domains.length > 0 ? (
+ round.domains.map((domain, idx) => (
+
+ {domain}
+
+ ))
+ ) : (
+ -
+ )}
+
+ |
+
+
+ {round.categories && round.categories.length > 0 ? (
+ round.categories.map((category, idx) => (
+
+ {category}
+
+ ))
+ ) : (
+ -
+ )}
+
+ |
+
+
+ {round.subcategories && round.subcategories.length > 0 ? (
+ round.subcategories.map((subcategory, idx) => (
+
+ {subcategory}
+
+ ))
+ ) : (
+ -
+ )}
+
+ |
+
Error loading challenges
+{error}
++ Available challenges and their configurations +
+{definition.description}
+ +No challenges available
+No active rounds available at the moment.
+| + Challenge Definition + | + {timeRanges.map((timeRange) => ( ++ {timeRange.label} + | + ))} ++ Actions + | +
|---|---|---|
|
+ {definition.definition_name}
+ |
+ {timeRanges.map((timeRange) => (
+ + + Go to Challenge + + | +