diff --git a/README.md b/README.md index e55ed51212f6a5cfc586f6d288260edd9fee5174..ba5d996c57c58036795a3cbdb6e0d68d48dc234a 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ emoji: 🥇 colorFrom: green colorTo: indigo sdk: gradio -python_version: "3.10.0" app_file: app.py pinned: true license: apache-2.0 diff --git a/app.py b/app.py index 5007340fe9cb0fa39ee6b2cf148750cafc197bd2..91899449cdec5a2c0301b2a8d6f5aa9a84e66618 100644 --- a/app.py +++ b/app.py @@ -90,7 +90,7 @@ grouped_dfs = get_grouped_dfs() domain_df, freq_df, term_length_df, variate_type_df, overall_df = grouped_dfs['domain'], grouped_dfs['frequency'], grouped_dfs['term_length'], grouped_dfs['univariate'], grouped_dfs['overall'] overall_df = rename_metrics(overall_df) overall_df = format_df(overall_df) -overall_df = overall_df.sort_values(by=['MASE_Rank']) +overall_df = overall_df.sort_values(by=['Rank']) domain_df = pivot_existed_df(domain_df, tab_name='domain') print(f'Domain dataframe is {domain_df}') freq_df = pivot_existed_df(freq_df, tab_name='frequency') @@ -107,7 +107,7 @@ model_info_df = get_model_info_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH) # ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) -def init_leaderboard(ori_dataframe, model_info_df, sort_val: str | list | None = None): +def init_leaderboard(ori_dataframe, model_info_df, sort_val: str|None = None): if ori_dataframe is None or ori_dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") model_info_col_list = [c.name for c in fields(ModelInfoColumn) if c.displayed_by_default if c.name not in ['#Params (B)', 'available_on_hub', 'hub', 'Model sha','Hub License']] @@ -120,19 +120,11 @@ def init_leaderboard(ori_dataframe, model_info_df, sort_val: str | list | None = new_cols = ['T'] + [col for col in merged_df.columns if col != 'T'] merged_df = merged_df[new_cols] if sort_val: - if isinstance(sort_val, list): - assert sort_val[0] == 'Test Leak.' - # ipdb.set_trace() - leakage_order = pd.Categorical(merged_df[sort_val[0]], categories=['No', 'Yes', 'N/A'], ordered=True) - merged_df['leakage_order'] = leakage_order - merged_df = merged_df.sort_values(by=['leakage_order', sort_val[1]]) - merged_df = merged_df.drop(columns=['leakage_order']) - elif sort_val in merged_df.columns: + if sort_val in merged_df.columns: merged_df = merged_df.sort_values(by=[sort_val]) else: print(f'Warning: cannot sort by {sort_val}') print('Merged df: ', merged_df) - # ipdb.set_trace() # get the data type datatype_list = [col2type_dict[col] if col in col2type_dict else 'number' for col in merged_df.columns] # print('datatype_list: ', datatype_list) @@ -170,14 +162,13 @@ def init_leaderboard(ori_dataframe, model_info_df, sort_val: str | list | None = # ], filter_columns=[ ColumnFilter(ModelInfoColumn.model_type.name, type="checkboxgroup", label="Model types"), - ColumnFilter(ModelInfoColumn.testdata_leakage.name, type="checkboxgroup", label="Test Leak."), - ColumnFilter(ModelInfoColumn.replication_code_available.name, type="checkboxgroup", label="Replication Code"), ], # bool_checkboxgroup_label="", - column_widths=[30, 180] + [130 for _ in range(len(merged_df.columns)-2)], + column_widths=[40, 150] + [180 for _ in range(len(merged_df.columns)-2)], interactive=False, ) + demo = gr.Blocks(css=custom_css) with demo: gr.HTML(TITLE) @@ -185,8 +176,7 @@ with demo: with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem('🏅 Overall', elem_id="llm-benchmark-tab-table", id=5): - # leaderboard = init_leaderboard(overall_df, model_info_df, sort_val='Rank') - leaderboard = init_leaderboard(overall_df, model_info_df, sort_val=['Test Leak.', 'MASE_Rank']) + leaderboard = init_leaderboard(overall_df, model_info_df, sort_val='Rank') print(f'FINAL Overall LEADERBOARD {overall_df}') with gr.TabItem("🏅 By Domain", elem_id="llm-benchmark-tab-table", id=0): leaderboard = init_leaderboard(domain_df, model_info_df) @@ -206,27 +196,6 @@ with demo: with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=4): gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") - # Trigger the column filters once on initial load so default selections take effect - demo.load( - js=""" - () => { - // Make the JS fire one legitimate `input` event once the checkboxgroup - // component is ready. `querySelector` looks for the *wrapper* div Gradio - // puts around the checkbox-group. - const target = document.querySelector( - 'div[data-testid="checkboxgroup-model types"]'); - - if (!target) { return []; } // safety guard - - // Ask Gradio's front-end to re-compute its filters: - target.dispatchEvent(new Event('input', { bubbles: true })); - - return []; // load() must return something - } - """ - ) - - with gr.Row(): with gr.Accordion("📙 Citation", open=False): citation_button = gr.Textbox( diff --git a/requirements.txt b/requirements.txt index c1b5fcb5811f169dbf9950e8fe92baf5749dbe4d..195467bb08638766436e14eb28f4cc60d70a125e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ gradio==4.44.0 gradio[oauth]==4.44.0 gradio_leaderboard==0.0.9 gradio_client -huggingface-hub>=0.18.0,<0.35.3 +huggingface-hub>=0.18.0 matplotlib numpy pandas diff --git a/results/Chronos_small/config.json b/results/Chronos_small/config.json index 17e16ba50222f632859d572f452dbcb2dc16cb5d..22561f50905857de747befdc29fbb85aab6b2c64 100644 --- a/results/Chronos_small/config.json +++ b/results/Chronos_small/config.json @@ -3,8 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/amazon/chronos-t5-small", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb", - "org": "AWS AI Labs", - "testdata_leakage": "Yes", - "replication_code_available": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb" } \ No newline at end of file diff --git a/results/CleanTS-65M/all_results.csv b/results/CleanTS-65M/all_results.csv deleted file mode 100644 index cc8e5ac9d76fcdf2d281253ba7f325fe6850df4e..0000000000000000000000000000000000000000 --- a/results/CleanTS-65M/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/5T/long,CleanTS-65M,5488182.517148061,5488182.517148061,417.7214097307747,0.9766023740693816,5.350978519272389,0.8205310833753562,18.26306155646523,2342.687029278145,6.191022350175815,1.1039129646724686,0.7326658660220792,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,CleanTS-65M,3732160.358360173,3732160.358360173,311.66779368017416,1.1878145329818788,5.392097438028544,0.8106531956909149,22.990637599623486,1931.880006201258,5.869585016674671,0.9469328354209515,0.6663037056398805,Web/CloudOps,2 -bitbrains_fast_storage/5T/short,CleanTS-65M,1728140.8981336788,1728140.8981336788,160.21010091409664,0.8210056031312811,2.1954628914194774,0.7351953053431415,15.704410189030739,1314.587729340906,4.1274803599533305,0.5030200953728761,0.39408421209140043,Web/CloudOps,2 -bitbrains_fast_storage/H/short,CleanTS-65M,2349041.8300321284,2349041.8300321284,271.34103433090985,1.6635248535212532,3.4192269532863664,0.5932220897962239,12887.523738577462,1532.6584192285404,4.368574369354714,0.7734100912901811,0.901442216509749,Web/CloudOps,2 -bitbrains_rnd/5T/long,CleanTS-65M,2212210.4735384285,2212210.4735384285,159.03522338438958,3.3781216704958763,1.1769168586921777,0.7024461351546175,117.64601634111567,1487.350151624838,5.697703374411028,0.6092281148037341,0.5724758116529886,Web/CloudOps,2 -bitbrains_rnd/5T/medium,CleanTS-65M,2257006.7955528204,2257006.7955528204,147.9236898690365,4.45054545828052,0.48410432953184873,0.7207701120949027,162.32105483150028,1502.333783003238,6.321084055135416,0.62239036889577,0.585655498281893,Web/CloudOps,2 -bitbrains_rnd/5T/short,CleanTS-65M,1729522.6961120467,1729522.6961120467,126.17161989084374,1.922932251274716,1.4504602211670892,0.6651300214191744,52.96299675505438,1315.113187566776,5.381063537009379,0.516257847323543,0.4390736306343669,Web/CloudOps,2 -bitbrains_rnd/H/short,CleanTS-65M,1734649.0702123663,1734649.0702123663,151.34637706971725,5.619562557039445,1.4062218881076785,0.5853200599355576,194.99397840816366,1317.060769369571,5.764102379858145,0.6623658015932248,0.5840054322420504,Web/CloudOps,2 -bizitobs_application/10S/long,CleanTS-65M,8503036.737537434,8503036.737537434,1727.9619401041666,3.816936660064947,0.056564937167697484,0.05646290461222331,69.70237104523383,2915.996697106743,0.11245834584517234,0.06664059038898695,0.061655565697102385,Web/CloudOps,2 -bizitobs_application/10S/medium,CleanTS-65M,11207147.073972829,11207147.073972829,1703.2339238357545,3.5762804943269515,0.055665247837702436,0.05404658357302348,46.98213520274279,3347.707734252324,0.13145463298204493,0.06688098487498421,0.05553268386511726,Web/CloudOps,2 -bizitobs_application/10S/short,CleanTS-65M,3205592.7355155624,3205592.7355155624,669.8989510027568,2.0243552635767132,0.02999925265709559,0.03055323321786192,18.75493358108299,1790.4169166748738,0.06904922790093794,0.025835326346387705,0.022646740653164812,Web/CloudOps,2 -bizitobs_l2c/5T/long,CleanTS-65M,353.77026795402406,353.77026795402406,13.156052063230485,1.2396629207255008,0.910846787850378,1.1084931073113093,7.075684102584815,18.808781671177538,1.056084975104401,0.7386926574282008,0.5691355578219337,Web/CloudOps,7 -bizitobs_l2c/5T/medium,CleanTS-65M,157.42158371412836,157.42158371412836,8.689132310257477,0.8515150939343408,0.5993117567919233,0.8514387030990757,8.122555250805553,12.546775829436356,0.6601449357053149,0.45717615172754544,0.36371007020496937,Web/CloudOps,7 -bizitobs_l2c/5T/short,CleanTS-65M,20.749322239424302,20.749322239424302,2.6550411208044915,0.27572556493209166,0.14855580358396775,0.20868761789364118,3.1835256949459834,4.555142395076613,0.15644202288953063,0.09118485609637135,0.07478995298255846,Web/CloudOps,7 -bizitobs_l2c/H/long,CleanTS-65M,166.5031982421875,166.5031982421875,8.933871120876736,0.9245677364022553,0.9479636808763366,0.908265395391555,5.89926680870527,12.903611829336292,0.7881898853033701,0.5457066554086751,0.4294200463645112,Web/CloudOps,7 -bizitobs_l2c/H/medium,CleanTS-65M,162.55013137090774,162.55013137090774,8.7831298101516,0.8823811073621215,0.8297425346995897,0.9366540182204474,5.4369774235348265,12.749514946495326,0.7720025140927356,0.5318318636822951,0.4200076213299944,Web/CloudOps,7 -bizitobs_l2c/H/short,CleanTS-65M,150.63622120921573,150.63622120921573,7.267065805102152,0.7219975025537452,0.6442495052763026,0.763564376071805,3.9346266718654053,12.273394852656526,0.6615674693143918,0.39171349017439844,0.30999986821399717,Web/CloudOps,7 -bizitobs_service/10S/long,CleanTS-65M,159941.78537706228,159941.78537706228,103.76097113510288,2.623606135142696,0.11635794385400398,0.12418603667856327,42.15558238084946,399.9272251010955,0.2962788808291809,0.07686944641962316,0.06804511943686083,Web/CloudOps,2 -bizitobs_service/10S/medium,CleanTS-65M,74364.32587832087,74364.32587832087,79.22645188728495,2.61153363050725,0.10765373786655814,0.10854794661983226,32.792994017183226,272.69823226108537,0.20523910336465678,0.05962769106820453,0.04468579637228366,Web/CloudOps,2 -bizitobs_service/10S/short,CleanTS-65M,13567.967424586244,13567.967424586244,32.330862894028265,1.2602634395314907,0.08824208875464007,0.07231173209025213,21.79729710699016,116.48161839786673,0.08629330864877571,0.02395173735538282,0.021026624496567115,Web/CloudOps,2 -car_parts/M/short,CleanTS-65M,2.7423013386354897,2.7423013386354897,0.552149004294134,1.0087704520426521,0.7742172853785505,1.8470528645551552,18.236812140168727,1.6559895345790956,3.970893031786191,1.3239966726097314,1.1361740674445804,Sales,1 -covid_deaths/D/short,CleanTS-65M,907298.4461215684,907298.4461215684,150.41957131497037,30.883482186664057,0.08371486233851726,0.35534725143494816,513.3537528062479,952.5221499375059,0.3583910775218091,0.05659609306458412,0.045606899477847136,Healthcare,1 -electricity/15T/long,CleanTS-65M,350327.47669720906,350327.47669720906,63.5598211404422,0.9503734908184518,0.16098716200823368,0.1457204267246587,7.691358860377664,591.8846819247893,0.9337706580832442,0.10027341106541028,0.08000586129328355,Energy,1 -electricity/15T/medium,CleanTS-65M,266413.06531666795,266413.06531666795,57.50715103992059,0.9052327141694809,0.16140570000041277,0.14557454956947147,7.6929297244792485,516.1521726358109,0.8893821634390044,0.09909061148365231,0.07959325920844915,Energy,1 -electricity/15T/short,CleanTS-65M,168270.14738997005,168270.14738997005,58.824865179442654,1.1184414488021077,0.1789007336969428,0.17974754381010324,11.81212169826814,410.20744433758153,0.8085997408546413,0.11595540596957606,0.09671070806559352,Energy,1 -electricity/D/short,CleanTS-65M,2004870643.2372088,2004870643.2372088,5094.528292711688,1.5685728359905822,0.4683037421528801,0.10478110669620402,13.260611009679934,44775.78188303593,0.7358148011618476,0.08372001928424815,0.06661866392477252,Energy,1 -electricity/H/long,CleanTS-65M,7332529.034604677,7332529.034604677,270.6062350422402,1.2222560322733247,0.30543379452062525,0.14747578906583356,10.342021857344754,2707.864293978684,1.067980905861503,0.10672702198364212,0.08492648298076319,Energy,1 -electricity/H/medium,CleanTS-65M,5051193.216852333,5051193.216852333,237.29693170236976,1.0887284547281066,0.21561814362185347,0.13371077749307628,9.190469252311713,2247.485977009052,0.8772678280466714,0.09262481101382142,0.07416529477520094,Energy,1 -electricity/H/short,CleanTS-65M,1810796.9237911555,1810796.9237911555,184.9744263272905,1.010247791321694,0.19177217623690232,0.1347787143572796,10.733978206117245,1345.6585465084206,0.6352469367722453,0.0873211395346192,0.07145429755773598,Energy,1 -electricity/W/short,CleanTS-65M,179595311707.0655,179595311707.0655,45951.4512642113,1.924060073440388,0.2800341575325654,0.12567990862068024,15.033833047543906,423786.87061666447,0.966941804659472,0.10484604949531215,0.0859818742485093,Energy,1 -ett1/15T/long,CleanTS-65M,10.306158042900146,10.306158042900146,1.6543900389898392,1.0397151129519404,0.7758217277307676,0.3997011853778173,7.488848396478441,3.2103205514247555,0.6118902392215627,0.31532836067538406,0.24557477949445408,Energy,7 -ett1/15T/medium,CleanTS-65M,12.253657164687201,12.253657164687201,1.7322943408905513,1.057443626820784,0.7206279448470898,0.4075127849881611,7.4515702676535325,3.5005224131102493,0.6672029949384995,0.3301769953046115,0.2566701635559736,Energy,7 -ett1/15T/short,CleanTS-65M,5.9019814987755606,5.9019814987755606,1.1416246317681813,0.7136816287465364,0.5226301806554544,0.2426294411931719,4.819353770834878,2.429399411125219,0.45925759361307344,0.215814566676143,0.16760824934693597,Energy,7 -ett1/D/short,CleanTS-65M,36504.136039806544,36504.136039806544,125.84460061693949,1.5925663914889419,1.470830005691165,0.448790473029727,11.122889268275403,191.06055594969501,0.5018269066442289,0.33053503027649894,0.25804511854091944,Energy,7 -ett1/H/long,CleanTS-65M,159.91409875778925,159.91409875778925,7.214580519741805,1.3804573243752154,3365371955908.4795,0.4710505248377563,10.22757818475775,12.645714640058475,0.6055758732213825,0.3454906284480588,0.2737574985522353,Energy,7 -ett1/H/medium,CleanTS-65M,140.5262155328478,140.5262155328478,6.822481995537167,1.2937400491959585,2767290270185.285,0.4395823612099602,9.743631952986485,11.854375375060796,0.5677965061979523,0.32678073016095716,0.258182559665144,Energy,7 -ett1/H/short,CleanTS-65M,98.03359114250966,98.03359114250966,4.84050932980719,0.8177207096211127,0.4613921817392111,0.25797659770718645,6.007863682098215,9.90119140015532,0.46212911014600394,0.22592637378993966,0.17972196293332462,Energy,7 -ett1/W/short,CleanTS-65M,1131268.7257254464,1131268.7257254464,837.9969863891602,1.4855798247079222,0.6447298532085759,0.5381618110196931,10.392326532006834,1063.6111722454998,0.42323627878325676,0.33345900777080917,0.25313049549027955,Energy,7 -ett2/15T/long,CleanTS-65M,12.641062444656614,12.641062444656614,2.281659266153971,0.937506541131916,0.13980276683271597,0.16663955370585123,7.034217303118602,3.555427181740137,0.18947513758754242,0.12159374423492621,0.09675950063026013,Energy,7 -ett2/15T/medium,CleanTS-65M,11.976971080719478,11.976971080719478,2.1906759728325738,0.9031625396324539,0.1361398755520238,0.16255583301423088,6.701355274893454,3.460776080696276,0.1844310120153314,0.11674508181005905,0.09273032879775872,Energy,7 -ett2/15T/short,CleanTS-65M,7.639649296667249,7.639649296667249,1.74599394961482,0.751064059264478,0.10665639585155917,0.12709814977965184,5.619841339795207,2.7639915514826106,0.131216514983691,0.08288854614196466,0.06522859429438249,Energy,7 -ett2/D/short,CleanTS-65M,120120.96169084821,120120.96169084821,211.10713917565724,1.3901142993044433,0.43525348438156974,0.14276145110054622,12.384938333786268,346.58471069977713,0.19835956273612088,0.12082217860334402,0.09926532928408927,Energy,7 -ett2/H/long,CleanTS-65M,242.87549674220816,242.87549674220816,9.78132933762969,1.0790337949478506,0.18940505364485652,0.1831886881873721,7.711336176477741,15.584463312613885,0.21341723273696586,0.1339477784927137,0.10802245409948179,Energy,7 -ett2/H/medium,CleanTS-65M,253.32394365583147,253.32394365583147,10.102260330745152,1.0702309638712895,0.17757451631873677,0.1747808378367197,7.69356694133209,15.916153544617226,0.21448038645828998,0.13613444314711665,0.10822252941997056,Energy,7 -ett2/H/short,CleanTS-65M,99.00531064328693,99.00531064328693,6.239137008076622,0.7288205595855097,0.1210485771598253,0.10991796618770985,4.987141686297209,9.950141237353716,0.12619547030852085,0.07912961336646579,0.062262632906822026,Energy,7 -ett2/W/short,CleanTS-65M,3116920.4065290177,3116920.4065290177,1257.4354030064173,0.8709673884733995,0.15224901347288064,0.16006774215825967,9.503753023524526,1765.4802198067862,0.14809670794897695,0.10547954123446687,0.08915801591299392,Energy,7 -hierarchical_sales/D/short,CleanTS-65M,29.153439542479926,29.153439542479926,2.3054563049148458,0.745720599965087,0.6202869676143766,1.052882916378354,7.184936344468844,5.399392516059555,1.6564738292620151,0.7072884629594557,0.5775059687380758,Sales,1 -hierarchical_sales/W/short,CleanTS-65M,476.72122636993055,476.72122636993055,9.183625812879054,0.747419318126174,0.5625651143379824,0.47086217136011793,6.727918806541047,21.83394665125686,1.0024071705368696,0.4216247531147799,0.35732705318572333,Sales,1 -hospital/M/short,CleanTS-65M,3164.5460900307116,3164.5460900307116,18.584228619758278,0.7705296515264353,0.1907209945095461,0.17357669301535036,5.369740849613075,56.25429841381645,0.2042161662873408,0.06746506541093732,0.05396887023740965,Healthcare,1 -jena_weather/10T/long,CleanTS-65M,1467.8021032886204,1467.8021032886204,9.91342335312748,0.6374243012539635,0.8369992394120576,0.6433001134239098,6.534794547400555,38.311905503232545,0.23443711076596843,0.06066193519168585,0.04975128726723508,Nature,21 -jena_weather/10T/medium,CleanTS-65M,1559.77809871079,1559.77809871079,10.063912272858191,0.6185721979483099,0.8196521325106934,0.6546619971856625,6.08630822253793,39.49402611422125,0.24237416684971813,0.06176205852824723,0.051295154609447585,Nature,21 -jena_weather/10T/short,CleanTS-65M,748.1843526618301,748.1843526618301,5.583768914622556,0.29220630302475936,0.3233816162830876,0.5528585786326504,2.265995042925185,27.352958755166323,0.16989616327661536,0.034682204719020525,0.029822066795762173,Nature,21 -jena_weather/D/short,CleanTS-65M,418.1406242594038,418.1406242594038,10.700311713985034,1.3034188092681354,0.8245833318573368,0.4755352611224803,7.865257399454948,20.44848708974343,0.12313331223041782,0.06443336455447513,0.051413827772477584,Nature,21 -jena_weather/H/long,CleanTS-65M,1375.294339980632,1375.294339980632,11.940894247410158,1.0883991944445894,2.7000078934416667,0.6966790734144745,7.657828170845286,37.08496110259025,0.2232925650548231,0.07189741680398751,0.05905766666433646,Nature,21 -jena_weather/H/medium,CleanTS-65M,1345.9004229035627,1345.9004229035627,10.697815845285852,0.7655084682731764,2.79351218695486,0.6862423476955247,7.119980041412167,36.68651554595452,0.22449128956830378,0.06546183083711472,0.05404261017981618,Nature,21 -jena_weather/H/short,CleanTS-65M,1104.9613603672099,1104.9613603672099,8.566707332513985,0.5368868229803038,1.666315686255163,0.6206586529907702,4.698604734502142,33.24095907712667,0.20376495248316953,0.05251336787535976,0.042841740044608986,Nature,21 -kdd_cup_2018/D/short,CleanTS-65M,2974.623326464278,2974.623326464278,21.701980333837458,1.219805501483326,0.5142869594384242,0.4801226475984066,9.439144277578558,54.5401075032336,1.2217563179272777,0.4861474023098953,0.3791924518528169,Nature,1 -kdd_cup_2018/H/long,CleanTS-65M,4375.788436609605,4375.788436609605,25.182258330115,1.0477889124552735,0.9004897250875985,0.6369750928465591,7.839462395431958,66.14974252867206,1.552140264357224,0.5908775394654775,0.458216979722089,Nature,1 -kdd_cup_2018/H/medium,CleanTS-65M,5500.53832347396,5500.53832347396,26.492773258814694,1.0727541916038499,0.8466308493052767,0.5888711367724193,9.05088311139548,74.16561415827391,1.5524206241285592,0.5545417248157081,0.43813830468284815,Nature,1 -kdd_cup_2018/H/short,CleanTS-65M,4972.147362873047,4972.147362873047,23.872244337004055,0.9764807621426917,0.8240941214719566,0.5225553728086353,7.9199142573167345,70.51345519029009,1.475974323000653,0.4996893086394619,0.40165367467527224,Nature,1 -loop_seattle/5T/long,CleanTS-65M,92.20174206559065,92.20174206559065,5.354954664232315,0.8454478529900591,0.17166914982583023,0.12061257913044021,6.753253190770574,9.602173819796779,0.16976840728522274,0.09467669941123137,0.07510375447858089,Transport,1 -loop_seattle/5T/medium,CleanTS-65M,74.51142957594864,74.51142957594864,4.867850248702909,0.7737179588352738,0.15593407412954405,0.1109053855857613,6.017319161376385,8.632000322981263,0.15358102311985514,0.08660905857471395,0.06866858244997465,Transport,1 -loop_seattle/5T/short,CleanTS-65M,43.11260150657473,43.11260150657473,3.5075349229408124,0.5579530038926219,0.09257529315945257,0.07490157786251206,4.111420926825049,6.566018695265399,0.11259272426167817,0.060146480043020756,0.047269422758287244,Transport,1 -loop_seattle/D/short,CleanTS-65M,18.57973240516749,18.57973240516749,3.049241252186621,0.9176378483844494,0.05682349533949843,0.0561943626748273,6.962407212922172,4.310421372112881,0.07703298774913034,0.0544940143308299,0.043573038211785105,Transport,1 -loop_seattle/H/long,CleanTS-65M,60.61264933562172,60.61264933562172,4.358683961488391,0.9131533227378191,0.1349941218928615,0.10075194553325051,8.05470249349768,7.785412598932809,0.1391357232945755,0.07789550494437279,0.06329274501206944,Transport,1 -loop_seattle/H/medium,CleanTS-65M,66.2239799928616,66.2239799928616,4.531657566940575,0.9275631153962406,0.12036321337562597,0.10313099785116804,8.617610168431597,8.137811744742047,0.14383491375237448,0.08009654139730336,0.06571625071811742,Transport,1 -loop_seattle/H/short,CleanTS-65M,60.70895523423057,60.70895523423057,4.261321281999581,0.8655888545195888,0.12095751806492265,0.09790238747792573,8.357494879422589,7.7915951662179275,0.13791915882241718,0.07542972063456517,0.0622124855449094,Transport,1 -m_dense/D/short,CleanTS-65M,9316.96240198206,9316.96240198206,51.946601132993344,0.7504809074520636,0.13026995624410745,0.10738054213700471,6.927894005298149,96.52441350239876,0.16709803777374177,0.08992725056156602,0.07237018288142993,Transport,1 -m_dense/H/long,CleanTS-65M,34765.609654224536,34765.609654224536,82.58546437958141,0.7254612665150665,0.2810987441059,0.2003287257677243,7.556326878963531,186.4553824758742,0.32278184699178003,0.1429676546375075,0.11766625157903284,Transport,1 -m_dense/H/medium,CleanTS-65M,34512.06633884006,34512.06633884006,79.96313526153564,0.6978936781622972,0.2730701257273608,0.19687438170115154,7.44780491775512,185.7742348627496,0.323223710059924,0.13912575802749044,0.11450026427958455,Transport,1 -m_dense/H/short,CleanTS-65M,50343.301783057315,50343.301783057315,96.31221923192342,0.841403228679521,0.30340629458202995,0.22977066790892017,9.42045395622259,224.37313070654722,0.3979820616722905,0.17083389376193264,0.14114884457775367,Transport,1 -m4_daily/D/short,CleanTS-65M,383465.66690247733,383465.66690247733,170.69666058763607,3.2081064716704435,0.036927138060732216,0.030332773207370232,26.08011128676324,619.2460471431992,0.09565640050632376,0.026367916606962305,0.02092342695969357,Econ/Fin,1 -m4_hourly/H/short,CleanTS-65M,416266.4552957866,416266.4552957866,169.205297327176,1.322701379817929,0.10123713118220633,0.09574189621785689,9.673663173801586,645.187147497365,0.08808229659126059,0.02310026050859238,0.020823279011280376,Econ/Fin,1 -m4_monthly/M/short,CleanTS-65M,1908783.3338688153,1908783.3338688153,577.6685870080171,0.9837793846017734,0.16600613850283993,0.13589435501860161,8.067107808366893,1381.5872516308245,0.2871412603835251,0.12005936357740439,0.09642437481217975,Econ/Fin,1 -m4_quarterly/Q/short,CleanTS-65M,1883431.565311517,1883431.565311517,584.988072511673,1.2479256582402214,0.12082483374203003,0.10567335962267195,10.103191593957614,1372.3817126847462,0.22971253166569353,0.09791670195605363,0.0788195700145567,Econ/Fin,1 -m4_weekly/W/short,CleanTS-65M,367975.06675794435,367975.06675794435,298.10729381658547,2.1940876307583523,0.07092927528801889,0.07255722331380036,14.157770313365264,606.6094845598315,0.1105148407721758,0.05431052587822617,0.042407753828505486,Econ/Fin,1 -m4_yearly/A/short,CleanTS-65M,4162170.1811671406,4162170.1811671406,973.9929443163,3.6487848096801136,0.17898699063501528,0.15901303755530824,37.38846090225593,2040.1397454995922,0.3271458629844328,0.15618428247966717,0.13193186447748662,Econ/Fin,1 -restaurant/D/short,CleanTS-65M,157.55816897095514,157.55816897095514,7.623011608512125,0.729013932346573,0.6455305182790182,0.41960701008765783,5.128291537730095,12.552217691346623,0.5776821413519493,0.3508286565641627,0.27525791304525254,Sales,1 -saugeen/D/short,CleanTS-65M,1044.869330921173,1044.869330921173,11.873578217824301,2.636848440138105,0.2706558911005656,0.30933215459187824,34.78444458777649,32.324438601794355,1.0469962525841157,0.38458802183609575,0.328085682050341,Nature,1 -saugeen/M/short,CleanTS-65M,397.759041922433,397.759041922433,12.60980578831264,0.7388380321378877,0.38100790693646386,0.3585934695743379,4.870683760161396,19.94389736040659,0.5987755907042731,0.37858417405178296,0.2980638209858531,Nature,1 -saugeen/W/short,CleanTS-65M,1255.5320473432541,1255.5320473432541,17.289710915088655,1.4025695837802723,0.47267574444413185,0.45805939808487894,14.113494559227897,35.43348765424107,1.0695892856066285,0.5219042993020577,0.4228650601068679,Nature,1 -solar/10T/long,CleanTS-65M,21.67777349450969,21.67777349450969,1.9364289433029929,0.8487347798643053,2.6994967977587954,1.4704553988534845,6.117334493905399,4.655939593090711,1.0063068677381686,0.4185281414356188,0.324399494876512,Energy,1 -solar/10T/medium,CleanTS-65M,23.045038762722257,23.045038762722257,2.008993427111654,0.8822361224160759,3.3693835700083974,1.4663263069301515,6.601560215948542,4.800524842423196,1.0652555400654822,0.44580362531892637,0.35073008304774533,Energy,1 -solar/10T/short,CleanTS-65M,36.74563303097377,36.74563303097377,2.705380199371028,1.2000939009804032,6.852672141710116,1.5538207939760238,11.889313469323726,6.061817634255733,1.7633820794603803,0.786994800823228,0.6341716334851415,Energy,1 -solar/D/short,CleanTS-65M,156768.750486618,156768.750486618,260.6531283413407,1.0170102433051629,1.2094920840576617,0.4263525755155986,5.835141668924148,395.9403370289746,0.5719989491517509,0.3765550047846042,0.30897672487822153,Energy,1 -solar/H/long,CleanTS-65M,952.9622441869583,952.9622441869583,15.582906187479804,1.1897389378572798,4.551584204645209,1.4675696661485855,8.014566285976032,30.87008655943417,1.0703222989347025,0.5402878266167355,0.39539951318004596,Energy,1 -solar/H/medium,CleanTS-65M,719.216486843137,719.216486843137,12.558621390312547,0.9578050952378071,4.802171791139955,1.4231838226318358,7.407784415238228,26.818211850217324,0.9660545844700282,0.4523908542633308,0.355301571573648,Energy,1 -solar/H/short,CleanTS-65M,769.8665664988606,769.8665664988606,11.87201372019108,0.9178560271155648,3.826693847001346,1.4241380780068964,6.3027752866255335,27.746469442054437,1.0227441044790493,0.43760637965046906,0.34642641108658595,Energy,1 -solar/W/short,CleanTS-65M,1428398.523922217,1428398.523922217,976.2975027265339,1.070836182186609,0.22128818097123265,0.1936384287737582,6.169907035377626,1195.1562759414423,0.24396126547451438,0.19928672010457912,0.149285378680503,Energy,1 -sz_taxi/15T/long,CleanTS-65M,15.73027650990717,15.73027650990717,2.699990148625822,0.5094621704736275,6886944535416.432,0.398718404124605,3.9420435081189225,3.96614126197078,0.36709894798502724,0.2499062634592007,0.19841673921452202,Transport,1 -sz_taxi/15T/medium,CleanTS-65M,16.575554460541817,16.575554460541817,2.7664459081796497,0.5396955805361071,10366029297642.486,0.4076075570196168,3.908486704392643,4.071308691384359,0.3789032209855931,0.25746396177472386,0.20292486987960812,Transport,1 -sz_taxi/15T/short,CleanTS-65M,17.052128991106997,17.052128991106997,2.7614426509328234,0.5515912063321681,1200949502644.5789,0.3991129708650348,3.9597880693291128,4.1294223556215455,0.38611866322008004,0.25820670619598285,0.20292541416074839,Transport,1 -sz_taxi/H/short,CleanTS-65M,7.291834672928875,7.291834672928875,1.8692658472264934,0.5697771938661932,1.0598877103855977,0.30021190461822045,4.121075664148837,2.7003397328723056,0.2515533110968186,0.17413364972779155,0.13760897611444112,Transport,1 -temperature_rain/D/short,CleanTS-65M,189.99358632464586,189.99358632464586,5.987437476609687,1.428506549050293,37.81032682708845,1.4943231913885242,20.33263286000924,13.78381610166959,1.6227195876027847,0.7048796937783194,0.5724113313069737,Nature,1 -us_births/D/short,CleanTS-65M,204714.1703125,204714.1703125,305.8128889973958,0.4501687478600908,0.029061880807081858,0.02894541874527931,3.4816281268105844,452.4535007185821,0.0424138442439747,0.028667476815044057,0.022875985906004082,Healthcare,1 -us_births/M/short,CleanTS-65M,91195325.33333333,91195325.33333333,8114.59765625,0.9191595239565,0.024908685435851414,0.02526882415016492,5.28980637396075,9549.624355613854,0.02966106814002031,0.02520388499568461,0.018538053172532615,Healthcare,1 -us_births/W/short,CleanTS-65M,3899898.0089285714,3899898.0089285714,1519.235072544643,1.3805831792570447,0.020336782626275505,0.02053045361701931,8.920201263600932,1974.8159430510407,0.026807932787474797,0.020623467142071045,0.015951417115224253,Healthcare,1 diff --git a/results/CleanTS-65M/config.json b/results/CleanTS-65M/config.json deleted file mode 100644 index adc7e7ddb0beb003ced4b44f4159ada11db77769..0000000000000000000000000000000000000000 --- a/results/CleanTS-65M/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "CleanTS-65M", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/EINK/CleanTS-65M", - "code_link": "https://github.com/Taihuachen-cfair/gift-eval/blob/main/notebooks/CleanTS.ipynb", - "org": "Shandong University", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/Credence/all_results.csv b/results/Credence/all_results.csv deleted file mode 100644 index 757ab4aa939b3abe52719240a22497542bbf7400..0000000000000000000000000000000000000000 --- a/results/Credence/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[mean],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -us_births/W/short,Credence,1981293.4285714286,1981293.4285714286,1083.517299107143,1083.517299107143,0.9851020567651093,0.014681688376835414,0.014698320509071038,5.875969468433975,1407.5842527434827,0.019107818211127305,0.014708641091713007,0.011495080911903467,Healthcare,1 -us_births/M/short,Credence,40867930.666666664,40867930.666666664,5254.341145833333,5254.341145833333,0.5927523081570498,0.016176501909891765,0.016360392021733113,3.926255227348656,6392.8030367489555,0.01985600264656178,0.01631994777531297,0.012191297260903702,Healthcare,1 -us_births/D/short,Credence,133748.92,133748.92,217.39641927083332,217.39641927083332,0.32004981611634753,0.02095420440038045,0.020664739053686813,2.906081303081846,365.7169944096118,0.03428300060366135,0.020379150236448283,0.016607321670457812,Healthcare,1 -loop_seattle/5T/short,Credence,39.341390133789815,39.341390133789815,3.4488411642572583,3.4488411642572583,0.5475653017247267,0.09420587692959513,0.07341289413122502,4.062700942496223,6.272271529022785,0.10755560898652276,0.05914001171713785,0.04666659247956807,Transport,1 -temperature_rain/D/short,Credence,177.56792622396415,177.56792622396415,5.652711626659176,5.652711626659176,1.312655580709039,21.572101249152418,1.4994294631885086,18.812045122520473,13.325461576394423,1.56875914113273,0.6654736112269262,0.5384378173256141,Nature,1 -solar/W/short,Credence,1618500.0510948906,1618500.0510948906,1019.4490654938412,1019.4490654938412,1.1135661677897826,0.2294308756389757,0.19920882231417994,7.083897484048863,1272.2028341011078,0.2596883960120456,0.2080950344849133,0.14975132761771653,Energy,1 -solar/H/short,Credence,612.8842461422718,612.8842461422718,10.855683811191163,10.855683811191163,0.8363616938018386,4.3890899273190005,1.4030977378835967,3.8725069803458037,24.756499068775287,0.9125328005647738,0.40014412065098237,0.30192240009823956,Energy,1 -solar/H/medium,Credence,654.3222665754258,654.3222665754258,11.452482319110212,11.452482319110212,0.8733596288079221,4.209102444885201,1.4064806977271924,4.367720358164698,25.579723739231934,0.9214413573837426,0.41254514556579364,0.30496866283542734,Energy,1 -solar/H/long,Credence,659.8980072739254,659.8980072739254,11.677593137767387,11.677593137767387,0.8883293339654287,5.199635621910256,1.4007543238194116,4.607906696278808,25.688480049896402,0.8906665482727325,0.40488349454488537,0.3027778017842999,Energy,1 -solar/D/short,Credence,126225.54379562044,126225.54379562044,246.8741474680657,246.8741474680657,0.9622487048832767,1.071720927997227,0.42039874022610235,5.4565046948888645,355.2823437712891,0.5132619978753026,0.3566490718570327,0.27276837547316396,Energy,1 -loop_seattle/5T/medium,Credence,71.41364503968734,71.41364503968734,5.093492446757692,5.093492446757692,0.8083730075140683,0.16475848317023278,0.1164860430850618,5.671783217460128,8.45065944407224,0.15035459638406284,0.09062369701274862,0.07121101323088352,Transport,1 -solar/10T/short,Credence,23.539989542439034,23.539989542439034,2.1887697168800373,2.1887697168800373,0.9660921516137635,5.732001036809698,1.5235234096415824,4.146622526999156,4.851802710584905,1.4113888707543,0.6367128680457225,0.4850725346939111,Energy,1 -solar/10T/medium,Credence,19.14013862500173,19.14013862500173,1.8458668655432633,1.8458668655432633,0.8075838638230288,3.6547726236736673,1.4500841494337575,4.122946170140241,4.37494441393279,0.9708175500099963,0.40960519231842096,0.3076506932632776,Energy,1 -solar/10T/long,Credence,18.056756893755068,18.056756893755068,1.8420211284603418,1.8420211284603418,0.806373794011146,3.2992514245977604,1.451856543261004,4.279447364950502,4.249324286725487,0.9184234751321878,0.39812340313784844,0.3004464647174967,Energy,1 -saugeen/W/short,Credence,942.9406433105469,942.9406433105469,14.756837844848633,14.756837844848633,1.1970570094750175,0.38110325336456297,0.39088950434610864,9.993948972552701,30.70733859048268,0.9269265492835072,0.4454474210311822,0.35492562559280716,Nature,1 -saugeen/M/short,Credence,398.6060732886905,398.6060732886905,12.38219488234747,12.38219488234747,0.7254962333638438,0.3544088999430339,0.35536092788013146,4.510588903279304,19.965121419332526,0.5994128398116464,0.37175063660480695,0.29144970720918795,Nature,1 -saugeen/D/short,Credence,1208.375546875,1208.375546875,12.9156396484375,12.9156396484375,2.8698198740037757,0.279986317952474,0.32440258311666154,27.68817149753579,34.761696547708944,1.1259396485299231,0.4183406510767468,0.34336834472401007,Nature,1 -restaurant/D/short,Credence,137.24760647787372,137.24760647787372,7.040552813543417,7.040552813543417,0.6754662227188343,0.6717557948967472,0.3893615799211119,4.575243882488138,11.71527236037958,0.539163978037998,0.32402255327623436,0.2535126833884472,Sales,1 -m4_yearly/A/short,Credence,3780024.204455308,3780024.204455308,895.934612433683,895.934612433683,3.348256083413248,0.16374748665209743,0.15027195955494627,29.41936183421947,1944.2284342266234,0.31176604018430804,0.14366726741840957,0.11598507225523212,Econ/Fin,1 -m4_weekly/W/short,Credence,232332.47075208914,232332.47075208914,244.55150859760016,244.55150859760016,1.9494262628433383,0.05878148325153201,0.05877691661898692,14.170818467317096,482.00878700713446,0.08781452588996383,0.04455349230564091,0.035531322684503125,Econ/Fin,1 -m4_quarterly/Q/short,Credence,1742952.2965591634,1742952.2965591634,548.1433355496724,548.1433355496724,1.1560533456661968,0.11048622191604227,0.1000720684147396,8.754063292796006,1320.2091866667051,0.22097976942769262,0.09174954175930349,0.07306034864599445,Econ/Fin,1 -loop_seattle/5T/long,Credence,85.82239456797888,85.82239456797888,5.55685168164488,5.55685168164488,0.8769157370110361,0.17667073546712408,0.1251199539657702,6.009984734394903,9.264037703290011,0.16379009152002544,0.09824628036397032,0.07698314146112534,Transport,1 -loop_seattle/D/short,Credence,17.645019802097444,17.645019802097444,2.86297519160252,2.86297519160252,0.8645312995221023,0.0535209201560793,0.05283242815704088,6.756373593918668,4.200597552979509,0.07507028967822517,0.051165191205409855,0.04134675922365924,Transport,1 -loop_seattle/H/short,Credence,47.946900669674115,47.946900669674115,3.746456097549025,3.746456097549025,0.7653407189523878,0.10460991644081032,0.08581307221906702,6.294736471175906,6.924370055801041,0.12256839163085016,0.06631608283376958,0.05339762466401302,Transport,1 -loop_seattle/H/medium,Credence,57.86505272454891,57.86505272454891,4.164693070639029,4.164693070639029,0.8570583795027924,0.11197156709291126,0.09470824299503841,6.73576679840819,7.606908223749575,0.1344512536547473,0.0736104850964862,0.059128058621891286,Transport,1 -m4_monthly/M/short,Credence,1815345.2099774666,1815345.2099774666,549.0623064496076,549.0623064496076,0.9111372657455721,0.15733080874314462,0.12986576380123901,7.380592623105529,1347.34747187853,0.2800250587707948,0.11411399645706309,0.09084637608334828,Econ/Fin,1 -m4_hourly/H/short,Credence,828029.1790862912,828029.1790862912,176.71600902234107,176.71600902234107,0.6964065261388817,0.08728426046993422,0.07786551172130717,5.320036363968451,909.9610865780422,0.12422978591526504,0.02412563821954452,0.019395067795332222,Econ/Fin,1 -loop_seattle/H/long,Credence,54.64845212156271,54.64845212156271,4.00427861694294,4.00427861694294,0.8397277271999716,0.11943499325967795,0.09290104400763127,6.745365559909182,7.392459138984991,0.13211311906639867,0.0715618074782964,0.05734788367588174,Transport,1 -m_dense/D/short,Credence,7972.15925925926,7972.15925925926,42.987706886574074,42.987706886574074,0.6401850558696381,0.11753934078343803,0.09248046558949191,6.674480116870909,89.28694898617188,0.15456891596489558,0.07441807933546361,0.06155807237551119,Transport,1 -m4_daily/D/short,Credence,352801.7620344686,352801.7620344686,175.02748212440778,175.02748212440778,3.2512786127729063,0.03761074586027258,0.029845498968524658,24.072089857688447,593.9711794645162,0.09175213209320293,0.027036908885540886,0.021327812914881717,Econ/Fin,1 -m_dense/H/short,Credence,43176.36068793403,43176.36068793403,85.52203759087456,85.52203759087456,0.7634620809393522,0.28628258212707786,0.20327998800888086,7.927807493511263,207.78922177999038,0.3685663383029902,0.15169479903271224,0.12493999679182302,Transport,1 -m_dense/H/medium,Credence,35982.63861111111,35982.63861111111,77.82327365451388,77.82327365451388,0.6750128309610246,0.26559301938140745,0.1872690853568675,7.668932035708439,189.69090281589973,0.3300382172157605,0.1354026688341385,0.11224754902476472,Transport,1 -m_dense/H/long,Credence,33365.56061728395,33365.56061728395,76.83492283950618,76.83492283950618,0.6635170516510428,0.2672928797984264,0.18675558962070568,6.952728745461807,182.66242256491603,0.3162156812193223,0.1330126203625371,0.11005157685452689,Transport,1 -sz_taxi/15T/short,Credence,16.542044675684966,16.542044675684966,2.713050236195435,2.713050236195435,0.5413397080011091,1052505187116.6085,0.3928081980379438,3.809533145696896,4.0671912514270785,0.3802997913230212,0.253681810097318,0.19921639274901,Transport,1 -sz_taxi/15T/medium,Credence,16.26387031750801,16.26387031750801,2.740928010859041,2.740928010859041,0.5343902512638006,9547619092255.111,0.40399731733067784,3.8099304304134582,4.032848908341101,0.37532390158845375,0.25508909914303285,0.2012139307529757,Transport,1 -sz_taxi/15T/long,Credence,15.437054843304843,15.437054843304843,2.6829066958522523,2.6829066958522523,0.5059875993489118,6579843666924.634,0.39717084255889173,3.840094691304683,3.929001761682583,0.36366138787427643,0.24832505346930456,0.19733732948712,Transport,1 -sz_taxi/H/short,Credence,7.16485830453726,7.16485830453726,1.8405209810305865,1.8405209810305865,0.560501863683958,1.1060429566945784,0.29464194692709156,3.884994825869058,2.67672529493358,0.24935348013052103,0.17145588781257642,0.13474393705755836,Transport,1 -kdd_cup_2018/H/short,Credence,4366.834940322303,4366.834940322303,22.11540043388925,22.11540043388925,0.9232946272744946,0.8672966286987673,0.4865883925935503,7.410279287889337,66.08203190219186,1.3832166080652961,0.46291538401008847,0.37108780090840277,Nature,1 -kdd_cup_2018/H/medium,Credence,5179.8799228146545,5179.8799228146545,24.97020068765824,24.97020068765824,1.0231086828735965,0.9228104975619303,0.54850995879905,8.400142448957727,71.97138266571412,1.5064913881423707,0.5226715244705741,0.4151927584293543,Nature,1 -kdd_cup_2018/H/long,Credence,4096.371096336147,4096.371096336147,23.719996162660532,23.719996162660532,0.9993973293827084,0.9820791882338883,0.5971977879268109,7.430312720550339,64.00289912446269,1.5017666496739275,0.5565669626653209,0.4358402935082522,Nature,1 -kdd_cup_2018/D/short,Credence,2888.927228269001,2888.927228269001,21.12797560489862,21.12797560489862,1.1952953836900297,0.5291491156602859,0.4646283894662181,9.049681395728863,53.74874164358642,1.2040288863995243,0.47328908848004403,0.37458848125983474,Nature,1 -hospital/M/short,Credence,2834.9423776135036,2834.9423776135036,17.72574792867741,17.72574792867741,0.7534969901084262,0.19072928070141304,0.171835050294532,5.087405135235105,53.244176936201235,0.19328872633073854,0.06434858115093305,0.0507050420998614,Healthcare,1 -hierarchical_sales/W/short,Credence,463.5034832065388,463.5034832065388,8.880850751521224,8.880850751521224,0.7153194852123632,0.5190059380716755,0.4543076030266486,6.562817201134447,21.52913103695871,0.9884129365880346,0.40772420194954395,0.3430515558311208,Sales,1 -hierarchical_sales/D/short,Credence,28.729503773834747,28.729503773834747,2.292235123317761,2.292235123317761,0.7405168655034932,0.6308665744765213,1.0598728859067335,6.879417539248687,5.359991023671098,1.6443858877424427,0.7032323508612238,0.5677235036013336,Sales,1 -electricity/W/short,Credence,35677167338.926125,35677167338.926125,25591.681211289415,25591.681211289415,1.3913983663991643,0.12404143190513252,0.09128269235749974,10.49396140509632,188884.00498434514,0.430970971545659,0.05839177179682473,0.046217533866786224,Energy,1 -electricity/H/short,Credence,1525633.6620027788,1525633.6620027788,160.67214860830222,160.67214860830222,0.8772902814770303,0.16777715137842344,0.11950123753924799,9.817165267459965,1235.1654391225406,0.5830862954147924,0.07584872838969985,0.06213982642124559,Energy,1 -electricity/H/medium,Credence,5290059.0672032265,5290059.0672032265,233.38883924329602,233.38883924329602,1.0428183929415498,0.23639100108804828,0.12653187804705238,8.500259797002409,2300.0128406605095,0.897770802276334,0.09109935464083072,0.07259659109428923,Energy,1 -electricity/H/long,Credence,8082463.757674925,8082463.757674925,270.9681885674737,270.9681885674737,1.1622827572419159,0.2912307404545631,0.13844394046323913,9.334767830385333,2842.967421142023,1.121265560031176,0.1068697781920744,0.08449070789043954,Energy,1 -electricity/D/short,Credence,1207132443.7733333,1207132443.7733333,4100.2335774211715,4100.2335774211715,1.3495531914085952,0.36024277470196775,0.09100998989390749,10.744090104684702,34743.811589595825,0.5709562248343537,0.06738045646679959,0.05300851707025143,Energy,1 -electricity/15T/short,Credence,131392.11671461287,131392.11671461287,50.44386328695057,50.44386328695057,0.9636366174137869,0.16142363722445216,0.1621331247068244,10.250187797556046,362.48050528906083,0.7145205404168453,0.09943479974947253,0.08247345849118831,Energy,1 -electricity/15T/medium,Credence,235986.55371074952,235986.55371074952,51.61692567416355,51.61692567416355,0.8166610134195745,0.12697121274678602,0.13014899771855473,6.932790746300936,485.784472488314,0.8370555552892645,0.08894115977235362,0.0718116872997907,Energy,1 -car_parts/M/short,Credence,1.3443089131499977,1.3443089131499977,0.4581129398338979,0.4581129398338979,0.8370634232671614,0.8049785677915271,1.882188799020695,13.648863809842867,1.1594433634938783,2.7802262494483663,1.0985078362949185,0.9442805842923128,Sales,1 -covid_deaths/D/short,Credence,378188.1140062069,378188.1140062069,108.68806111041765,108.68806111041765,34.97108897386375,0.09075509984307899,0.36807812346997815,622.2833335716725,614.9700106559725,0.23138544838779088,0.04089441000156739,0.03322237663759045,Healthcare,1 -electricity/15T/long,Credence,304050.5199430368,304050.5199430368,57.02846970557594,57.02846970557594,0.8642380743404394,0.1308116295869595,0.13206093708343256,7.0548848664548025,551.4077619539253,0.8699133493064254,0.08996940288182395,0.07201943067928405,Energy,1 -jena_weather/H/short,Credence,1075.1322678401755,1075.1322678401755,8.347192584123826,8.347192584123826,0.5210019579149167,1.395962111432094,0.6077056093037256,4.3659107858696204,32.78920962512173,0.20099576256484544,0.05116775786617829,0.04101708441768538,Nature,21 -jena_weather/H/medium,Credence,1186.5049882192461,1186.5049882192461,9.803821769593254,9.803821769593254,0.7456613391720759,2.20877425457439,0.6701472923918745,6.077787788960267,34.4456817064091,0.2107792314111413,0.059991323008195575,0.04826633343479608,Nature,21 -jena_weather/H/long,Credence,1130.2408585482804,1130.2408585482804,10.76051781063988,10.76051781063988,0.9077538308252998,2.155136504315446,0.6517823862605664,6.585881604300189,33.619054991898274,0.20242395594625853,0.064790238267697,0.05271971803611088,Nature,21 -jena_weather/D/short,Credence,360.8901181175595,360.8901181175595,9.340482197110616,9.340482197110616,0.9937645238911081,0.6019073589602502,0.44512733380400055,6.466172778180113,18.997108151441353,0.11439364040524394,0.056244969136883385,0.044242303975310475,Nature,21 -jena_weather/10T/short,Credence,788.7378164518448,788.7378164518448,5.466740783055624,5.466740783055624,0.27077702219171607,0.34787774990199616,0.5432388317757615,2.095932056787003,28.08447643186258,0.17443980223730696,0.03395531276477491,0.027615336053660053,Nature,21 -jena_weather/10T/medium,Credence,1454.8164051226552,1454.8164051226552,9.531055528604966,9.531055528604966,0.5982117847501426,0.7790376010579546,0.6450781945562306,5.254830764954998,38.14205559644964,0.23407714347343386,0.058491924924733145,0.04750007854072305,Nature,21 -jena_weather/10T/long,Credence,1403.6030564888445,1403.6030564888445,9.603791792935164,9.603791792935164,0.6463866061709543,0.842861191998725,0.6357578268223568,5.518140532401427,37.46469079665338,0.22925285496906522,0.05876724564494566,0.04812969741164127,Nature,21 -ett2/W/short,Credence,2964300.0,2964300.0,1166.6429966517858,1166.6429966517858,0.7712872750954578,0.12006293875830513,0.1294604556465046,8.67896865903544,1721.7142620074912,0.14442543109405384,0.0978634616918564,0.08384630584244074,Energy,7 -ett2/H/short,Credence,106.10571594238282,106.10571594238282,6.302439953031994,6.302439953031994,0.7239941760542563,0.11788169814215134,0.10784536704876845,5.071351588078193,10.300762881572549,0.13064232602877393,0.07993246952550748,0.06344846420135918,Energy,7 -ett2/H/medium,Credence,233.66578311011904,233.66578311011904,9.449562000093007,9.449562000093007,1.032354057589049,0.17500831834954556,0.1678265140953696,7.527016676251642,15.286130416495832,0.20599042459635983,0.127338916757391,0.1027503914161881,Energy,7 -ett2/H/long,Credence,226.2981398809524,226.2981398809524,9.15840076264881,9.15840076264881,1.0217813610000894,0.17940548926936442,0.1728095737269313,8.01697175490175,15.043209095168237,0.20600517628013984,0.12541725316838293,0.10239122253536842,Energy,7 -ett2/D/short,Credence,104521.9746031746,104521.9746031746,192.6950706845238,192.6950706845238,1.2724323302823013,0.42282790834941564,0.13220255239697806,12.488432766901184,323.29858428884995,0.1850322992280923,0.11028446677892514,0.09198330814035699,Energy,7 -ett2/15T/short,Credence,7.6752898534139,7.6752898534139,1.6792806761605399,1.6792806761605399,0.7270627958244034,0.10533737187218753,0.12432981407460246,5.261593292780878,2.7704313478976337,0.13152223676251082,0.07972143068923164,0.06354002244339592,Energy,7 -ett2/15T/medium,Credence,11.214409790039063,11.214409790039063,2.084752943250868,2.084752943250868,0.8610712135269182,0.12968421052994128,0.15661658323096042,6.4771817574897375,3.348792288279323,0.1784631954248445,0.11110025343347499,0.08826969731334394,Energy,7 -ett2/15T/long,Credence,11.190497039310516,11.190497039310516,2.114925769624256,2.114925769624256,0.8782253118995644,0.1333790227136247,0.15985195841485994,6.932672596729436,3.3452200285348224,0.17827282177199527,0.11270821697021899,0.09089333221966853,Energy,7 -ett1/W/short,Credence,1253685.2857142857,1253685.2857142857,894.6405552455357,894.6405552455357,1.56493604769043,0.6707758222307477,0.5668904919966399,8.252324477635039,1119.680885660859,0.445547748666724,0.35599882998835986,0.26151685293320026,Energy,7 -ett1/H/short,Credence,93.07295207069033,93.07295207069033,4.802978152320499,4.802978152320499,0.7993522891032343,0.44706320422036305,0.25579450661474556,5.52860614848358,9.647432408194957,0.45028514893373367,0.22417464472785556,0.17307522213873766,Energy,7 -ett1/H/medium,Credence,140.69910714285714,140.69910714285714,6.895627267020089,6.895627267020089,1.262242807376277,3277745247375.108,0.4354628725381994,8.852059419337555,11.861665445579602,0.5681457201383867,0.33028423684694397,0.25615459786096345,Energy,7 -ett1/H/long,Credence,142.68213872354497,142.68213872354497,7.136190812045304,7.136190812045304,1.3543608574786439,3848029630019.5146,0.4718904249402749,9.327672371124457,11.94496290172326,0.5720183882162103,0.3417367136167985,0.26268251189377545,Energy,7 -ett1/D/short,Credence,41578.201587301584,41578.201587301584,137.32098524305556,137.32098524305556,1.6730259584770841,1.2093455844455294,0.47705758852649643,10.7880273791868,203.90733578589462,0.5355693811170812,0.36067812272450184,0.27508666226535256,Energy,7 -ett1/15T/short,Credence,4.662322132928031,4.662322132928031,1.0652648131052653,1.0652648131052653,0.6776549174663714,0.47372050907300867,0.23782750697385283,4.4623681646332365,2.159241101157541,0.4081864440644926,0.2013793901085708,0.15492806223767028,Energy,7 -ett1/15T/medium,Credence,8.843387063647073,8.843387063647073,1.543362310500372,1.543362310500372,0.9725446187423443,0.7157097092589441,0.3798302858706109,6.735816570377376,2.9737832913053825,0.566806002136499,0.294166364987163,0.2276705717681897,Energy,7 -ett1/15T/long,Credence,8.392706124441965,8.392706124441965,1.5329517376612103,1.5329517376612103,0.9781559345058916,0.7324397854734935,0.38279931160659103,7.023077284469047,2.897016762885911,0.5521742430960129,0.2921821082604248,0.22835065268151128,Energy,7 -bizitobs_service/10S/short,Credence,1997.9256756671523,1997.9256756671523,17.842970648467855,17.842970648467855,0.7237281656425067,0.05660382793693946,0.045697773864466266,5.523405269582212,44.69816188242143,0.03311382788882575,0.013218643322133988,0.010252276296669338,Web/CloudOps,2 -bizitobs_service/10S/medium,Credence,4556.090228174603,4556.090228174603,26.533218974764385,26.533218974764385,0.9647815539561216,0.06107691689143105,0.053858580242839446,9.224766822666446,67.49881649462162,0.05080119754019902,0.019969524926733585,0.01712799940726285,Web/CloudOps,2 -bizitobs_service/10S/long,Credence,129421.48566137566,129421.48566137566,76.17236400462963,76.17236400462963,1.3370194159847177,0.07860183392882977,0.07584869148135491,17.690718788730695,359.7519779811859,0.266515781940229,0.05643092574182126,0.05114256791376083,Web/CloudOps,2 -bizitobs_l2c/H/short,Credence,53.148751395089285,53.148751395089285,4.4130919925750245,4.4130919925750245,0.4394223234234872,0.40957613242901775,0.6008141524671522,2.716785663925974,7.290319018745976,0.392966886461398,0.23787697294599847,0.18519021372096006,Web/CloudOps,7 -bizitobs_l2c/H/medium,Credence,70.43136160714286,70.43136160714286,4.814887927827381,4.814887927827381,0.48739709599198294,0.48642187228409317,0.7550630961382749,3.3618633245902725,8.392339459718182,0.5081689015175606,0.2915487774247358,0.2310097444693461,Web/CloudOps,7 -bizitobs_l2c/H/long,Credence,80.96149553571429,80.96149553571429,5.304670836433532,5.304670836433532,0.567811158897224,0.6271207733074948,0.7826679040163869,4.518210895075608,8.997860608817758,0.5496152954115319,0.32402460491221463,0.2596866164869739,Web/CloudOps,7 -bizitobs_l2c/5T/short,Credence,18.752176766168503,18.752176766168503,2.5036689122517903,2.5036689122517903,0.26211307580694493,0.14346513937073793,0.20338597296761687,2.3325914088500728,4.330378362934179,0.14872271583610872,0.08598612152962629,0.0688893920556652,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Credence,79.86641754357993,79.86641754357993,5.566847363945579,5.566847363945579,0.5466309265585393,0.3667817146245512,0.6491725086326049,3.3221314757504254,8.9368013037988,0.470207186369805,0.2928980456170996,0.2268416466401471,Web/CloudOps,7 -bizitobs_l2c/5T/long,Credence,96.21248511904761,96.21248511904761,6.004980856274802,6.004980856274802,0.5920033658093712,0.4431110912448344,0.7644439044400726,3.509862998208682,9.808796313465155,0.5507492488242376,0.33717069761732316,0.26147389840582913,Web/CloudOps,7 -bizitobs_application/10S/short,Credence,350140.7411111111,350140.7411111111,322.68848524305554,322.68848524305554,1.0363998400381071,0.014510410096910264,0.014646855095585802,5.98163681328812,591.7269143034742,0.022820543229205716,0.012444805786338428,0.009627715825301733,Web/CloudOps,2 -bizitobs_application/10S/medium,Credence,2891940.6933333334,2891940.6933333334,951.5125,951.5125,2.1562148620175603,0.030659735997517902,0.031199743209209332,16.877748526605277,1700.5706963644097,0.06677640616761417,0.03736309540638263,0.030324168755055052,Web/CloudOps,2 -bizitobs_application/10S/long,Credence,6060475.164444445,6060475.164444445,1497.650138888889,1497.650138888889,3.1371308382989205,0.045429212782118054,0.046441943993136246,40.28921595812408,2461.803234307008,0.09494191063847077,0.05775837957825591,0.05071972157378911,Web/CloudOps,2 -bitbrains_rnd/H/short,Credence,1818641.0937801444,1818641.0937801444,149.75174539903512,149.75174539903512,5.821635419912829,1.3724220529567637,0.5525362211662677,201.93884283165897,1348.570018122954,5.902002318764386,0.6553869185185583,0.6166622943900338,Web/CloudOps,2 -bitbrains_rnd/5T/short,Credence,1583829.069178179,1583829.069178179,119.75410507263618,119.75410507263618,1.6432523465160396,1.0642778200798646,0.6526269984224173,52.60856245988648,1258.5027092454663,5.149429799470652,0.48999922903598125,0.40117422401416386,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Credence,2661676.643867556,2661676.643867556,191.70914324027754,191.70914324027754,4.388420485709358,2.126143836495376,0.7265995961416354,159.79294402692142,1631.4645702152272,6.864403118729712,0.8066180931982825,0.7392441131936254,Web/CloudOps,2 -bitbrains_fast_storage/5T/short,Credence,1584125.514225858,1584125.514225858,153.81169814644826,153.81169814644826,0.6797086451842964,1.9402938693640246,0.7305929533302803,13.27901284781038,1258.6204806159235,3.951757024078375,0.48293069109935455,0.3834838314310236,Web/CloudOps,2 -bitbrains_rnd/5T/long,Credence,2601235.93952657,2601235.93952657,192.63408900834602,192.63408900834602,3.314314874292069,2.180930377394704,0.6953672589385378,115.38980282967576,1612.834752703007,6.178406623419138,0.7379378013964702,0.6916969658437743,Web/CloudOps,2 -bitbrains_fast_storage/H/short,Credence,3455853.832133464,3455853.832133464,351.5712851839025,351.5712851839025,1.0135300721552434,4.280307864664333,0.5771279368595401,18.285717614805222,1858.9926928671516,5.298732994898757,1.0020923567966016,0.7365040793156743,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Credence,3092077.0591349504,3092077.0591349504,305.3724750419639,305.3724750419639,0.982785470940544,4.76438113700308,0.8080981329653673,20.957243144873516,1758.430282705274,5.342596876048292,0.927805922837249,0.6860463744559588,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Credence,3979524.15212993,3979524.15212993,382.79015117880357,382.79015117880357,0.8960704218847759,5.079025463954972,0.8231851048126867,15.810022827829737,1994.8744702687259,5.2718576144049445,1.0116000797480518,0.7373601867620561,Web/CloudOps,2 diff --git a/results/Credence/config.json b/results/Credence/config.json deleted file mode 100644 index b85fa0016cc1447e0780d8c35e90c7ebc25db885..0000000000000000000000000000000000000000 --- a/results/Credence/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "Credence", - "model_type": "agentic", - "model_dtype": "float32", - "org": "ContinualIST", - "model_link": "https://app.credence.continualist.ai", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/Credence.ipynb", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/DLinear/config.json b/results/DLinear/config.json index 9eadf456cca6354a8e10365b4d4b30480bdec41f..ca288eba8c06216a3f58190bf5f32d86a8b842cf 100644 --- a/results/DLinear/config.json +++ b/results/DLinear/config.json @@ -1,8 +1,5 @@ { "model": "DLinear", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "The Chinese University of Hong Kong", - "testdata_leakage": "No", - "replication_code_available": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/DeOSAlphaTimeGPTPredictor-2025/all_results.csv b/results/DeOSAlphaTimeGPTPredictor-2025/all_results.csv deleted file mode 100644 index b7590b52831856459b5a8a217671a6d05cad3611..0000000000000000000000000000000000000000 --- a/results/DeOSAlphaTimeGPTPredictor-2025/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[mean],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,num_variates,model -bitbrains_fast_storage/5T/long,4253620.759478553,3942542.6739726365,439.27634263683444,351.5178219117222,0.9039245531592373,4.6281072811839135,0.7718292500679558,16.66245616890376,2062.4307890152077,5.450388819032396,0.9289566546633562,0.6717524145217136,2.0,DeOSAlphaTimeGPTPredictor-2025 -bitbrains_fast_storage/5T/medium,3390856.937975832,2938339.3379816045,346.56688551014156,289.0369573141382,0.9944598806113677,4.2767788058971545,0.7593947512182156,21.183553172197,1841.4279616579663,5.594766748606289,0.8781741558533743,0.638447648013664,2.0,DeOSAlphaTimeGPTPredictor-2025 -bitbrains_fast_storage/5T/short,1551046.723143738,1551046.7231391892,199.03606752846943,152.61767237339856,0.6903232578959704,1.7680923118860032,0.6547266354607726,13.58432050068248,1245.4102629830356,3.910280264577749,0.4791817524275708,0.3799864694504142,2.0,DeOSAlphaTimeGPTPredictor-2025 -bitbrains_fast_storage/H/short,2863379.3530778517,2858801.8119597975,333.7310782677839,317.24704607234844,1.009603623087645,3.9762860880015842,0.483425179076808,19.185955232386707,1692.1522842455686,4.8231836388213525,0.9042571104362971,0.6829068010829976,2.0,DeOSAlphaTimeGPTPredictor-2025 -bitbrains_rnd/5T/long,2460772.825727656,2408570.118859373,222.0122219114635,181.69137884896642,3.3142125023001303,1.6193283618889214,0.6555396986648099,115.77886533466787,1568.6850626328842,6.009279106112113,0.6960187682159416,0.6315904645719728,2.0,DeOSAlphaTimeGPTPredictor-2025 -bitbrains_rnd/5T/medium,2208052.9063216285,2208052.9063216285,190.83315102306372,144.0089043382807,4.393525889829723,0.32305618616953297,0.6856240051864132,160.5133725802978,1485.9518519526853,6.252156934968296,0.6059187916090644,0.6035812787955468,2.0,DeOSAlphaTimeGPTPredictor-2025 -bitbrains_rnd/5T/short,1478593.9452985392,1478593.9452985392,132.24529301855372,117.6014834052782,1.6544388112624189,1.0387110630549943,0.5783237763109859,52.89403977264285,1215.9744838188562,4.975416577471558,0.48119131851919283,0.4043434461782052,2.0,DeOSAlphaTimeGPTPredictor-2025 -bitbrains_rnd/H/short,1783555.8804036537,1783555.8804030593,169.2135864729052,148.29365223143196,5.781759136287309,1.2192616701026289,0.5174088677310278,201.11955856694308,1335.4983640587698,5.8447943639973925,0.6490056059492009,0.6090687621404847,2.0,DeOSAlphaTimeGPTPredictor-2025 -bizitobs_application/10S/long,6242166.897780693,6242166.897779226,1548.4186111112454,1517.7727777779935,3.162220056910506,0.045887777540439344,0.04694717831083904,46.64809741595336,2498.432888388371,0.0963545699834774,0.05853442932772233,0.05183357021206572,2.0,DeOSAlphaTimeGPTPredictor-2025 -bizitobs_application/10S/medium,3158747.093364888,3158747.0933510866,1198.610625011638,990.344791669118,2.1551507567526325,0.030904585520496675,0.031468766530429126,17.109516540426156,1777.28644100005,0.06978880767152827,0.03888792520996555,0.032571434984432206,2.0,DeOSAlphaTimeGPTPredictor-2025 -bizitobs_application/10S/short,330682.1688930182,330682.1688917817,361.4938541675514,312.1507986122335,1.0240725901806635,0.014836207495836553,0.014869072172421302,6.584896618107508,575.049709931811,0.022177368772688753,0.012038408600073285,0.009465487395238809,2.0,DeOSAlphaTimeGPTPredictor-2025 -bizitobs_l2c/5T/long,71.76699466885863,71.76699466875343,5.1773234442847045,5.177323443727447,0.5177408840924764,0.4239388838612668,0.7123863777304614,3.4590991505986834,8.471540277229773,0.47566432159098626,0.2906989712528437,0.2303326741032111,7.0,DeOSAlphaTimeGPTPredictor-2025 -bizitobs_l2c/5T/medium,69.30413079386094,69.30413079318365,5.2389994305602565,5.238999430349592,0.5193918306440176,0.36818355182680057,0.6304700306495206,3.3012861138951686,8.324910257400804,0.43801271795643903,0.2756484224943615,0.21389099516999327,7.0,DeOSAlphaTimeGPTPredictor-2025 -bizitobs_l2c/5T/short,20.796484375013037,18.631319754529244,2.6896362304725394,2.5361993698875196,0.2639866333364688,0.14270905494991945,0.1653895500092193,2.013703845538512,4.5603162582229935,0.15661971907454936,0.08710335212225287,0.06955900884674134,7.0,DeOSAlphaTimeGPTPredictor-2025 -bizitobs_l2c/H/long,75.96023065504829,75.96023065499504,5.085054718630753,5.085054718514457,0.5469474777486598,0.5924043065248448,0.7568616594592258,4.614917158073707,8.715516660246815,0.532368912138001,0.31060981857856795,0.2475699455709395,7.0,DeOSAlphaTimeGPTPredictor-2025 -bizitobs_l2c/H/medium,66.89926990338093,66.89926990338093,4.662852550711636,4.662852550697028,0.4704569255579601,0.44389625158329654,0.7154494065515075,3.7467534570786833,8.179197387481215,0.4952628252993955,0.28234280441101617,0.2210817991528417,7.0,DeOSAlphaTimeGPTPredictor-2025 -bizitobs_l2c/H/short,47.7616451205187,47.761645120408474,4.527385651120547,4.257519918777629,0.4260981553127596,0.3789699883770351,0.508568173365855,2.722583522033143,6.910980040523201,0.37251952350997714,0.22949122731907196,0.17948148016365645,7.0,DeOSAlphaTimeGPTPredictor-2025 -bizitobs_service/10S/long,127076.42243398998,127076.42243395673,79.42200438184732,75.45964947091149,1.2985016180215645,0.08195709067054473,0.0787758091519217,16.251980995698705,356.47780075902193,0.26409015111112305,0.0559029207124086,0.05047527419900773,2.0,DeOSAlphaTimeGPTPredictor-2025 -bizitobs_service/10S/medium,5948.295238335104,5948.295238335104,46.433770927866526,28.979506448669007,0.9590529440287021,0.06441918751562445,0.05834050738632913,8.456956522343942,77.12519198215831,0.05804623694938459,0.021810659458517706,0.01769820600240515,2.0,DeOSAlphaTimeGPTPredictor-2025 -bizitobs_service/10S/short,1955.9106878539405,1955.9106878411258,19.65338293652902,17.747296627017594,0.7260925792297359,0.05836188099374469,0.04773929010625359,5.309902408325444,44.225679054730236,0.03276379690316555,0.013147764708106795,0.010208833214830786,2.0,DeOSAlphaTimeGPTPredictor-2025 -car_parts/M/short,1.3394600888639876,1.339460088862455,0.48362268995694985,0.45101323792316167,0.8384127237589359,0.7785075259297707,1.8298891156523993,15.399481758281741,1.1573504606919947,2.7752076832203385,1.0814834794035164,0.9328574448342613,1.0,DeOSAlphaTimeGPTPredictor-2025 -covid_deaths/D/short,358791.05764801614,358791.05764504505,136.35700757035056,101.76966635342906,33.916951071636554,0.08764320944666984,0.1400870630363708,590.762443172419,598.9917008170607,0.22537354246126404,0.03829133223363033,0.03181424860101123,1.0,DeOSAlphaTimeGPTPredictor-2025 -electricity/15T/long,316119.8529768915,301448.91936380346,59.71403826308797,59.02531437689754,0.9040155318285895,0.13831809860832778,0.12937102134746206,7.4223454570586895,562.2453672347042,0.8870110003711399,0.0931196672449461,0.07445031627374542,1.0,DeOSAlphaTimeGPTPredictor-2025 -electricity/15T/medium,245526.12763987406,240116.12032448663,54.79427162786796,54.38962232546216,0.8556548344189464,0.13165588285119179,0.1259242787334942,7.180637396708639,495.5059309835476,0.8538066156817281,0.09371879620841937,0.07522088522791406,1.0,DeOSAlphaTimeGPTPredictor-2025 -electricity/15T/short,144065.9828621229,143022.6599099412,56.85607637131312,55.702191019165014,1.0581410281030779,0.17361034036167078,0.16279361898283753,10.380229834802687,379.5602493177106,0.7481880832376662,0.10979999818978063,0.09087798759641126,1.0,DeOSAlphaTimeGPTPredictor-2025 -electricity/D/short,1180566865.841817,1180566865.841817,4251.209920750075,4179.36230630656,1.3763940062284636,0.4039522552728689,0.09069691375596056,11.162884282955497,34359.37813526037,0.5646387062966604,0.06868080239424007,0.05412539110984783,1.0,DeOSAlphaTimeGPTPredictor-2025 -electricity/H/long,6575584.818809653,6450800.09264038,279.8932897345126,253.54705105126112,1.144049401490494,0.29030430335502827,0.12943613493775744,9.290428822669746,2564.290314844288,1.011355341724897,0.09999888194206247,0.07976140872341046,1.0,DeOSAlphaTimeGPTPredictor-2025 -electricity/H/medium,4886055.97121122,4741568.389193524,247.566379951903,229.15208896412497,1.0456241708076948,0.21637195396034342,0.11974627945159859,8.665759305551648,2210.442483126128,0.8628085446907899,0.08944561186336435,0.0711945876650941,1.0,DeOSAlphaTimeGPTPredictor-2025 -electricity/H/short,1568839.2158561724,1497918.3992814068,172.59574973290387,165.51642595724473,0.9074777822981339,0.17948404936304313,0.1142611007151426,9.940571663832813,1252.5331196643751,0.5912850807302402,0.07813557322183602,0.06381346164497988,1.0,DeOSAlphaTimeGPTPredictor-2025 -electricity/W/short,29395794717.297134,29395794717.297134,23643.86088474908,23643.8608847633,1.3635557497399038,0.12850979827443412,0.08910185097979569,10.97984374034458,171452.01870287853,0.3911969310475399,0.05394748855225393,0.043553793142031824,1.0,DeOSAlphaTimeGPTPredictor-2025 -ett1/15T/long,8.09003766743483,8.09003766743483,1.5517131115533138,1.5517131115150467,1.0044400796193644,0.7146971268068739,0.3973296053447555,7.029236447420757,2.8442991522401893,0.5421262146796589,0.29575804456148486,0.22985128943962974,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett1/15T/medium,8.631577148460376,8.63157714845876,1.5448723493374275,1.5448723493317673,0.9899225182223955,0.6927709018820842,0.38790635971813475,6.820593503996374,2.937954585840273,0.5599770158530061,0.2944541798644596,0.22932511415299706,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett1/15T/short,4.44892694383327,4.448926943828179,1.0932999247596507,1.0595989408947368,0.6751031107377107,0.4559715139224807,0.2355906895230078,4.810845251378434,2.109247956934681,0.3987356616373223,0.20030830580102948,0.15642445108661351,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett1/D/short,42060.292063505185,42060.2920635174,136.30761408730785,136.93593439981436,1.6288935332389554,1.1616377573188807,0.46812579442597474,9.96362452226127,205.0860601394088,0.5386653397302716,0.35966677391246177,0.2761893641170094,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett1/H/long,141.5620205030697,141.5620205028619,7.220682715593782,7.001847201558025,1.3206593911042752,2267774406939.624,0.4673595933057494,8.727281281121034,11.897983883963683,0.5697686789590044,0.3353032975344297,0.25782678572961454,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett1/H/medium,134.41744326639196,134.41744326639196,6.66846458799428,6.668464587986121,1.2267932617041875,3293537138068.2505,0.43795205979120916,8.310043139788538,11.59385368487941,0.5553181702177943,0.3194036817966904,0.24767421232002307,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett1/H/short,98.04556412472718,98.04556412472718,4.89483836037888,4.894838360377725,0.815035338079106,0.4436365876883383,0.2578675769624287,5.611072639956796,9.901796005004705,0.46215733888299987,0.2284621365408063,0.17604642943593804,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett1/W/short,1001010.3571458799,1001010.3571440697,801.5694056920333,800.4866071428896,1.4441483369955501,0.5297911167173551,0.5109938894004435,7.739463438486893,1000.50505103463,0.3981248396100472,0.31853272679552225,0.23747014640347563,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett2/15T/long,11.056184779586244,10.850311259926812,2.093517097794991,2.0884621465775326,0.8656004641239669,0.1321820120883892,0.1587395852709635,6.754839937372683,3.3250841763158667,0.1771997458118978,0.11129792818176008,0.08914688430064874,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett2/15T/medium,10.991811126240052,10.991811126240052,2.0503414965936058,2.0503414965451054,0.8470324155417513,0.1298428251795036,0.15523655846015547,6.355190838004052,3.3153900413435333,0.17668312929670793,0.10926640522503395,0.08720802717142152,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett2/15T/short,7.498252650677291,7.49825265067168,1.7334863571895873,1.6455150785918775,0.7236933910825138,0.10645821528786173,0.12580321175712306,5.34133849425684,2.738293748062327,0.12999654557973084,0.07811845462805862,0.06281700151480411,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett2/D/short,101776.23492084746,101776.23492084631,222.04579613103235,190.62036210342978,1.2742572379270396,0.36247067527317184,0.1325021531847723,10.83025557818198,319.0238782925878,0.18258578270795806,0.10909706258040223,0.09198037762457413,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett2/H/long,220.84508928706228,220.8450892861193,10.071580997928558,9.100840153778526,1.0069646760456616,0.17777589995428214,0.1698159677011355,6.729109204325192,14.860857622862818,0.20350800738436536,0.12462900138210854,0.10067781931225084,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett2/H/medium,224.64012276830596,224.6401227680292,9.703669084846057,9.31343238467317,1.0031231116756554,0.16778200892113057,0.16325636364172338,6.568052130035229,14.987999291709976,0.2019729288774629,0.1255044906277177,0.10058578998485238,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett2/H/short,104.57532457625437,102.14484747031099,6.295174280803538,6.227093796505019,0.7202662092352479,0.11713471286859643,0.10737862814040787,5.2490319320498635,10.226207731913838,0.1296967593478573,0.0789768756263061,0.0627255436134226,7.0,DeOSAlphaTimeGPTPredictor-2025 -ett2/W/short,2954849.7142996034,2954849.7142996034,1509.4345703126253,1180.033621653956,0.7651932745008136,0.12030822890173286,0.12894910573984775,6.102820239970382,1718.9676303814142,0.14419503080905566,0.09898672983874489,0.0866468906016945,7.0,DeOSAlphaTimeGPTPredictor-2025 -hierarchical_sales/D/short,28.903637866889174,28.90363786688866,2.3195691869231267,2.284841844512912,0.73815801120245,0.6053563253408021,1.040062963515211,6.807029823013262,5.376210362968433,1.6493618014205191,0.70096417479952,0.5684473728224652,1.0,DeOSAlphaTimeGPTPredictor-2025 -hierarchical_sales/W/short,450.3789672856262,450.37896728560054,8.75041240759359,8.750412407570824,0.7124265352172441,0.513843610035865,0.4523889715388913,6.5038621563974255,21.222133900379156,0.9743185478841987,0.40173571377664136,0.3398582822825932,1.0,DeOSAlphaTimeGPTPredictor-2025 -hospital/M/short,2029.1258259692104,2029.1258259639478,16.61691487101014,16.61691487100847,0.7443310113747209,0.18992392595104704,0.1704683531786961,4.885548360577601,45.04581918411509,0.1635267839158291,0.06032325966484719,0.047892525894090154,1.0,DeOSAlphaTimeGPTPredictor-2025 -jena_weather/10T/long,1399.3250419990766,1391.2911375663787,10.08374880120996,9.595989376658023,0.6241920999878635,0.7801082080252444,0.5997969327573386,5.738901634589255,37.407553274694,0.22890322069325755,0.05871950443415084,0.0485058832880773,21.0,DeOSAlphaTimeGPTPredictor-2025 -jena_weather/10T/medium,1424.370786048148,1422.2240981241625,9.894016494093766,9.50355451840132,0.5942903710102938,0.7368490756510262,0.5936458308888161,5.373854923418483,37.74083711377018,0.23161487250073112,0.05832315161305223,0.04773714361323882,21.0,DeOSAlphaTimeGPTPredictor-2025 -jena_weather/10T/short,759.0725198415837,738.1843750006267,5.621278191378988,5.226546223960061,0.26343062958846103,0.33317428148237416,0.42951180463714606,2.260254301959869,27.551270748217473,0.17112792194342122,0.03246340259297783,0.02670504345286187,21.0,DeOSAlphaTimeGPTPredictor-2025 -jena_weather/D/short,349.4009920642669,349.4009920642669,9.827138264996151,9.243148561513651,0.9881361987712975,0.5806005900697853,0.44123424348380613,6.275877127672205,18.692270917795256,0.11255802202533859,0.05565886156613879,0.0443666263226294,21.0,DeOSAlphaTimeGPTPredictor-2025 -jena_weather/H/long,1341.0650793651917,1280.9198412716457,12.88958669244824,11.353404017860706,0.848679507700969,2.0154693480586623,0.5885197621266812,6.354005177440056,36.620555421309376,0.22049631648974094,0.06836007091529255,0.05511376635902902,21.0,DeOSAlphaTimeGPTPredictor-2025 -jena_weather/H/medium,1235.7421627018116,1235.7421626994706,10.663989451807481,9.99287574405354,0.7294967251342954,1.925273364391764,0.6012966534801629,5.798438377005324,35.153124508381005,0.21510820338219955,0.06114817894530063,0.04974594646115791,21.0,DeOSAlphaTimeGPTPredictor-2025 -jena_weather/H/short,1098.5380878806175,1082.2564745201207,8.411170702140346,8.233126762218854,0.5197023695631651,1.0454592079080458,0.5492415656834666,4.363655498655423,33.14420142167577,0.20317183962398788,0.050468537029991545,0.04081315116061153,21.0,DeOSAlphaTimeGPTPredictor-2025 -kdd_cup_2018/D/short,2890.3727641468035,2890.3727641468035,21.71406054872549,21.100617561010385,1.1915478125323853,0.5092614670607326,0.4641057599201362,9.098005629521849,53.762187122054506,1.2043300408538633,0.4726762241201443,0.3752128249284896,1.0,DeOSAlphaTimeGPTPredictor-2025 -kdd_cup_2018/H/long,4074.3718164320326,4074.3718164320326,24.97580497817963,23.73963270663827,1.0077571619741115,0.9440227702250731,0.6001084442617497,7.55887128427067,63.83080617093927,1.4977286463167236,0.5570277126463301,0.4398758686576078,1.0,DeOSAlphaTimeGPTPredictor-2025 -kdd_cup_2018/H/medium,5162.125594031812,5162.125594031708,26.337250565738664,25.19616223403388,1.0355005771671983,0.8758914582401748,0.5530126460117214,8.437830428368406,71.84793381880787,1.5039073774978038,0.5274013079340047,0.41900703788885973,1.0,DeOSAlphaTimeGPTPredictor-2025 -kdd_cup_2018/H/short,4248.6639332977975,4248.6639332977975,22.74372853721177,22.171244770032132,0.9326107374705869,0.8398279994767779,0.4925777727582769,7.385201784162187,65.18177608271931,1.3643726270235133,0.4640843083637569,0.3738848939158539,1.0,DeOSAlphaTimeGPTPredictor-2025 -loop_seattle/5T/long,77.11275533082305,76.26844627912176,5.218633842943717,5.069633642935327,0.8026927443962691,0.16191277474992558,0.11609966805172223,6.500225712019464,8.781386868303805,0.1552567255119691,0.08963215960482573,0.07164677523234288,1.0,DeOSAlphaTimeGPTPredictor-2025 -loop_seattle/5T/medium,65.28178691431181,65.04676438339199,4.869518228616982,4.741585256228038,0.7544920277888282,0.15321824238156487,0.10961362494289678,5.908934467465426,8.079714531733593,0.14375472415580481,0.08436254266079049,0.06722841412013317,1.0,DeOSAlphaTimeGPTPredictor-2025 -loop_seattle/5T/short,41.20642213777828,39.48958434118928,3.648289078751747,3.4821369308945482,0.5517518947080956,0.0935271301506315,0.07413102849721089,4.23630344005342,6.419222860890379,0.1100754998932326,0.059710962501675395,0.04733966464858467,1.0,DeOSAlphaTimeGPTPredictor-2025 -loop_seattle/D/short,16.954627838011934,16.954627838000278,2.8908717861120823,2.835673253677245,0.8626503798096022,0.05315495698690858,0.05225526162100914,6.65466072962664,4.117599766612992,0.07358701485678003,0.05067727357556304,0.040881989395174616,1.0,DeOSAlphaTimeGPTPredictor-2025 -loop_seattle/H/long,48.53299363638576,48.532993636225356,4.394733995439638,3.8958870829088723,0.8221961743620819,0.12143411915204987,0.09067764938361811,7.169324816436328,6.96656254090699,0.12450179950713408,0.06962471802278614,0.056986323021464655,1.0,DeOSAlphaTimeGPTPredictor-2025 -loop_seattle/H/medium,45.79930340631264,45.799303405866034,4.774017567088997,3.8346156637099957,0.7993525494125979,0.11227852237731702,0.08791354625238003,6.97529546938046,6.767518260502388,0.11961511420597191,0.06777639496411644,0.0553994187856082,1.0,DeOSAlphaTimeGPTPredictor-2025 -loop_seattle/H/short,39.99444752841507,39.99444752826104,4.075954878793375,3.5146181503346905,0.7249853379588023,0.10246250665834389,0.08120649798905903,6.584976470875178,6.324116343678411,0.11194329036198067,0.06221231532333908,0.05083510114255265,1.0,DeOSAlphaTimeGPTPredictor-2025 -m4_daily/D/short,355053.27898889576,355053.27898889576,182.77459687311375,171.29611643058774,3.2334241675245994,0.03667107287218712,0.030032586985326185,25.289477981404925,595.8634734474804,0.09204443968693242,0.02646051614169074,0.020984501292426423,1.0,DeOSAlphaTimeGPTPredictor-2025 -m4_hourly/H/short,524888.7468544954,524888.74685422,184.97413446058692,179.83831521745918,0.7026456829966296,0.08595879104794038,0.07853409403197069,5.450904856636703,724.4920612775932,0.09890916765274817,0.02455189954746391,0.01993858655807464,1.0,DeOSAlphaTimeGPTPredictor-2025 -m4_monthly/M/short,1816001.8201495733,1795522.5163709847,555.0341335723622,545.869526475876,0.9049255926322908,0.15686037505119715,0.12877609934631393,7.460456798739623,1347.5911175685148,0.2800756976842591,0.11345042756601133,0.09030605030825117,1.0,DeOSAlphaTimeGPTPredictor-2025 -m4_quarterly/Q/short,1734981.6866695217,1702830.7906676312,561.5495734305604,540.9206061199349,1.1325982000962613,0.1098312443097516,0.09820457299554819,8.751259652718709,1317.1870355684082,0.22047391484416878,0.09054058416212608,0.0722018678624429,1.0,DeOSAlphaTimeGPTPredictor-2025 -m4_weekly/W/short,230569.2575534201,230569.25755338106,249.35544133878093,249.35544133852406,1.9898576085037938,0.06014517029480914,0.0600645180966819,13.669943895790501,480.1762775829464,0.08748067109185284,0.04542869435054725,0.0366340884948834,1.0,DeOSAlphaTimeGPTPredictor-2025 -m4_yearly/A/short,3295300.592534206,3295300.592514397,818.0042513815904,818.0042513817816,2.9572461143373148,0.15779903586010974,0.1351394788916291,29.207010380936264,1815.2962823002936,0.2910911720716508,0.13117077284628423,0.10693787298370853,1.0,DeOSAlphaTimeGPTPredictor-2025 -m_dense/D/short,7833.570370414966,7833.57037038976,46.134473379651375,43.863246527787545,0.6614871883282976,0.1220631525197228,0.09669913963035626,7.499356335145696,88.50745940548192,0.1532195041991084,0.07593376796368202,0.06262679111641507,1.0,DeOSAlphaTimeGPTPredictor-2025 -m_dense/H/long,33158.79901238634,33158.799012367665,84.81094425300059,77.57057098768628,0.6618874921210435,0.26586954123966955,0.1891220281153741,7.102992215281159,182.09557658654404,0.31523436169235014,0.1342861253952138,0.11246622901632454,1.0,DeOSAlphaTimeGPTPredictor-2025 -m_dense/H/medium,34187.495555574875,33244.25777780607,79.91017849436965,76.3162673611413,0.6623563249696179,0.25951886451188766,0.18574232313378589,7.239070196011131,184.89860885245943,0.3217002199460585,0.13278066367244493,0.11056874143264282,1.0,DeOSAlphaTimeGPTPredictor-2025 -m_dense/H/short,43336.59111112981,43336.59111112981,89.82139122857336,86.36722222228076,0.7687776769784612,0.28625055021444,0.20635660807296174,8.007660876804296,208.17442472871093,0.3692495557269625,0.15319393088996813,0.12771561532765952,1.0,DeOSAlphaTimeGPTPredictor-2025 -restaurant/D/short,137.62853740546763,137.62853740546763,7.08888314888116,7.0695383277301,0.6778660814216722,0.636700599637461,0.3906647666246966,4.6051194352411695,11.731518972642347,0.539911684777353,0.3253565337978677,0.25477450749931735,1.0,DeOSAlphaTimeGPTPredictor-2025 -saugeen/D/short,1065.7159375007227,1065.7159375007227,12.316854858417477,12.220719401044551,2.7150354225147337,0.2735478464762931,0.31319437662768135,28.1984425287095,32.645304984035896,1.0573892206161633,0.3958320131746541,0.3288654873311478,1.0,DeOSAlphaTimeGPTPredictor-2025 -saugeen/M/short,392.646484375755,392.64648437546776,12.768525623144779,12.19125075568336,0.7143579153981151,0.3479558172680732,0.35563718704952724,4.6935604167640115,19.815309343427955,0.5949150318609729,0.36601792109533543,0.2932331336151843,1.0,DeOSAlphaTimeGPTPredictor-2025 -saugeen/W/short,907.9439239506402,907.9439239506402,14.385106277487498,14.385106277467237,1.1671291864276832,0.3630518913279545,0.3851867675783005,10.389655591437212,30.132107857742685,0.9095627312968267,0.4342263945655614,0.350148571341534,1.0,DeOSAlphaTimeGPTPredictor-2025 -solar/10T/long,18.80319181692724,18.40688726050459,1.9477469072527454,1.8709905892013798,0.8186809213712213,2.9404132683492583,1.357354144357515,5.328555160231914,4.336264730955335,0.9372142615093426,0.4043846426232464,0.3194498740412499,1.0,DeOSAlphaTimeGPTPredictor-2025 -solar/10T/medium,20.173338310120435,19.571514529436325,2.0579187169032864,1.9093761232309583,0.8357536449135131,3.5471902003901987,1.353209077155651,5.089605269381387,4.491473957413131,0.996675870088541,0.4236981282740315,0.33173821039407747,1.0,DeOSAlphaTimeGPTPredictor-2025 -solar/10T/short,23.815074714672978,23.815074714645778,2.34341084974562,2.308319195750749,1.0192779366614757,5.411258468967385,1.3197214159816126,5.1465064215440615,4.8800691301119326,1.4196115690617732,0.6714898040684493,0.5199798844919598,1.0,DeOSAlphaTimeGPTPredictor-2025 -solar/D/short,123798.74148431614,123798.74148431614,254.33213104494772,245.21808090040446,0.9561017212942808,1.065309401092407,0.4182334435828432,4.664311463399523,351.85045329559404,0.5083040848438055,0.35425660454976826,0.2727292479834795,1.0,DeOSAlphaTimeGPTPredictor-2025 -solar/H/long,652.854014599495,652.854014599495,12.366314554213716,10.864412256727627,0.825937782884831,4.459468810961277,1.1941705061079826,4.811222981051886,25.551008093605304,0.8859000159943209,0.3766889727694429,0.28654990965068294,1.0,DeOSAlphaTimeGPTPredictor-2025 -solar/H/medium,603.5896593693805,603.5896593693805,11.445277230464601,10.270328657259938,0.7835878401920398,3.6642062038055583,1.1759299482537997,4.404438154667622,24.56806177477843,0.8849989272643313,0.36996120929890386,0.2850024227849284,1.0,DeOSAlphaTimeGPTPredictor-2025 -solar/H/short,588.2543779630774,588.2543779630774,11.064874813919044,9.665173997984033,0.7440553016398815,3.522677695682349,1.2377798262678776,4.103001319256935,24.253955923993832,0.8940089179939837,0.3562615424568007,0.27294675614942343,1.0,DeOSAlphaTimeGPTPredictor-2025 -solar/W/short,955186.6131531405,955186.6131531405,768.0469177725836,768.0469177746065,0.8387951669406846,0.16764145785131884,0.1530267008909601,4.4762029580493845,977.3364892152906,0.19949880510028145,0.15677757256358346,0.12235764216103137,1.0,DeOSAlphaTimeGPTPredictor-2025 -sz_taxi/15T/long,15.600195868953676,15.579733128561866,2.8021362826776994,2.698012375356528,0.5089470465511851,6334988927385.851,0.3988973677328475,3.8112374490563576,3.9497083270734805,0.36557796707054574,0.2497232194471518,0.1980404579291058,1.0,DeOSAlphaTimeGPTPredictor-2025 -sz_taxi/15T/medium,16.40649205395576,16.32286992521618,2.8214082375670047,2.7555689102566867,0.53719561770002,9524933370965.082,0.4061782967331419,3.8232887519052947,4.0504928161837,0.3769659653609456,0.25645168168751875,0.2022432887641494,1.0,DeOSAlphaTimeGPTPredictor-2025 -sz_taxi/15T/short,16.610637138470764,16.610637138469496,2.7577188600126092,2.719239305955147,0.5426127994831207,1026926057730.3031,0.3924660828233243,3.8603915702596523,4.075614939916522,0.38108742756532993,0.25426050481162676,0.1997074672454658,1.0,DeOSAlphaTimeGPTPredictor-2025 -sz_taxi/H/short,7.158167651578888,7.158167651576342,1.8844109478170346,1.8444965069111492,0.5624451478519223,1.0607816256024378,0.29524866739909483,3.942628190398139,2.6754752197654303,0.24923702677849716,0.17182623179973372,0.1352493850038147,1.0,DeOSAlphaTimeGPTPredictor-2025 -temperature_rain/D/short,180.44707241609044,180.44707241607912,5.7659389208737295,5.735549409489232,1.327752004679803,19.735320254678147,1.4632016736177644,18.911672832050773,13.433058937415923,1.5814261953054163,0.6752258011518504,0.5471146718345656,1.0,DeOSAlphaTimeGPTPredictor-2025 -us_births/D/short,157681.28000103257,157681.28000104247,269.4766666670808,232.98156250108607,0.34298029573050015,0.02253959973663114,0.022174895604550025,3.6081882000133683,397.0910223122422,0.03722406113398077,0.02184013094814161,0.018176382837941485,1.0,DeOSAlphaTimeGPTPredictor-2025 -us_births/M/short,41875856.00003636,41875856.00003636,5445.316406257483,5335.279947918206,0.6016601079779709,0.016455836594109513,0.016646799941862193,3.4488602977103437,6471.155692767414,0.020099365461953362,0.016571343142754772,0.01243337653612138,1.0,DeOSAlphaTimeGPTPredictor-2025 -us_births/W/short,2057186.5714337921,2057186.5714337921,1119.4234793532196,1119.423479353628,1.0174865839357723,0.015180262071758228,0.015214066420294317,6.411477597879721,1434.2895702868673,0.019470340278204105,0.015196063967799083,0.011881630832069218,1.0,DeOSAlphaTimeGPTPredictor-2025 diff --git a/results/DeOSAlphaTimeGPTPredictor-2025/config.json b/results/DeOSAlphaTimeGPTPredictor-2025/config.json deleted file mode 100644 index 0f7fedec87477862bb4d5b68fbce601da011e1ab..0000000000000000000000000000000000000000 --- a/results/DeOSAlphaTimeGPTPredictor-2025/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "DeOSAlphaTimeGPTPredictor-2025", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/vencortexio/DeOSAlphaTimeGPTPredictor-2025", - "org": "vencortex®", - "testdata_leakage": "No", - "replication_code_available": "No" - -} \ No newline at end of file diff --git a/results/FFM/all_results.csv b/results/FFM/all_results.csv deleted file mode 100644 index d6e1aa74ee2f6766ef57221d9ae06a4ebbc42765..0000000000000000000000000000000000000000 --- a/results/FFM/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,FFM,8903325.654779235,8903325.654779235,1521.842824846526,6.026356387392948,0.24435385771160803,0.26451350037941007,56.324858941119004,2983.844106983345,0.4784732308379391,0.2440345497701022,0.20076095840073002,Econ/Fin,1 -electricity/W/short,FFM,187037308362.7029,187037308362.7029,46409.90172579654,1.8805847993835016,0.26979581169214834,0.11477051486873681,14.212165553355,432478.1015990323,0.9867723259745242,0.10589208217685121,0.08150800038476508,Energy,1 -ett1/W/short,FFM,1337116.8064313617,1337116.8064313617,880.9140491485596,1.5126805821746863,0.7932343049240964,0.5277601581599031,9.280081076424592,1156.3376697277322,0.46013436594431056,0.35053673167271937,0.27395368727129027,Energy,7 -ett2/W/short,FFM,4735916.554199219,4735916.554199219,1583.3739852905273,1.095557593006039,0.17350879737309047,0.20438712356345995,9.355843282627584,2176.2161092591928,0.18255114837938483,0.13282078838540698,0.10591533324066518,Energy,7 -hierarchical_sales/W/short,FFM,486.21555258990344,486.21555258990344,9.306594112540706,0.7626060525804392,0.5381990211924879,0.4831804400467772,6.964157097654858,22.05029597510889,1.0123398738192417,0.4272702879005156,0.3596575849769412,Sales,1 -m4_quarterly/Q/short,FFM,1997384.7285338554,1997384.7285338554,637.6570988175074,1.4710379100983204,0.12477054152782754,0.1155058610541552,11.852726702112408,1413.288621808672,0.23655962790037938,0.10673256948128601,0.08570863061323504,Econ/Fin,1 -saugeen/W/short,FFM,1256.465722322464,1256.465722322464,16.833587718009948,1.3657479177262961,0.42489904463291167,0.4399150416254997,11.702472652822255,35.44666024215066,1.0699869111248752,0.5081358413598819,0.4095872044176101,Nature,1 -solar/W/short,FFM,4412870.5239507295,4412870.5239507295,1732.167406068231,1.925741705345728,0.40095363339803514,0.31448185433001413,7.532298697365111,2100.6833469018434,0.4288019717486064,0.35357865820956236,0.24630574082454176,Energy,1 -us_births/W/short,FFM,4752528.200892857,4752528.200892857,1550.942103794643,1.4098151092165498,0.020696418892060007,0.020940262890819992,10.743096251854045,2180.029403676211,0.029593685393373464,0.021053886982275057,0.017220055039796817,Healthcare,1 -car_parts/M/short,FFM,1.3620483124344471,1.3620483124344471,0.5065986595164934,0.9103871668813626,0.7734574487545944,1.8572054449995945,15.798228900968143,1.1670682552594973,2.798509957737571,1.2147716184073418,1.052012640208941,Sales,1 -hospital/M/short,FFM,2989.3365553715394,2989.3365553715394,18.676233340325123,0.7790385271832528,0.19704455572274757,0.176953577045926,5.184764416093549,54.674825608972355,0.1984823132297911,0.06779906391139473,0.05341921431843045,Healthcare,1 -saugeen/M/short,FFM,475.42613583519346,475.42613583519346,13.553694770449685,0.7940762812593081,0.3633029205458505,0.3748179276784261,5.009784342747583,21.804268752590477,0.6546295173041806,0.4069225510813588,0.3171531672880446,Nature,1 -us_births/M/short,FFM,34769641.333333336,34769641.333333336,4617.528645833333,0.5245094141501722,0.014076686464250088,0.014172534147898356,4.152179717787372,5896.578782084857,0.01831473349478932,0.014342012492045746,0.011598779272062472,Healthcare,1 -m4_weekly/W/short,FFM,351903.4098372115,351903.4098372115,294.6991564838744,2.414611834147162,0.0699703792991719,0.07113597225114457,18.979276229352518,593.2144720395917,0.10807447722116773,0.05368961610968277,0.04325778099369191,Econ/Fin,1 -m4_daily/D/short,FFM,339794.8624749177,339794.8624749177,176.0174868925099,3.3741438001904362,0.04058859636763184,0.03166203879116062,27.40230870692873,582.9192589672413,0.0900449156774763,0.02718983722218732,0.02163566596158682,Econ/Fin,1 -m4_monthly/M/short,FFM,1992971.5945184445,1992971.5945184445,595.6411925169626,1.0091579856600952,0.17149350445610967,0.14050318781160037,8.532769963210017,1411.726458815037,0.2934052223067547,0.12379468799656276,0.09916274632836239,Econ/Fin,1 -covid_deaths/D/short,FFM,1937968.876722612,1937968.876722612,229.19435108078633,46.904010885616515,0.13114590968409565,0.4166446893615998,1206.7106463083994,1392.1095060097148,0.5237879517236514,0.08623548591614802,0.07826842323579895,Healthcare,1 -electricity/D/short,FFM,2366175411.860256,2366175411.860256,5292.63182259472,1.6054804854258216,0.5776496655848486,0.10767028554226901,13.33796867416281,48643.34910201246,0.7993717751449078,0.08697551820174601,0.06822454041262206,Energy,1 -ett1/D/short,FFM,39894.15685686384,39894.15685686384,134.0275868733724,1.6736078025323005,1.310252400806972,0.46792196281372556,10.186714117926616,199.73521686689065,0.5246111921427492,0.35202791592087634,0.27358389505076636,Energy,7 -ett2/D/short,FFM,112809.3660063244,112809.3660063244,206.618015640501,1.3815428403227308,0.43983696576148745,0.146943105591668,12.215874723483559,335.87105562451256,0.19222785562260086,0.11825293491194133,0.09746656616685791,Energy,7 -hierarchical_sales/D/short,FFM,28.91550464157528,28.91550464157528,2.3041974784027897,0.7483529487185092,0.6160227059355408,1.0324880588542533,7.1696378830110135,5.37731388720942,1.6497003504368586,0.7069022689262194,0.5758409194279211,Sales,1 -jena_weather/D/short,FFM,385.42164064675427,385.42164064675427,10.325465324556543,1.3536151491401143,0.8102246873244618,0.4691723734613449,8.97650500032507,19.632158328791927,0.11821767892396508,0.062176176660555374,0.053429336148115555,Nature,21 -kdd_cup_2018/D/short,FFM,2921.8721580590504,2921.8721580590504,21.203871572070778,1.1938691555283396,0.499401752662925,0.4655215198811845,9.003399367502311,54.05434448829299,1.2108747106168427,0.4749892371620306,0.3711647520266537,Nature,1 -loop_seattle/D/short,FFM,20.569189475170724,20.569189475170724,3.2242180598901644,0.9696327838947865,0.059578336421611755,0.059269362621939714,7.105965597784465,4.5353268322327915,0.08105234874854864,0.05762108361723748,0.045740786654947935,Transport,1 -m_dense/D/short,FFM,9748.717210015191,9748.717210015191,49.10663908781829,0.7403552881165397,0.13535597312883416,0.10639983859327104,7.767380897349011,98.73559241740129,0.1709259155557126,0.0850108561709504,0.07245215492838857,Transport,1 -restaurant/D/short,FFM,157.8624317858264,157.8624317858264,7.683618248856189,0.7294316943214855,0.7358042028333956,0.41465949761975524,4.849040559841046,12.564331728580967,0.578239657413416,0.3536179145769713,0.2759202630807392,Sales,1 -saugeen/D/short,FFM,1323.2915472157797,1323.2915472157797,14.209435221354166,3.1562061215585904,0.36287815650304156,0.3738298479715983,46.073959098858914,36.37707447302188,1.178262092109868,0.46024698561257327,0.3976597760943255,Nature,1 -solar/D/short,FFM,171558.94199741483,171558.94199741483,269.10841558971543,1.0520004141694028,1.2616755711771277,0.4315916088665779,5.9268831618636435,414.19674310334074,0.5983732386929068,0.3887700154024606,0.3184198145879067,Energy,1 -temperature_rain/D/short,FFM,180.9158579841163,180.9158579841163,5.978583749759163,1.3934424077537544,38.301636306732334,1.4943918377270062,19.866697225636518,13.450496570168564,1.5834790660586977,0.7038373760429651,0.567642007080697,Nature,1 -us_births/D/short,FFM,335847.93416666664,335847.93416666664,443.1394807942708,0.6524744239529675,0.04231023281812668,0.04192886193593343,4.442829945415574,579.5238857602563,0.05432566172493097,0.04154073045498288,0.0320536207323296,Healthcare,1 -bitbrains_fast_storage/5T/short,FFM,1925980.3702073216,1925980.3702073216,184.39015867073712,0.9326270084326382,2.662725566418194,0.7633623864099987,15.351325345592338,1387.796948478891,4.357339202703256,0.5789394967681019,0.463407358503358,Web/CloudOps,2 -bitbrains_fast_storage/H/short,FFM,2365267.504241499,2365267.504241499,301.02721263096237,1.0932375246120125,4.531720553714175,0.593986588728926,20.17899867093467,1537.9426205946368,4.383636059788031,0.8580253428156845,0.6375167879850012,Web/CloudOps,2 -bitbrains_rnd/5T/short,FFM,1644012.0709018894,1644012.0709018894,130.77018991149436,1.8563746879976826,1.2329744799900078,0.7149251344516805,55.422971186533864,1282.1903411357805,5.246352753071329,0.5350738684040487,0.44642554010248897,Web/CloudOps,2 -bitbrains_rnd/H/short,FFM,1868346.0325005893,1868346.0325005893,187.8276843285316,5.97410615222248,2.756648021321376,0.6346152037646346,204.11090011373895,1366.8745489256098,5.982111853654963,0.8220258528841933,0.6828458044182137,Web/CloudOps,2 -bizitobs_l2c/5T/short,FFM,48.37379949078674,48.37379949078674,4.425946377288728,0.44041353182042,0.2033397270518167,0.27119293968592373,3.9935704181152567,6.955127568261185,0.23886722562384824,0.15200490882832055,0.12284941854218998,Web/CloudOps,7 -bizitobs_l2c/H/short,FFM,138.13516845778813,138.13516845778813,7.510476562711927,0.7503251404968043,0.7627360160402257,0.8074236032035615,4.508258048748346,11.753091868005972,0.6335218036314787,0.4048339544672052,0.31209505203205196,Web/CloudOps,7 -electricity/15T/short,FFM,998119.7611286793,998119.7611286793,176.89841116941727,3.129958320567258,0.5450151336240092,0.37836765184478316,19.95403234443766,999.0594382361238,1.9693431067801377,0.3487016420683808,0.27715969582974587,Energy,1 -electricity/H/short,FFM,8034799.087615832,8034799.087615832,449.27749829477017,2.2099201640798336,0.38483740953869605,0.24904051768652222,17.05502340411815,2834.572117201436,1.3381204757955754,0.2120910652208045,0.17122420663674814,Energy,1 -ett1/15T/short,FFM,18.127773022846807,18.127773022846807,2.6398723631387666,1.653777378196805,0.6909139367212065,0.4833555943909146,11.58652802175894,4.257672254042907,0.8048772486032805,0.49904574084803616,0.3960202957496257,Energy,7 -ett1/H/short,FFM,231.41552172416732,231.41552172416732,7.806424689860571,1.2160256044835422,0.7670988934025877,0.35362169689365797,9.373713637383625,15.21234767299799,0.7100225022661327,0.36435777772061534,0.2935843919706979,Energy,7 -ett2/15T/short,FFM,18.942287952204545,18.942287952204545,3.03341216586885,1.5530101743371663,0.2101785426979617,0.2536662155851012,13.80374205035556,4.352273882949526,0.20661792937421952,0.14400687031801324,0.11748823018646072,Energy,7 -ett2/H/short,FFM,185.27654113769532,185.27654113769532,8.772275366101946,1.0106281542260591,0.16425984144601724,0.1439041348085517,7.098187045581009,13.611632566951524,0.1726333659470811,0.11125685446003822,0.0877765442545518,Energy,7 -jena_weather/10T/short,FFM,1682.1913129023262,1682.1913129023262,15.512732299597609,0.8951834347663801,1.6317469102559847,0.7062022846768654,7.4615603297121,41.014525633028185,0.254751619597487,0.09635351419308769,0.07636469236496916,Nature,21 -jena_weather/H/short,FFM,1205.8485611425901,1205.8485611425901,10.854233035906148,0.6232310056510915,7.937455786068072,0.6285532575688864,5.11805917980683,34.72533025246254,0.2128640528222397,0.06653575408793062,0.05452517024124796,Nature,21 -kdd_cup_2018/H/short,FFM,4374.709375475365,4374.709375475365,23.31149707793361,0.9735486174317145,0.975063750697791,0.5117546737833404,7.922238751356478,66.14158582522319,1.3844631793623465,0.487951852946113,0.3900756783894771,Nature,1 -loop_seattle/5T/short,FFM,90.67872247120175,90.67872247120175,6.2121768834298114,0.9604270961401266,0.17998532640456508,0.12432782591750194,8.984265513845916,9.522537606709765,0.16329049623280067,0.10652511839558697,0.08617347979886263,Transport,1 -loop_seattle/H/short,FFM,76.41736778088392,76.41736778088392,5.56559141234788,1.1184708557691394,0.1660041760566858,0.12339684244156868,9.31555842335957,8.74170279641695,0.1547370301764054,0.09851662843937072,0.07876411594456545,Transport,1 -m4_hourly/H/short,FFM,2169159.1112414515,2169159.1112414515,309.27451736378015,1.3621265265448046,0.1368417498999052,0.1092942845992648,10.87614107270496,1472.8065423678195,0.20107062453056862,0.04222280290644927,0.03243002062523019,Econ/Fin,1 -m_dense/H/short,FFM,137506.30608747693,137506.30608747693,199.20132395850288,1.5863930115142315,0.721875311864336,0.4383265476591057,11.585364846393796,370.81842738391106,0.6577395509506928,0.3533335446504133,0.2836598322388975,Transport,1 -solar/10T/short,FFM,33.76773212994392,33.76773212994392,4.066903168325586,1.8083853981916684,5.752847406730299,1.5793279518248682,13.482172940232473,5.811000957661591,1.6904195359756449,1.1830616819284265,0.9498005857350587,Energy,1 -solar/H/short,FFM,759.1029981651507,759.1029981651507,15.460827829468677,1.1951937423231926,5.885568878113821,1.4285997437643079,8.894619757555327,27.551823862770878,1.0155694036007659,0.5698912629578828,0.4479010898221619,Energy,1 -sz_taxi/15T/short,FFM,17.524045977752852,17.524045977752852,2.8229738650650185,0.5634309962355036,1122747846603.6555,0.4090973759827378,4.1274890467335315,4.186173190128766,0.3914251091269081,0.2639601380566616,0.20849808909741915,Transport,1 -sz_taxi/H/short,FFM,7.486033306353622,7.486033306353622,1.8895230014354756,0.5763611510668891,1.0675545343731203,0.30029287536302185,4.274431671813202,2.7360616415486003,0.254881027345567,0.1760207286581334,0.13985266478592145,Transport,1 -bizitobs_application/10S/short,FFM,4290054.275219238,4290054.275219238,907.4960084512498,3.1094718662659244,0.04566837741269006,0.0453676947289043,22.076382201598303,2071.2446198407465,0.07987963052738864,0.03499849567055938,0.0271495273155525,Web/CloudOps,2 -bizitobs_service/10S/short,FFM,18621.45959861812,18621.45959861812,44.804353199820355,1.7272001860040211,0.2580500524017526,0.10380233414481005,11.466363434066375,136.46046899603607,0.10109428020831218,0.03319249794654011,0.02587570749644841,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,FFM,3249376.7462154776,3249376.7462154776,361.32921818580763,1.2161009897283157,6.205909436942313,0.8258723387207225,23.085401247059746,1802.60276994558,5.476805068392319,1.0978179588495853,0.7909914275998953,Web/CloudOps,2 -bitbrains_rnd/5T/medium,FFM,2238198.9023383656,2238198.9023383656,168.1930228162108,4.681356695216211,1.22870926405452,0.7910235379491989,163.25777915043963,1496.0611292117596,6.294691803085119,0.7076737851047087,0.6606272411448033,Web/CloudOps,2 -bizitobs_l2c/5T/medium,FFM,169.8443991978963,169.8443991978963,8.689821378227805,0.8149402153973799,0.4533944651206525,0.8029959153155891,7.876875941352694,13.032436426006317,0.6856978257589569,0.4572124068369977,0.37957433706965676,Web/CloudOps,7 -bizitobs_l2c/H/medium,FFM,122.29606686546688,122.29606686546688,7.358589099702381,0.7456420927931457,0.7041552198468384,0.8555460793631418,5.8174869889890735,11.05875521319949,0.6696244416477167,0.445573757824193,0.36463757493077753,Web/CloudOps,7 -electricity/15T/medium,FFM,507317.997731206,507317.997731206,89.92236823334565,1.4808995001872334,0.23185427811836218,0.21118746790732887,10.827981782686521,712.2625904336167,1.227300159140256,0.15494529450980563,0.12413131869575235,Energy,1 -electricity/H/medium,FFM,28029615.71998818,28029615.71998818,544.8627081346941,2.174867765901625,0.5053658946317727,0.2439922618188031,16.71953524379721,5294.300305043923,2.066539848143402,0.21267786737653374,0.17280103268723068,Energy,1 -ett1/15T/medium,FFM,19.85721762536064,19.85721762536064,2.2906231936197434,1.376162914295835,0.8336200321637157,0.4836621634165446,8.706930367953362,4.456143806629297,0.8493453670013891,0.4365950205988723,0.33765733719602115,Energy,7 -ett1/H/medium,FFM,584.0368293012891,584.0368293012891,13.878229338782175,2.1104927121975967,9945979851991.016,0.5688434101286388,19.137189233981605,24.166853938841296,1.1575350702271765,0.6647343180436466,0.5464801119917947,Energy,7 -ett2/15T/medium,FFM,20.98671334826757,20.98671334826757,3.160663490598164,1.46333640836688,0.2212280883555348,0.2443356532142276,10.405834927466259,4.581125773024309,0.24413647193215815,0.1684374696942688,0.12777534577395555,Energy,7 -ett2/H/medium,FFM,656.096297055199,656.096297055199,18.704188728332518,2.405299738261849,0.3627549968528361,0.37825115947496324,13.898897895061918,25.614376764918543,0.3451701701687809,0.2520509503502675,0.1990072281921189,Energy,7 -jena_weather/10T/medium,FFM,1926.374128825796,1926.374128825796,14.208390945376772,0.7036384464289958,1.9747424047383977,0.6784715467136659,6.999450482410453,43.89047879467478,0.2693551222083168,0.08719665368380178,0.07144563597917303,Nature,21 -jena_weather/H/medium,FFM,1800.3404606132488,1800.3404606132488,15.924051587815796,0.8785873572684332,9.325166498643796,0.6790451551240588,6.895490458023387,42.43041904828715,0.2596392529331903,0.09744209344774263,0.08188831994943238,Nature,21 -kdd_cup_2018/H/medium,FFM,4974.846994237112,4974.846994237112,26.16719571156635,1.1120831005939529,1.337844628049038,0.5712933336672651,8.92025803073768,70.53259526089418,1.4763749589217015,0.5477267971050995,0.43590724995081154,Nature,1 -loop_seattle/5T/medium,FFM,115.42348564356597,115.42348564356597,6.426456309024272,1.003943562886726,0.25120724739581807,0.14261178360204332,7.679238503852029,10.743532270327389,0.19114951532209784,0.11433986307291034,0.0903485393507555,Transport,1 -loop_seattle/H/medium,FFM,83.21965811407603,83.21965811407603,5.719363556656183,1.1675833832381963,0.1757784340588301,0.1257715514908142,9.489892579410606,9.122480918811288,0.16123883143557477,0.10108911212177372,0.0807171356372815,Transport,1 -m_dense/H/medium,FFM,240915.42568549264,240915.42568549264,275.0003300476074,2.095950817120103,1.3317939923637112,0.5626516660054525,16.543627957556797,490.8313617582852,0.8539845898355463,0.4784658486757375,0.3910936733211742,Transport,1 -solar/10T/medium,FFM,37.41426938412807,37.41426938412807,3.88470965682656,1.7033299800969257,3.0663892572628018,1.5418579955460672,9.94247905875921,6.116720476213382,1.3573245818324349,0.8620325108851009,0.6579120639654515,Energy,1 -solar/H/medium,FFM,983.5601470427501,983.5601470427501,13.690905777497303,1.0455738702578845,7.071313426499879,1.4143191943203446,12.413647471955263,31.361762498985133,1.1297238834720922,0.49317838063805464,0.42557155311202866,Energy,1 -sz_taxi/15T/medium,FFM,17.817374883961474,17.817374883961474,2.9159937226874195,0.5682958840161885,12711415608987.596,0.41826856875012064,4.470808963040199,4.221063240933672,0.39284038111829417,0.2713818817615482,0.21855963250478558,Transport,1 -bizitobs_application/10S/medium,FFM,17323808.556267902,17323808.556267902,1840.92953125,5.4143932837938324,0.07837244828542074,0.07596422831217448,34.86971033605848,4162.1879530203705,0.16343687478105956,0.07228788624533995,0.060193281483599925,Web/CloudOps,2 -bizitobs_service/10S/medium,FFM,97700.40386644426,97700.40386644426,98.65642349336828,2.9584254775977197,0.291520272722793,0.148381351935012,17.93502578177396,312.57063820270173,0.23524801386105512,0.074251144684931,0.06136236817418266,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,FFM,3901222.4296035883,3901222.4296035883,400.8393293930655,1.1014988906899956,5.350570186340807,0.834311535384054,17.599881681366256,1975.1512422099702,5.219735002019173,1.0592986669101137,0.7691787082924886,Web/CloudOps,2 -bitbrains_rnd/5T/long,FFM,2266322.6068212753,2266322.6068212753,186.5880843397865,3.575875218295355,2.1594129610135386,0.765453309302522,118.73072905929484,1505.4310368865374,5.7669671727542635,0.7147769182705863,0.6258167438851238,Web/CloudOps,2 -bizitobs_l2c/5T/long,FFM,188.33929817018054,188.33929817018054,9.587026627555726,0.9017646722802146,0.620701139708539,0.935032177576943,6.960276639937921,13.723676554414292,0.7705639241120555,0.5382972142636155,0.43215523166732117,Web/CloudOps,7 -bizitobs_l2c/H/long,FFM,136.3912622845362,136.3912622845362,8.372255936879961,0.85197018076168,1.121070377788256,0.8392989567347935,7.07992271745898,11.678666973783274,0.713366715018113,0.5114015776278429,0.4074497610384762,Web/CloudOps,7 -electricity/15T/long,FFM,915689.1485853452,915689.1485853452,105.1431972690502,1.5786287304181277,0.24462121896393357,0.2184506975353301,11.585968387422673,956.916479419884,1.5096530760229208,0.16587628554200876,0.1329953217762604,Energy,1 -electricity/H/long,FFM,31657501.822802987,31657501.822802987,617.9180436998888,2.556276400671459,0.5823649685853195,0.27482424482139384,21.720536642765055,5626.499962037055,2.219089981594679,0.24370670034174544,0.2004605727449194,Energy,1 -ett1/15T/long,FFM,21.24160052080003,21.24160052080003,2.4722622759380037,1.4760885108467687,0.8720636984570064,0.502471110934303,8.955835428889452,4.608861087166766,0.878453434163714,0.4712156095349367,0.36088051261909876,Energy,7 -ett1/H/long,FFM,616.9636765212609,616.9636765212609,14.421514965239025,2.191309403335181,9402057694702.428,0.5828434091396433,20.621307071931156,24.838753521891167,1.1894740852447687,0.6906151029681526,0.5715399902644217,Energy,7 -ett2/15T/long,FFM,21.895749345204187,21.895749345204187,3.305423843141586,1.5551762466180148,0.23366433525342997,0.26254744257245743,11.09707810921701,4.6792894060107235,0.2493677858371655,0.17615200802899486,0.13294420532924328,Energy,7 -ett2/H/long,FFM,701.3028972645916,701.3028972645916,19.229853715846147,2.5414949061301226,0.3967475939552263,0.4293437352256169,15.353277795722923,26.482124107869286,0.3626523115187354,0.2633380491614895,0.2075045900793346,Energy,7 -jena_weather/10T/long,FFM,1876.2482393973567,1876.2482393973567,14.334149543684784,0.7063948746329407,2.0626856763703185,0.6590750896366974,7.2283348130965965,43.3156812182073,0.2650560712726606,0.08771311581005149,0.0720972797065268,Nature,21 -jena_weather/H/long,FFM,3026.2305518025964,3026.2305518025964,19.04631158363015,0.9611873917466696,22.621762230893857,0.6644196969964516,7.670204311613768,55.01118569711615,0.3312283037599923,0.11467990371021626,0.095337868214275,Nature,21 -kdd_cup_2018/H/long,FFM,3742.0834871491106,3742.0834871491106,24.256956087959118,1.066468898955776,1.5130114404158097,0.6069607647040769,7.854992519693923,61.17257136290014,1.4353557165467177,0.569166209808712,0.4486484394446559,Nature,1 -loop_seattle/5T/long,FFM,117.832344120998,117.832344120998,6.421727048324727,1.0033839564354472,0.2503571352252254,0.14186460191046285,7.23297520929841,10.855060760815574,0.1919197059887196,0.11353745448419687,0.08899748487848717,Transport,1 -loop_seattle/H/long,FFM,99.79885624274381,99.79885624274381,6.262540669239465,1.28716648341709,0.20495595280395,0.13722915138458525,12.285066303290883,9.989937749693128,0.1785335326559143,0.11191996758087912,0.09078503709758484,Transport,1 -m_dense/H/long,FFM,256392.56262683257,256392.56262683257,278.1699989902826,2.096545477512151,1.1949037043848045,0.5479910405476888,18.54087065645608,506.35221202916904,0.876570577136847,0.48155341433172555,0.3969049836987396,Transport,1 -solar/10T/long,FFM,40.063244973946084,40.063244973946084,3.9042207894244045,1.7075141789782105,2.7071622825212986,1.5583811837294579,9.086662130658409,6.32955329971603,1.3680316996962663,0.843834872641864,0.615702515669713,Energy,1 -solar/H/long,FFM,2388.0745081546534,2388.0745081546534,28.44062954669838,2.173562911775244,13.270997722579681,1.4584968752524567,18.5748895475193,48.86792923947825,1.6943403857023536,0.9860885858212636,0.8116077346270839,Energy,1 -sz_taxi/15T/long,FFM,17.267480421202134,17.267480421202134,2.901999698269401,0.5483788689963521,8619488570936.882,0.41724632088615005,4.62243488717857,4.155415794021356,0.3846178604509939,0.2686039063969709,0.22046708944985985,Transport,1 -bizitobs_application/10S/long,FFM,9446313.25280816,9446313.25280816,1681.6269859483507,5.424601473332471,0.07397242652045356,0.07470968882242839,46.1878621550681,3073.4855218152825,0.11853206078915546,0.06485363627331699,0.05296788880134404,Web/CloudOps,2 -bizitobs_service/10S/long,FFM,104006.5942731868,104006.5942731868,91.7940036029412,2.5317170685556736,0.27489340914304927,0.14103439309924998,16.89595051367137,322.50053375643705,0.2389187112330676,0.0680039340843433,0.05604926924540719,Web/CloudOps,2 diff --git a/results/FFM/config.json b/results/FFM/config.json deleted file mode 100644 index d1f36d55a0c487bce44024c8309fa9329505bcd8..0000000000000000000000000000000000000000 --- a/results/FFM/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "FFM", - "model_type": "deep-learning", - "model_dtype": "mixed-precision (float16/float32)", - "model_link": "TBD", - "code_link": "TBD", - "replication_code_available": "No", - "org": "IBM Research", - "testdata_leakage": "No" -} diff --git a/results/FlowState-9.1M/all_results.csv b/results/FlowState-9.1M/all_results.csv deleted file mode 100644 index e61f6338937636445c08816d29a78fa09caa20ce..0000000000000000000000000000000000000000 --- a/results/FlowState-9.1M/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[mean],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -loop_seattle/5T/short,FlowState-9.1M,41.20642213772202,41.20642213772202,3.6482890787508464,3.6482890787508464,0.5760316745958542,0.09734227274101462,0.07782384936531508,6.593895242486975,6.4192228608860455,0.11007549989315832,0.06256010311547563,0.049961607574821335,Transport,1 -loop_seattle/5T/medium,FlowState-9.1M,65.28178691411289,65.28178691411289,4.869518228536789,4.869518228536789,0.7734527843739306,0.15530373116641838,0.11332293038028682,8.269508257047189,8.079714531721581,0.1437547241555911,0.08663873543621015,0.06910445419530627,Transport,1 -loop_seattle/5T/long,FlowState-9.1M,77.11275533061361,77.11275533061361,5.218633842700092,5.218633842700092,0.8250182708680949,0.16325547842842192,0.12005953753436054,9.319874052199266,8.781386868292138,0.1552567255117628,0.09226651942508741,0.07396257116366077,Transport,1 -loop_seattle/D/short,FlowState-9.1M,18.37072676116345,18.37072676116345,2.8908717861116484,2.8908717861116484,0.8751111080929995,0.05345322222163434,0.05338208177141361,12.284504977392556,4.286108580188263,0.07659848596512975,0.05166374061520772,0.041998676836883936,Transport,1 -loop_seattle/H/short,FlowState-9.1M,60.254646585128924,60.254646585128924,4.224341119794517,4.224341119794517,0.8578659174207104,0.1156957368292313,0.09733176797613834,10.324014368171769,7.762386655219445,0.13740213765004466,0.07477513370613476,0.06071775786388836,Transport,1 -loop_seattle/H/medium,FlowState-9.1M,72.49868649068385,72.49868649068385,4.774017567039151,4.774017567039151,0.9802017879961532,0.1182101050643606,0.10903377222941017,11.2153856481414,8.514616050690945,0.15049488816363543,0.08438022754819177,0.06815404056535827,Transport,1 -loop_seattle/H/long,FlowState-9.1M,59.69476156690747,59.69476156690747,4.394733995057727,4.394733995057727,0.9201987421572889,0.13256338970497952,0.10205463124666585,10.089497539677955,7.726238513462257,0.13807820232396775,0.07853976662411824,0.06319115108866927,Transport,1 -m_dense/D/short,FlowState-9.1M,9472.558935185185,9472.558935185185,48.308813657407406,48.308813657407406,0.7324154243118424,0.13463613879159259,0.10566210358231155,12.396491269683509,97.32707195423679,0.1684875581125931,0.08362970225057244,0.06984659577170221,Transport,1 -m_dense/H/short,FlowState-9.1M,45185.8003233507,45185.8003233507,89.82139122856988,89.82139122856988,0.794888521288021,0.2976044740697495,0.21334691312577989,12.069100793905335,212.56951880114585,0.3770453948873275,0.15932078181343987,0.1313047568761031,Transport,1 -m_dense/H/medium,FlowState-9.1M,34831.29833333333,34831.29833333333,79.91017849392361,79.91017849392361,0.6918537392919458,0.26913952516783457,0.1942931811014811,8.942685213575544,186.6314505471501,0.3247151566079737,0.13903361972568543,0.11462594957546576,Transport,1 -m_dense/H/long,FlowState-9.1M,35997.86469135802,35997.86469135802,84.8109442515432,84.8109442515432,0.7444252434529841,0.2882623402318473,0.20252094786844135,9.243858424452394,189.73103249431293,0.32845249091839185,0.14682029360376359,0.12117489021344204,Transport,1 -sz_taxi/15T/short,FlowState-9.1M,16.730177058405054,16.730177058405054,2.7577188600113978,2.7577188600113978,0.5514772960536524,1195954521887.0005,0.4006040205448975,5.440699762571138,4.090253911238892,0.3824562487334538,0.2578585176248867,0.20255364364698247,Transport,1 -sz_taxi/15T/medium,FlowState-9.1M,16.707470494457798,16.707470494457798,2.821408237555088,2.821408237555088,0.5509006403082922,9710347132076.936,0.4138293991740952,5.205794460068225,4.087477277546359,0.38040798312325286,0.2625791275003582,0.20777358525869283,Transport,1 -sz_taxi/15T/long,FlowState-9.1M,16.2041913589521,16.2041913589521,2.80213628263555,2.80213628263555,0.5290821987226246,6334988927381.315,0.4129524796097367,4.415962703632027,4.025443001577852,0.3725878168442675,0.2593607311389226,0.20691493769848118,Transport,1 -sz_taxi/H/short,FlowState-9.1M,7.417397474631285,7.417397474631285,1.8844109478159847,1.8844109478159847,0.5756606690704787,1.3483923811178942,0.30121186655810755,5.345870852971551,2.7234899439196183,0.2537098957828191,0.17554450907732358,0.13832774822300725,Transport,1 -bitbrains_fast_storage/5T/short,FlowState-9.1M,1924708.5933602322,1924708.5933602322,199.03606752846804,199.03606752846804,1.0032515815823284,3.0243938323240966,0.7950182794198272,19.25499771969558,1387.3386729130823,4.355900312997404,0.6249240259586876,0.5113067369825212,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,FlowState-9.1M,3390856.937972916,3390856.937972916,346.56688549225504,346.56688549225504,1.260891455352925,5.679820563609929,0.8348958701489698,25.40891853796344,1841.4279616571798,5.5947667486039006,1.052965919652013,0.7795465054578488,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,FlowState-9.1M,4983042.650290739,4983042.650290739,439.2763425860992,439.2763425860992,1.0374589320494734,5.767946014842698,0.8445433276088283,20.188168977655355,2232.272978443886,5.899230991338783,1.160876218531447,0.8255881443730684,Web/CloudOps,2 -bitbrains_fast_storage/H/short,FlowState-9.1M,2863379.353066434,2863379.353066434,333.7310782674274,333.7310782674274,1.104408431680801,4.252633291163891,0.5967120208908891,21.060018999150223,1692.1522842423,4.823183638812036,0.9512419723992579,0.6996132789973957,Web/CloudOps,2 -bitbrains_rnd/5T/short,FlowState-9.1M,1478593.945297488,1478593.945297488,132.24529301833925,132.24529301833925,1.9620477620454029,1.4658456323459939,0.7350835676392866,58.827425560648535,1215.9744838184263,4.975416577469799,0.541109564330338,0.4632044380693893,Web/CloudOps,2 -bitbrains_rnd/5T/medium,FlowState-9.1M,2208052.9063197314,2208052.9063197314,190.83315100806956,190.83315100806956,4.581663134278986,2.306887118167001,0.8003794722451701,166.06818056785903,1485.9518519520514,6.252156934965629,0.802932347322748,0.7238091593893922,Web/CloudOps,2 -bitbrains_rnd/5T/long,FlowState-9.1M,2460772.825721577,2460772.825721577,222.0122217941569,222.0122217941569,3.5251046217605357,3.5869625604067092,0.7694731327490271,121.73594787820016,1568.685062630985,6.009279106104837,0.8504788102526731,0.7270607240923831,Web/CloudOps,2 -bitbrains_rnd/H/short,FlowState-9.1M,1783555.8804027704,1783555.8804027704,169.21358647254425,169.21358647254425,5.926597311741625,2.107055746328823,0.6077552342230917,218.0120577597906,1335.4983640584403,5.844794363995951,0.740561458660828,0.664595981096118,Web/CloudOps,2 -bizitobs_application/10S/short,FlowState-9.1M,664005.4388888889,664005.4388888889,449.96708984375,449.96708984375,1.3377137863983317,0.019362943437364367,0.019491665098402236,16.336749352184412,814.8652887986387,0.03142609893095301,0.017353433107882715,0.013558368098991429,Web/CloudOps,2 -bizitobs_application/10S/medium,FlowState-9.1M,4575707.306666667,4575707.306666667,1198.610625,1198.610625,2.5893138141163443,0.03760066986083985,0.038373565673828124,58.83299716249604,2139.090298857593,0.08399578031721472,0.04706591152189688,0.042047922832965524,Web/CloudOps,2 -bizitobs_application/10S/long,FlowState-9.1M,6559583.573333333,6559583.573333333,1548.4186111111112,1548.4186111111112,3.3411338993333333,0.048364529079861114,0.049444936116536456,75.69303481763987,2561.1684000341197,0.09877402790664712,0.05971631662448403,0.052677139441578126,Web/CloudOps,2 -bizitobs_l2c/5T/short,FlowState-9.1M,22.821621295384,22.821621295384,3.0685904048738024,3.0685904048738024,0.31231757023136525,0.16200984930040224,0.2157041277204241,2.393935136307405,4.777198059049258,0.1640683122542177,0.105387811538073,0.08458085603866948,Web/CloudOps,7 -bizitobs_l2c/5T/medium,FlowState-9.1M,69.30413079294217,69.30413079294217,5.238999430338541,5.238999430338541,0.5193918306430902,0.38426043329664383,0.6304700306483677,4.155248919369638,8.324910257350657,0.43801271795380065,0.27564842249378013,0.2138909951695262,Web/CloudOps,7 -bizitobs_l2c/5T/long,FlowState-9.1M,71.76699466765874,71.76699466765874,5.177323443700397,5.177323443700397,0.5177408840898012,0.423938883860758,0.7123863777281746,4.617430555988597,8.471540277166763,0.4756643215874484,0.2906989712513249,0.23033267410204214,Web/CloudOps,7 -bizitobs_l2c/H/short,FlowState-9.1M,55.461908249627974,55.461908249627974,4.527385651119172,4.527385651119172,0.4498973653968915,0.3789699883769742,0.6063558033534459,3.8045774239835706,7.447275223169074,0.401427228293401,0.24403769417436816,0.1886214231576794,Web/CloudOps,7 -bizitobs_l2c/H/medium,FlowState-9.1M,66.8992699032738,66.8992699032738,4.662852550688244,4.662852550688244,0.4704569255569069,0.4438962515829857,0.749674333844866,4.24338559208035,8.179197387474751,0.49526282529900423,0.2823428044104843,0.22108179915235826,Web/CloudOps,7 -bizitobs_l2c/H/long,FlowState-9.1M,75.9602306547619,75.9602306547619,5.0850547185019845,5.0850547185019845,0.5469474777475324,0.5924043065238235,0.7568616594587053,7.1077522882651625,8.715516660230872,0.5323689121370271,0.3106098185778061,0.24756994557034062,Web/CloudOps,7 -bizitobs_service/10S/short,FlowState-9.1M,3520.50839720832,3520.50839720832,21.761772656314587,21.761772656314587,0.8059993822380633,0.10306755156744093,0.06974780545663581,9.507322336556879,59.33387225867127,0.043956430224570464,0.01612181718327782,0.0127153968223787,Web/CloudOps,2 -bizitobs_service/10S/medium,FlowState-9.1M,31802.868482142858,31802.868482142858,46.433770926339285,46.433770926339285,1.1440456397927963,0.10534807568504696,0.08318410812862335,15.995324064213758,178.33358764445597,0.13421805424840713,0.03494714858523884,0.026749382265129105,Web/CloudOps,2 -bizitobs_service/10S/long,FlowState-9.1M,135023.47957671958,135023.47957671958,79.42200438161376,79.42200438161376,1.3822690402599558,0.11028612086381862,0.09789428549468833,25.236081780178996,367.45541168517246,0.2722227322362745,0.05883836336303083,0.05352239744204552,Web/CloudOps,2 -car_parts/M/short,FlowState-9.1M,1.429560607254418,1.429560607254418,0.4836226899547534,0.4836226899547534,0.8944060493088195,0.8137632961798099,1.8800651255841114,21.08757561148329,1.1956423408588448,2.8670276838625437,1.1596776002833478,1.0076518334176803,Sales,1 -covid_deaths/D/short,FlowState-9.1M,520034.35426554666,520034.35426554666,136.357007570195,136.357007570195,34.60603694671917,0.09393392626416988,0.36803830177861646,918.8211149840132,721.1340750966817,0.2713301924039766,0.05130498526878134,0.042993758538717175,Healthcare,1 -electricity/15T/short,FlowState-9.1M,144065.98286205519,144065.98286205519,56.856076371304624,56.856076371304624,1.0817436193639087,0.1915398677018586,0.1772058353606645,16.26188852890285,379.56024931762175,0.7481880832374911,0.11207453593237973,0.09269706732095719,Energy,1 -electricity/15T/medium,FlowState-9.1M,253340.60338294544,253340.60338294544,54.7942716272715,54.7942716272715,0.8556548344188857,0.13508191714894052,0.13523208481556662,9.532384228961938,503.32951769486505,0.8672874346301549,0.09441604674743875,0.07564332966991412,Energy,1 -electricity/15T/long,FlowState-9.1M,316119.8529764912,316119.8529764912,59.7140382530052,59.7140382530052,0.9053667166654249,0.14293423834777463,0.13760582326768755,9.726931671510378,562.2453672343519,0.8870110003705841,0.09420621652697063,0.0749509701869353,Energy,1 -electricity/D/short,FlowState-9.1M,1180566865.8407207,1180566865.8407207,4251.209920748874,4251.209920748874,1.4344403736384097,0.49429796389693836,0.10163411985861288,15.199692356424537,34359.378135244544,0.5646387062964002,0.06986149925058828,0.05504415510373746,Energy,1 -electricity/H/short,FlowState-9.1M,1579386.9683294483,1579386.9683294483,172.5957497328681,172.5957497328681,0.9351379841940162,0.21026389299976914,0.12611722789637678,12.947069300985513,1256.7366344343784,0.5932694400880952,0.08147751963297314,0.06677232663588524,Energy,1 -electricity/H/medium,FlowState-9.1M,6221001.874001003,6221001.874001003,247.56637994989617,247.56637994989617,1.092976787129436,0.23200557334682143,0.13328747567829785,11.958698401046853,2494.193632018373,0.973565963834623,0.09663331596028776,0.07678815543188483,Energy,1 -electricity/H/long,FlowState-9.1M,8939375.961766891,8939375.961766891,279.8932897311374,279.8932897311374,1.2110867967690446,0.3095987619112468,0.14577881565036716,13.842587383560438,2989.8789209208608,1.1792074146755631,0.11038983560820556,0.08752998078933216,Energy,1 -electricity/W/short,FlowState-9.1M,29395794717.160362,29395794717.160362,23643.860884712838,23643.860884712838,1.3635557497389068,0.22346164837183186,0.09038805711913753,13.709397168603852,171452.01870249404,0.39119693104666264,0.05394748855213885,0.04355379314195652,Energy,1 -ett1/15T/short,FlowState-9.1M,4.980384614354088,4.980384614354088,1.0932999247596378,1.0932999247596378,0.6888563402341751,0.49305846536351217,0.23987056868416923,7.201910430402015,2.2316775336849384,0.42187994489597685,0.20667919314076327,0.1617466807586172,Energy,7 -ett1/15T/medium,FlowState-9.1M,8.6315771484375,8.6315771484375,1.544872349330357,1.544872349330357,0.9899225182222047,0.7127148305476998,0.38790635971795945,8.719024268920881,2.9379545858364624,0.5599770158522798,0.29445417986419087,0.22932511415283205,Energy,7 -ett1/15T/long,FlowState-9.1M,8.090037667410714,8.090037667410714,1.551713111514137,1.551713111514137,1.0126223652403492,0.7426191157153659,0.39777882893880206,8.355813417792739,2.8442991522360503,0.54212621467887,0.2957580445613115,0.2298512894395186,Energy,7 -ett1/D/short,FlowState-9.1M,43043.99444444444,43043.99444444444,136.9359343998016,136.9359343998016,1.6288935332388368,1.2419960627480158,0.4954109918503534,14.079257234165125,207.470466439068,0.5449280619678922,0.3596667739124283,0.2799498278363056,Energy,7 -ett1/H/short,FlowState-9.1M,98.04556412469773,98.04556412469773,4.894838360377721,4.894838360377721,0.8150353380790544,0.4436365876879011,0.2579571133568173,6.952233363517736,9.90179600500322,0.4621573388829306,0.22846213654080613,0.1762152118908626,Energy,7 -ett1/H/medium,FlowState-9.1M,134.41744326636905,134.41744326636905,6.668464587983631,6.668464587983631,1.2267932617039405,3365661524510.5957,0.43888593401227677,9.119016541001752,11.593853684878425,0.555318170217747,0.31940368179657114,0.24767421231989253,Energy,7 -ett1/H/long,FlowState-9.1M,155.68571635251323,155.68571635251323,7.22068271536045,7.22068271536045,1.33916082947429,2267774406926.194,0.47107417192408646,9.95334609492623,12.47740823859319,0.5975160415714699,0.3457828477696871,0.2699857075194963,Energy,7 -ett1/W/short,FlowState-9.1M,1173048.4285714286,1173048.4285714286,801.5694056919643,801.5694056919643,1.4509044072930064,0.5297911167144775,0.5278735501425607,8.922727965738348,1083.0736025642157,0.43098083698912737,0.31896359818216297,0.24500652359110595,Energy,7 -ett2/15T/short,FlowState-9.1M,7.959989992777507,7.959989992777507,1.7561996550787062,1.7561996550787062,0.7547916193814699,0.11044701391019925,0.1292610100337437,8.60130403004982,2.8213454224496344,0.13393930909778234,0.08337304839290896,0.06581268360764704,Energy,7 -ett2/15T/medium,FlowState-9.1M,10.991811126224578,10.991811126224578,2.050341496543279,2.050341496543279,0.8470324155409298,0.12984282517942974,0.15523655846005394,8.238194223435517,3.3153900413412263,0.17668312929658506,0.10926640522493662,0.08720802717137183,Energy,7 -ett2/15T/long,FlowState-9.1M,11.056184779575894,11.056184779575894,2.093517097594246,2.093517097594246,0.8656004641238805,0.13304481590146605,0.15894615657745845,8.728302116810942,3.325084176314322,0.17719974581181555,0.11156731014178155,0.09031154642244392,Energy,7 -ett2/D/short,FlowState-9.1M,119717.15555555555,119717.15555555555,222.04579613095237,222.04579613095237,1.4635575132775187,0.3624706752716549,0.15674171447753907,10.830255578141793,346.00166987394084,0.19802587337139363,0.1270826603909114,0.10325530356962487,Energy,7 -ett2/H/short,FlowState-9.1M,104.57532457624163,104.57532457624163,6.295174280802409,6.295174280802409,0.7215709730957005,0.11713471286857265,0.10737862814040411,6.602956702509036,10.226207731913215,0.12969675934784944,0.0798403205913801,0.0631822907791898,Energy,7 -ett2/H/medium,FlowState-9.1M,242.78725353422618,242.78725353422618,9.703669084821428,9.703669084821428,1.0379484333384368,0.17125420465428529,0.16814332689557757,9.244285748553247,15.581631927825345,0.2099724972418964,0.13076317291967424,0.10431498050240505,Energy,7 -ett2/H/long,FlowState-9.1M,265.77815806878306,265.77815806878306,10.071580997850528,10.071580997850528,1.0965159499861146,0.1877654904885175,0.18084840320405507,9.853142275736998,16.302704010954226,0.2232529902611188,0.13792255400799516,0.11143469439326707,Energy,7 -ett2/W/short,FlowState-9.1M,4955971.428571428,4955971.428571428,1521.9556361607142,1521.9556361607142,0.9778324368044548,0.14722657203674316,0.1582614013126918,6.102820239909918,2226.2011204227324,0.18674414426016983,0.12766874487189425,0.10406980997248803,Energy,7 -hierarchical_sales/D/short,FlowState-9.1M,29.732930018538134,29.732930018538134,2.319569186919876,2.319569186919876,0.7495294259800727,0.6318419237892607,1.0690176652748262,10.409452522840244,5.452791030154937,1.6728559020336073,0.7116181388678587,0.5828056079568731,Sales,1 -hierarchical_sales/W/short,FlowState-9.1M,450.37896728515625,450.37896728515625,8.750412407567946,8.750412407567946,0.7124265352171969,0.5166803930063173,0.452388971538867,8.61290730769748,21.222133900368178,0.9743185478836948,0.40173571377650935,0.3398582822825211,Sales,1 -hospital/M/short,FlowState-9.1M,2029.1258259507586,2029.1258259507586,16.616914870996986,16.616914870996986,0.7584542558926894,0.19929957068624418,0.17468420747361563,6.50496006333193,45.04581918392381,0.16352678391513475,0.0603232596648055,0.04789252589407568,Healthcare,1 -jena_weather/10T/short,FlowState-9.1M,769.5275096238605,769.5275096238605,5.621278191369678,5.621278191369678,0.2767616179581378,0.39370277928161435,0.5409789359463113,2.943629425991047,27.740358858959638,0.17230240076182607,0.03491518378141902,0.028705220857371594,Nature,21 -jena_weather/10T/medium,FlowState-9.1M,1424.3707860465001,1424.3707860465001,9.894016493760146,9.894016493760146,0.6254029435471754,0.9141056125584986,0.6582028373839364,7.134168263751859,37.740837113748555,0.2316148725005984,0.060719410166084245,0.049470015067881346,Nature,21 -jena_weather/10T/long,FlowState-9.1M,1399.3250419980004,1399.3250419980004,10.083748800287802,10.083748800287802,0.6608100770639971,0.9544456003029005,0.6474178345115097,7.37106124596258,37.40755327467971,0.22890322069317012,0.06170418471632936,0.05070029776307641,Nature,21 -jena_weather/D/short,FlowState-9.1M,398.70033172123016,398.70033172123016,9.827138264973959,9.827138264973959,1.0485371388820024,0.6242780152916303,0.44123424348377044,10.138610984284272,19.967481857290622,0.12023687611673844,0.05917543406788213,0.047547547386128125,Nature,21 -jena_weather/H/short,FlowState-9.1M,1098.5380878799044,1098.5380878799044,8.411170702131967,8.411170702131967,0.526016955075292,1.3175091402849612,0.6050199826334354,6.146329621532627,33.144201421665066,0.20317183962392238,0.051559939646816814,0.0419072690267179,Nature,21 -jena_weather/H/medium,FlowState-9.1M,1349.0420030381945,1349.0420030381945,10.663989451574901,10.663989451574901,0.8571477516894351,2.6584602221914055,0.6862809438554067,7.631882833911909,36.729307140731564,0.22475314018670384,0.0652548415077891,0.052881529582308406,Nature,21 -jena_weather/H/long,FlowState-9.1M,1344.8131965112434,1344.8131965112434,12.889586691881613,12.889586691881613,1.0851070705986812,2.5530645058483583,0.6719191616805142,13.687396490420356,36.671694759190544,0.22080422921459042,0.0776095916233129,0.06666390952483377,Nature,21 -kdd_cup_2018/D/short,FlowState-9.1M,3031.419290378869,3031.419290378869,21.71406054869319,21.71406054869319,1.223272467425053,0.5122704279217459,0.48005108592659085,12.181636446841818,55.05832625842225,1.233364972366011,0.48641801356059444,0.3812015815435914,Nature,1 -kdd_cup_2018/H/short,FlowState-9.1M,4400.043310172846,4400.043310172846,22.743728537194453,22.743728537194453,0.9542261628597182,0.8840861593739912,0.5019650321757874,10.160826590381683,66.33282226901585,1.3884661046462603,0.47606742916953976,0.3817186229134154,Nature,1 -kdd_cup_2018/H/medium,FlowState-9.1M,5346.50910268196,5346.50910268196,26.33725056482099,26.33725056482099,1.0846836992020321,0.9676168371097675,0.5905999808700999,11.210373175534803,73.11982701485255,1.5305304083425761,0.5512863542935927,0.4368045266133778,Nature,1 -kdd_cup_2018/H/long,FlowState-9.1M,4320.785902502329,4320.785902502329,24.9758049761873,24.9758049761873,1.0534284105933664,0.9931413860142767,0.6358806894330652,9.937313926015616,65.73268519163301,1.542354420888718,0.5860333121638589,0.45748987044545686,Nature,1 -m4_daily/D/short,FlowState-9.1M,460449.38954061264,460449.38954061264,194.8177779062743,194.8177779062743,3.5933811129151243,0.04256217416009881,0.03372789031974429,36.4567024138539,678.5642118035792,0.10481941775567499,0.030093962654342218,0.024315459514113697,Econ/Fin,1 -m4_hourly/H/short,FlowState-9.1M,524888.7468503716,524888.7468503716,188.91631142279184,188.91631142279184,0.7308697355223807,0.09173926594704629,0.0867135555823836,8.266989981253527,724.4920612749125,0.09890916765238221,0.02579124895572352,0.020490718600366213,Econ/Fin,1 -m4_monthly/M/short,FlowState-9.1M,1863447.246301143,1863447.246301143,555.0341335720486,555.0341335720486,0.9217787164197633,0.1597439799979329,0.1291678792089224,10.553067069674826,1365.0814064740398,0.28371077918144827,0.11535514714449419,0.09256760422999852,Econ/Fin,1 -m4_quarterly/Q/short,FlowState-9.1M,1835070.2141721903,1835070.2141721903,561.5495734303793,561.5495734303793,1.1533472901853319,0.11924825147073716,0.10135561777527134,12.789597196530536,1354.6476346903612,0.22674415917787266,0.09399350990139102,0.07549476248623303,Econ/Fin,1 -m4_weekly/W/short,FlowState-9.1M,230569.25755303193,230569.25755303193,249.35544133845352,249.35544133845352,2.022636456714285,0.06192654865519097,0.06137127862999366,16.35037889030763,480.17627758254775,0.08748067109178026,0.04542869435053447,0.03663408849488293,Econ/Fin,1 -m4_yearly/A/short,FlowState-9.1M,3295300.5925140013,3295300.5925140013,818.0042513808635,818.0042513808635,2.957246114331518,0.16630538438952275,0.135139478891435,42.9209268420369,1815.2962822949871,0.29109117207079993,0.13117077284613704,0.1069378729835909,Econ/Fin,1 -restaurant/D/short,FlowState-9.1M,138.683303161775,138.683303161775,7.088883148874568,7.088883148874568,0.681671230570501,0.6547431785474475,0.3927890905492656,6.158892375231965,11.776387525968012,0.5419766395607936,0.32624682728843574,0.2559205236124014,Sales,1 -saugeen/D/short,FlowState-9.1M,1065.7159375,1065.7159375,12.316854858398438,12.316854858398438,2.7363346763757384,0.2869456609090169,0.3157666778564453,39.327833877693664,32.64530498402489,1.0573892206158066,0.3989458688020571,0.32886548733101045,Nature,1 -saugeen/M/short,FlowState-9.1M,419.1245814732143,419.1245814732143,12.768525623139881,12.768525623139881,0.7480760568691176,0.35278070540655226,0.3556371870495024,7.5489379732842705,20.472532365909554,0.6146468386464715,0.3833494444247625,0.29564347249203715,Nature,1 -saugeen/W/short,FlowState-9.1M,907.9439239501953,907.9439239501953,14.38510627746582,14.38510627746582,1.1671291864275761,0.39806915521621705,0.39103698134422304,13.698295871517155,30.13210785773533,0.9095627312966047,0.4342263945655186,0.35654937897593736,Nature,1 -solar/10T/short,FlowState-9.1M,23.815074714604954,23.815074714604954,2.3434108497452564,2.3434108497452564,1.0258561153573258,5.817045293185954,1.5298146551832956,10.298084292870156,4.880069130105121,1.4196115690597917,0.6816979564563959,0.5199798844919239,Energy,1 -solar/10T/medium,FlowState-9.1M,20.57507829672466,20.57507829672466,2.0579187168323,2.0579187168323,0.8992403786637814,3.935542893892818,1.4561078073914613,9.428119094294988,4.535976002661903,1.0065510994343836,0.45666034074222656,0.35640086194325327,Energy,1 -solar/10T/long,FlowState-9.1M,18.803191816913145,18.803191816913145,1.947746906760062,1.947746906760062,0.8511553963682477,3.6116326980938003,1.4495610039586075,8.98398203054954,4.336264730953721,0.9372142615089935,0.4209743389961493,0.33731284559674707,Energy,1 -solar/D/short,FlowState-9.1M,123798.74148418491,123798.74148418491,254.33213104470804,254.33213104470804,0.9914601541317253,1.065309401092158,0.43119359422484166,5.072566028494322,351.8504532954092,0.5083040848435385,0.36742331836204006,0.277712421695133,Energy,1 -solar/H/short,FlowState-9.1M,757.1993341141831,757.1993341141831,11.305498109246692,11.305498109246692,0.8724446953179906,4.777759083050538,1.3993490626769542,10.642841134590933,27.517255206764048,1.0142951912516207,0.41672442548319527,0.3331443526320188,Energy,1 -solar/H/medium,FlowState-9.1M,801.285641917579,801.285641917579,11.445277230292922,11.445277230292922,0.8764719755896929,5.458095207860186,1.400396199470019,10.798993971727489,28.30698927681252,1.0196838280418565,0.4122856014484468,0.34501846917605444,Energy,1 -solar/H/long,FlowState-9.1M,719.3457268856448,719.3457268856448,12.36631455371173,12.36631455371173,0.942005746657252,5.876856700090995,1.4082166438168466,8.146907869423451,26.820621299396567,0.9299199543477858,0.42876272465383575,0.3335355229807926,Energy,1 -solar/W/short,FlowState-9.1M,955186.6131386862,955186.6131386862,768.0469177691606,768.0469177691606,0.838795166934673,0.16764145785004553,0.15302670088997722,4.476202957998712,977.3364892086482,0.19949880509892567,0.15677757256247188,0.12235764216039015,Energy,1 -temperature_rain/D/short,FlowState-9.1M,180.4470724159759,180.4470724159759,5.7659389208630145,5.7659389208630145,1.3497787807137442,24.105154695755104,1.4952793405611897,24.459044384745972,13.433058937411683,1.5814261953049171,0.6788034573857006,0.5497859033550305,Nature,1 -us_births/D/short,FlowState-9.1M,218382.50666666665,218382.50666666665,310.88701822916664,310.88701822916664,0.4573647670456565,0.0298173189163208,0.02927158196767171,5.9522450787192085,467.3141413082496,0.043806908712113006,0.02914313525633875,0.023686841289048206,Healthcare,1 -us_births/M/short,FlowState-9.1M,56510709.333333336,56510709.333333336,6193.608072916667,6193.608072916667,0.6995303835548405,0.018985547125339508,0.01924945041537285,6.844936444934768,7517.360529689482,0.02334887055249893,0.019237304446992797,0.014923346389001804,Healthcare,1 -us_births/W/short,FlowState-9.1M,2057186.5714285714,2057186.5714285714,1119.4234793526787,1119.4234793526787,1.0174865839349032,0.015180262071745736,0.015214066420282637,9.795766029253233,1434.2895702850842,0.01947034027817997,0.015196063967786272,0.011881630832056645,Healthcare,1 diff --git a/results/FlowState-9.1M/config.json b/results/FlowState-9.1M/config.json deleted file mode 100644 index 0a70177b17a700ded4b985a09a376c18b5c7b8db..0000000000000000000000000000000000000000 --- a/results/FlowState-9.1M/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "FlowState-9.1M", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/ibm-research/flowstate", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/flowstate.ipynb", - "org": "IBM Research", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/Kairos_10m/all_results.csv b/results/Kairos_10m/all_results.csv deleted file mode 100644 index eb56a8cfa0bfac79116c710a4f8ada24aaaeac20..0000000000000000000000000000000000000000 --- a/results/Kairos_10m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/5T/short,Kairos_10m,1810532.091,1912681.723,187.0098493,0.831729591,3.002209001,0.590534531,16.22975749,1345.56014,4.22472607,0.587164678,0.458352563,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Kairos_10m,3110730.865,3238868.958,329.4609586,1.060140721,6.177741877,0.695202082,30.83011185,1763.726414,5.358687963,1.000993385,0.877951505,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Kairos_10m,4586108.869,4712353.369,430.1446869,0.979887879,6.071620542,0.73735556,29.01335334,2141.520224,5.659398546,1.136743977,1.036897936,Web/CloudOps,2 -bitbrains_fast_storage/H/short,Kairos_10m,2869327.387,2453456.713,309.0994843,1.114115335,4.103139998,0.494369588,22.88990317,1693.908907,4.828190642,0.881033939,0.655800091,Web/CloudOps,2 -bitbrains_rnd/5T/short,Kairos_10m,1465660.69,1581001.544,128.4923884,1.76123324,1.312806388,0.57635041,55.65831158,1210.644741,4.953608811,0.525753762,0.441768547,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Kairos_10m,2108348.405,2161279.709,175.5262686,4.495163697,1.591177821,0.662557233,168.5697024,1452.015291,6.109368508,0.738528488,0.691730142,Web/CloudOps,2 -bitbrains_rnd/5T/long,Kairos_10m,2737334.922,2853808.602,229.7301491,3.430676412,3.511605216,0.647875606,126.7719811,1654.489324,6.337975891,0.88004445,0.825368471,Web/CloudOps,2 -bitbrains_rnd/H/short,Kairos_10m,1832654.855,1730415.982,179.0902405,5.842787788,2.499416124,0.553727085,201.9890565,1353.755833,5.924697933,0.783786523,0.649874582,Web/CloudOps,2 -bizitobs_application/10S/short,Kairos_10m,613666.7456,625286.3925,416.1294406,1.580968231,0.021784332,0.022089067,22.20717372,783.3688439,0.030211407,0.01604845,0.013137639,Web/CloudOps,2 -bizitobs_application/10S/medium,Kairos_10m,3790500.788,3729079.932,1075.250867,2.673845178,0.037245446,0.037884086,61.22701643,1946.920848,0.076449853,0.042221938,0.039055802,Web/CloudOps,2 -bizitobs_application/10S/long,Kairos_10m,5949571.921,5853577.146,1459.86961,3.579093613,0.049599603,0.05072162,94.99143183,2439.174434,0.094069216,0.05630134,0.053402801,Web/CloudOps,2 -bizitobs_l2c/5T/short,Kairos_10m,20.54262746,20.74002412,2.659062964,0.274390343,0.155866221,0.158145838,4.543189006,4.53239754,0.155660873,0.091322982,0.078075822,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Kairos_10m,131.2736865,132.1261307,7.491670865,0.73033687,0.558713023,0.674872034,20.16752317,11.45747296,0.602831584,0.394172068,0.366499537,Web/CloudOps,7 -bizitobs_l2c/5T/long,Kairos_10m,296.5507789,299.1918647,11.35085559,1.073609237,0.847646851,0.899640614,34.33167048,17.22064978,0.966913743,0.637333574,0.609135909,Web/CloudOps,7 -bizitobs_l2c/H/short,Kairos_10m,60.7329608,61.04508003,4.800797728,0.484780387,0.441340598,0.511886786,5.430103609,7.793135492,0.420069996,0.258775314,0.215633997,Web/CloudOps,7 -bizitobs_l2c/H/medium,Kairos_10m,69.93757513,70.26804824,4.776177979,0.491334673,0.502032132,0.602784975,9.386819422,8.362868834,0.506384423,0.289204838,0.255884742,Web/CloudOps,7 -bizitobs_l2c/H/long,Kairos_10m,82.14190456,82.31918035,5.257683624,0.56686766,0.626630707,0.667743657,13.46802478,9.06321712,0.553607483,0.321154504,0.291799819,Web/CloudOps,7 -bizitobs_service/10S/short,Kairos_10m,4058.543552,4079.008437,22.34945665,0.884871298,0.087248944,0.064748737,10.88395396,63.70669943,0.047195961,0.016557192,0.014147005,Web/CloudOps,2 -bizitobs_service/10S/medium,Kairos_10m,39408.41962,39177.0733,48.87135651,1.24555563,0.087958938,0.086401506,24.49902286,198.51554,0.149407464,0.036781732,0.03392636,Web/CloudOps,2 -bizitobs_service/10S/long,Kairos_10m,131583.8009,131476.6236,78.34476089,1.535978278,0.122186718,0.100468909,35.85316286,362.7448151,0.268732962,0.058040305,0.055512398,Web/CloudOps,2 -car_parts/M/short,Kairos_10m,1.379966796,1.345996134,0.547855546,0.971052955,0.71106495,1.795214667,18.41156084,1.17471988,2.816857769,1.3137014,1.069803943,Sales,1 -covid_deaths/D/short,Kairos_10m,3988612.078,3794762.04,286.3498588,41.03518789,0.119880061,0.152260507,1141.692506,1997.15099,0.751437744,0.107740523,0.093316017,Healthcare,1 -electricity/15T/short,Kairos_10m,153524.6974,155437.4988,59.07438236,1.098731204,0.182009491,0.172097834,18.18759412,391.8222778,0.772358953,0.116447254,0.100072249,Energy,1 -electricity/15T/medium,Kairos_10m,342047.7929,343177.9305,63.51896658,0.932210714,0.146614085,0.139066496,15.18092329,584.8485213,1.007752889,0.109449575,0.095450165,Energy,1 -electricity/15T/long,Kairos_10m,415426.3794,416555.2406,68.01105515,0.979369127,0.158566788,0.143257516,17.29451852,644.535786,1.01683423,0.107295778,0.094121759,Energy,1 -electricity/H/short,Kairos_10m,1606337.709,1649895.876,187.5725166,1.022751611,0.220597024,0.128594449,14.22096805,1267.413788,0.598309823,0.088547624,0.07415508,Energy,1 -electricity/H/medium,Kairos_10m,6211735.11,6234717.849,261.5160119,1.150560967,0.251110834,0.135597075,19.29738401,2492.335272,0.972840575,0.102078316,0.088152411,Energy,1 -electricity/H/long,Kairos_10m,9860003.068,9879399.064,301.4680021,1.290051537,0.330484509,0.149583873,25.40375538,3140.064182,1.238440419,0.118898894,0.105567037,Energy,1 -electricity/D/short,Kairos_10m,1247127328,1245713552,4262.463378,1.449482229,0.331268085,0.097179615,16.37080746,35314.68997,0.58033764,0.070046429,0.05695268,Energy,1 -electricity/W/short,Kairos_10m,36836035416,37253801527,25556.25897,1.412593911,0.248364726,0.092946908,14.83053764,191927.1617,0.437914454,0.05831095,0.046852236,Energy,1 -ett1/15T/short,Kairos_10m,5.300356797,5.31324402,1.133080507,0.720062519,0.520042192,0.246191844,8.79494876,2.302250377,0.435221135,0.214199372,0.179693587,Energy,7 -ett1/15T/medium,Kairos_10m,9.566173699,9.627244225,1.619898853,1.011907023,0.76232271,0.389823271,17.42351711,3.092923164,0.58951418,0.308754305,0.268566617,Energy,7 -ett1/15T/long,Kairos_10m,9.280986236,9.302320575,1.61754719,1.031722205,0.802001847,0.391256179,19.45241591,3.046471112,0.580660376,0.308306078,0.271595474,Energy,7 -ett1/H/short,Kairos_10m,107.2321071,108.1013412,5.147870834,0.85338299,0.514793758,0.268129726,8.974475367,10.35529367,0.483323921,0.240272193,0.198732952,Energy,7 -ett1/H/medium,Kairos_10m,138.9595444,139.5240566,7.001363107,1.289394967,4.30854E+12,0.432953863,23.30471769,11.7881103,0.564622566,0.335348712,0.287811738,Energy,7 -ett1/H/long,Kairos_10m,146.9223161,147.2141426,7.117938773,1.323733772,4.06479E+12,0.454740494,27.96227788,12.1211516,0.580455686,0.340862665,0.299351126,Energy,7 -ett1/D/short,Kairos_10m,40916.28663,40854.56691,135.4433603,1.732398179,1.169964409,0.485007665,14.62261282,202.2777463,0.53128923,0.355746492,0.283327069,Energy,7 -ett1/W/short,Kairos_10m,1562560.839,1542194.275,873.7720124,1.388739924,0.693768152,0.475561352,9.158197955,1250.024335,0.497414527,0.347694745,0.269048117,Energy,7 -ett2/15T/short,Kairos_10m,8.302789351,8.311591102,1.764882267,0.757097444,0.107651499,0.129594791,9.171380634,2.881456116,0.136792976,0.083785242,0.069264253,Energy,7 -ett2/15T/medium,Kairos_10m,12.21097657,12.24386233,2.173864332,0.89320208,0.134642523,0.162314071,17.29400971,3.494420777,0.186223999,0.115849159,0.102010919,Energy,7 -ett2/15T/long,Kairos_10m,11.75403925,11.74075185,2.1722814,0.899702535,0.13602665,0.164359369,20.04326974,3.428416435,0.182706505,0.1157648,0.104163117,Energy,7 -ett2/H/short,Kairos_10m,103.1219513,103.4147463,6.344779407,0.737790623,0.121383688,0.11105188,7.510896992,10.15489789,0.128792354,0.080469453,0.065633211,Energy,7 -ett2/H/medium,Kairos_10m,232.9115885,233.865732,9.670508716,1.034360918,0.173058682,0.167535172,19.72308539,15.26144123,0.205657718,0.130316313,0.114305765,Energy,7 -ett2/H/long,Kairos_10m,218.0616891,217.4376764,9.143644036,1.001303315,0.175363239,0.170617224,23.64694857,14.76691197,0.202221496,0.125215169,0.113164663,Energy,7 -ett2/D/short,Kairos_10m,124359.417,119712.8567,206.7277103,1.340607581,0.441081873,0.13563725,11.62884797,352.6463058,0.201828773,0.118315716,0.09688143,Energy,7 -ett2/W/short,Kairos_10m,4225332.79,4052880.539,1384.865595,0.953398771,0.164246292,0.189289749,5.64283276,2055.561429,0.172430072,0.11616898,0.091534065,Energy,7 -hierarchical_sales/D/short,Kairos_10m,27.91455607,28.35606058,2.318985732,0.753311602,0.628465036,1.016531245,9.224303992,5.283422761,1.620895593,0.711439141,0.596698377,Sales,1 -hierarchical_sales/W/short,Kairos_10m,480.0083695,482.9231101,9.405300158,0.741136701,0.587627419,0.471909615,8.387690391,21.90909331,1.00585719,0.431801931,0.365377768,Sales,1 -hospital/M/short,Kairos_10m,3993.181301,3898.252304,20.00846385,0.778758211,0.20181704,0.178820548,7.087910855,63.19162366,0.229400268,0.07263537,0.058375885,Healthcare,1 -jena_weather/10T/short,Kairos_10m,671.8771544,727.4382035,5.30533567,0.287055399,0.392783991,0.388796476,3.193699929,25.92059325,0.160999378,0.032952785,0.028034228,Nature,21 -jena_weather/10T/medium,Kairos_10m,1538.693578,1544.01571,10.11403831,0.646882761,0.767474677,0.543149908,15.4281034,39.22618485,0.24073043,0.062069681,0.056651488,Nature,21 -jena_weather/10T/long,Kairos_10m,1494.616802,1491.537582,9.925640064,0.674084581,0.814208848,0.49646408,17.36593219,38.66027421,0.236568839,0.060736691,0.056184652,Nature,21 -jena_weather/H/short,Kairos_10m,1078.625471,1088.699065,8.282835479,0.522555661,1.216712904,0.48348571,6.070476252,32.842434,0.201322019,0.050773252,0.044053348,Nature,21 -jena_weather/H/medium,Kairos_10m,1382.86239,1383.213599,10.64639829,0.784102826,2.235314318,0.491635757,15.19587935,37.18685776,0.227552972,0.065147198,0.059242461,Nature,21 -jena_weather/H/long,Kairos_10m,1392.874626,1406.725912,11.7966697,1.010064577,3.029660378,0.553565525,26.1871929,37.32123559,0.224715199,0.071029025,0.06460137,Nature,21 -jena_weather/D/short,Kairos_10m,417.7352865,432.4015161,10.46425314,1.151357989,0.684712558,0.465561918,9.177513542,20.43857349,0.123073616,0.063011906,0.050933146,Nature,21 -kdd_cup_2018/H/short,Kairos_10m,4419.601553,4529.073298,22.49060626,0.934864612,0.895238641,0.492621138,9.6108999,66.48008388,1.391548557,0.470769121,0.385794681,Nature,1 -kdd_cup_2018/H/medium,Kairos_10m,5026.305216,5051.372151,25.09016039,1.042052773,0.982277674,0.55276731,23.58832538,70.89644008,1.48399089,0.525182497,0.47633815,Nature,1 -kdd_cup_2018/H/long,Kairos_10m,4109.368354,4158.984394,24.29095432,1.017889098,1.017874715,0.606667263,27.18648876,64.10435519,1.504147212,0.569963946,0.529071639,Nature,1 -kdd_cup_2018/D/short,Kairos_10m,2738.983347,2861.86742,20.90804455,1.18084037,0.521594111,0.457834174,12.82975368,52.33529733,1.172366229,0.468362398,0.38959781,Nature,1 -loop_seattle/5T/short,Kairos_10m,40.64405025,41.11053169,3.530882594,0.559802009,0.095450682,0.075121614,5.910752047,6.375268642,0.109321782,0.060546841,0.049453158,Transport,1 -loop_seattle/5T/medium,Kairos_10m,77.39246813,78.53904624,5.151573297,0.817428835,0.16932764,0.117884492,13.80072234,8.797298911,0.156522025,0.091657075,0.079090824,Transport,1 -loop_seattle/5T/long,Kairos_10m,90.2883451,91.16576298,5.558082904,0.877834318,0.178302018,0.125476249,16.91393642,9.502017949,0.167997631,0.098268048,0.086300006,Transport,1 -loop_seattle/H/short,Kairos_10m,57.18461933,57.60757245,4.23055205,0.856369599,0.122813514,0.097658313,10.65703661,7.562051264,0.133855998,0.074885074,0.0626711,Transport,1 -loop_seattle/H/medium,Kairos_10m,66.81366939,66.56706103,4.867457479,0.993321252,0.134308115,0.110150638,16.42223228,8.173962894,0.144473881,0.086031767,0.074308162,Transport,1 -loop_seattle/H/long,Kairos_10m,60.70510834,61.31113802,4.628187224,0.960032499,0.148915903,0.107596221,17.61205959,7.791348301,0.139241802,0.082711888,0.07228013,Transport,1 -loop_seattle/D/short,Kairos_10m,18.45696981,18.54099565,2.945943744,0.88573819,0.054799714,0.054443332,15.87653927,4.296157564,0.076778074,0.05264795,0.046094922,Transport,1 -m4_daily/D/short,Kairos_10m,391497.8652,397876.8751,184.5510234,3.368832711,0.04615679,0.031635598,37.43967871,625.6979025,0.096653034,0.028508033,0.023419576,Econ/Fin,1 -m4_hourly/H/short,Kairos_10m,969630.2788,975116.8155,223.5987388,1.181314413,0.116265106,0.104532851,10.61118617,984.6980648,0.134433036,0.030526167,0.024118603,Econ/Fin,1 -m4_monthly/M/short,Kairos_10m,1877492.86,1890124.823,567.3343786,0.969122306,0.160023484,0.132755817,11.58481131,1370.216355,0.284777998,0.11791156,0.097795424,Econ/Fin,1 -m4_quarterly/Q/short,Kairos_10m,1857582.324,1858601.262,574.8336255,1.192798692,0.119648127,0.104618625,14.01880353,1362.931519,0.228130736,0.096217026,0.080001916,Econ/Fin,1 -m4_weekly/W/short,Kairos_10m,330414.0372,333779.3971,298.6387644,2.674855413,0.070933405,0.07033857,21.51909622,574.8165248,0.104722657,0.054407352,0.043742066,Econ/Fin,1 -m4_yearly/A/short,Kairos_10m,3178454.973,3183331.875,816.8244874,2.985468989,0.162086473,0.137889205,40.2147412,1782.822193,0.285883801,0.130981592,0.110006829,Econ/Fin,1 -m_dense/H/short,Kairos_10m,48160.97429,48326.53369,98.64674668,0.854007898,0.346444744,0.246750134,12.01473687,219.4560874,0.389260451,0.174974764,0.146597968,Transport,1 -m_dense/H/medium,Kairos_10m,42946.55675,42927.45376,98.68519071,0.832516702,0.360330447,0.25326675,12.16657343,207.2355103,0.360563619,0.17169977,0.143687786,Transport,1 -m_dense/H/long,Kairos_10m,45005.55068,44941.54037,102.4647671,0.863176632,0.369114845,0.257607789,13.79695231,212.145117,0.367254577,0.177381668,0.150249557,Transport,1 -m_dense/D/short,Kairos_10m,8355.244675,8344.594807,46.82486186,0.695106111,0.126786815,0.100242404,9.683785974,91.40702749,0.158239086,0.081060762,0.067145752,Transport,1 -restaurant/D/short,Kairos_10m,140.5706043,141.3613837,7.23156217,0.69355123,0.711404825,0.397673358,6.526674012,11.85624748,0.54565198,0.332813246,0.269470384,Sales,1 -saugeen/D/short,Kairos_10m,1117.759425,1329.733154,13.99998133,3.11030755,0.310290698,0.357092107,38.55055068,33.43290931,1.082899882,0.453462724,0.377373245,Nature,1 -saugeen/W/short,Kairos_10m,867.9728929,910.4976711,15.12883234,1.22722935,0.437968539,0.423039432,14.32639105,29.46137968,0.889316241,0.456676383,0.378713428,Nature,1 -saugeen/M/short,Kairos_10m,374.6974066,378.1033093,12.05295908,0.706306814,0.354173757,0.343810414,9.545841469,19.35710223,0.581158242,0.361865966,0.307696213,Nature,1 -solar/10T/short,Kairos_10m,30.11446408,29.83210635,2.446171777,1.080078207,6.329230346,1.095985989,22.61629389,5.487664721,1.59636106,0.7115911,0.632086323,Energy,1 -solar/10T/medium,Kairos_10m,18.87475381,18.75175701,1.751308057,0.767035677,3.74801772,0.953134309,18.09525531,4.344508466,0.964063694,0.388622218,0.353206899,Energy,1 -solar/10T/long,Kairos_10m,19.8978288,19.91892616,1.851552939,0.810446929,3.695683806,0.945560794,19.14383605,4.460698241,0.964108573,0.400183551,0.363499862,Energy,1 -solar/H/short,Kairos_10m,728.6604486,722.2543287,11.40852035,0.880963217,4.851503329,1.096192009,14.47191876,26.99371128,0.994997188,0.42052186,0.369250237,Energy,1 -solar/H/medium,Kairos_10m,731.1075893,726.8118343,11.26094998,0.862083883,5.024127648,0.869450865,15.68261525,27.03900126,0.974007934,0.405645701,0.357972002,Energy,1 -solar/H/long,Kairos_10m,719.706837,717.2635631,11.99304579,0.913622403,6.480797139,0.870100589,16.4780499,26.8273524,0.93015332,0.415820808,0.365656595,Energy,1 -solar/D/short,Kairos_10m,128887.22,138248.8152,253.4257179,0.988675235,1.125668135,0.424552004,7.250994189,359.0086629,0.518645257,0.366113858,0.298755026,Energy,1 -solar/W/short,Kairos_10m,1724042.309,1702531.917,1049.920951,1.14966831,0.23934989,0.205960952,5.211548907,1313.027916,0.268021813,0.214315106,0.161385581,Energy,1 -sz_taxi/15T/short,Kairos_10m,16.71656027,16.76992299,2.765511895,0.552011102,1.0565E+12,0.398875223,5.280649393,4.088589032,0.382300573,0.258587198,0.208897079,Transport,1 -sz_taxi/15T/medium,Kairos_10m,17.05466285,17.06151511,2.828485422,0.551494367,1.08737E+13,0.410716438,10.57946274,4.12972915,0.38434022,0.263237774,0.231693625,Transport,1 -sz_taxi/15T/long,Kairos_10m,16.15779234,16.11721364,2.77039906,0.52302319,7.00833E+12,0.40564965,11.45797907,4.019675652,0.372053995,0.256423186,0.230494889,Transport,1 -sz_taxi/H/short,Kairos_10m,7.251273563,7.275430271,1.868721965,0.57004514,1.101679632,0.299723346,5.339876235,2.692818888,0.250852698,0.174082984,0.141010543,Transport,1 -temperature_rain/D/short,Kairos_10m,170.5559356,180.4207189,5.995170112,1.414163307,39.72589316,1.434223286,24.49561994,13.05970657,1.53747275,0.705790029,0.583723713,Nature,1 -us_births/D/short,Kairos_10m,209347.3418,208010.0311,298.7584456,0.439924569,0.028429703,0.028101218,5.739075333,457.5449068,0.042891122,0.02800618,0.023432893,Healthcare,1 -us_births/W/short,Kairos_10m,3314698.973,3324767.554,1350.922852,1.228312098,0.018144656,0.018334756,10.53156755,1820.631476,0.024714894,0.018338645,0.014700383,Healthcare,1 -us_births/M/short,Kairos_10m,43947173.33,44131778.67,5834.16276,0.659422984,0.018011262,0.018105495,3.125986382,6629.266425,0.020590456,0.01812087,0.013579064,Healthcare,1 diff --git a/results/Kairos_10m/config.json b/results/Kairos_10m/config.json deleted file mode 100644 index 9aba0c6d414090d08837103059b20ee52444c2ff..0000000000000000000000000000000000000000 --- a/results/Kairos_10m/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "Kairos_10m", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/mldi-lab/Kairos_10m", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/kairos.ipynb", - "org": "ShanghaiTech University", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/Kairos_23m/all_results.csv b/results/Kairos_23m/all_results.csv deleted file mode 100644 index 4b140611d5132187c12b59c19d44cf958c73fd35..0000000000000000000000000000000000000000 --- a/results/Kairos_23m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/5T/short,Kairos_23m,1707341.794,1784558.671,179.0190917,0.808000842,2.704615412,0.560622395,15.76129876,1306.652897,4.102566949,0.562075675,0.441013666,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Kairos_23m,3328570.202,3531051.56,324.7435942,1.081494928,5.97465739,0.70911085,30.98600809,1824.436955,5.543143353,0.986660729,0.86709214,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Kairos_23m,4568745.259,4653083.681,413.6801344,0.976229155,5.614626186,0.732703378,28.1947039,2137.462341,5.648674774,1.09323308,0.992588002,Web/CloudOps,2 -bitbrains_fast_storage/H/short,Kairos_23m,3413698.149,3045016.897,353.0351745,1.109313467,4.187970438,0.48545739,21.90959844,1847.61959,5.266316019,1.006264929,0.741144631,Web/CloudOps,2 -bitbrains_rnd/5T/short,Kairos_23m,1534957.708,1663030.753,133.5207793,1.752172492,1.338067227,0.549458185,55.81945093,1238.934102,5.069360707,0.546328486,0.462226594,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Kairos_23m,2543514.582,2617378.394,211.131306,4.491108758,2.761358799,0.657647288,168.1060341,1594.839986,6.710304809,0.88833703,0.83501316,Web/CloudOps,2 -bitbrains_rnd/5T/long,Kairos_23m,2646024.019,2724295.42,215.879959,3.418166699,3.199160744,0.623240056,125.8029147,1626.660388,6.231369509,0.826987491,0.779093528,Web/CloudOps,2 -bitbrains_rnd/H/short,Kairos_23m,1829934.461,1698663.671,181.6947208,5.918637469,2.606476248,0.538474269,203.4749602,1352.750702,5.920298987,0.795185004,0.65899633,Web/CloudOps,2 -bizitobs_application/10S/short,Kairos_23m,428854.2467,436366.8048,360.2895687,1.284805938,0.017477108,0.01767147,19.2922527,654.869641,0.025255706,0.013894929,0.011657033,Web/CloudOps,2 -bizitobs_application/10S/medium,Kairos_23m,3848082.191,3803404.211,1085.956001,2.519400122,0.035465608,0.036134312,60.42480203,1961.652923,0.077028339,0.042642297,0.039821635,Web/CloudOps,2 -bizitobs_application/10S/long,Kairos_23m,5454809.804,5394829.469,1420.80945,3.036164922,0.042674628,0.04368477,79.92239677,2335.553426,0.090072967,0.054794946,0.051865761,Web/CloudOps,2 -bizitobs_l2c/5T/short,Kairos_23m,21.71855353,21.86138777,2.762863287,0.283967715,0.159403307,0.158433581,4.481306806,4.660316891,0.160054141,0.094887906,0.080466497,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Kairos_23m,125.9852388,126.4387094,7.447215776,0.75172113,0.526929307,0.694814941,20.48971538,11.22431462,0.590564027,0.391833077,0.362121991,Web/CloudOps,7 -bizitobs_l2c/5T/long,Kairos_23m,309.4200162,310.8392044,11.74248572,1.11161244,0.884080186,0.92823194,35.13115033,17.59033872,0.987671225,0.659323021,0.627991555,Web/CloudOps,7 -bizitobs_l2c/H/short,Kairos_23m,58.27923192,58.40539445,4.730869463,0.480441452,0.456774521,0.536242635,6.449291552,7.634083568,0.411496689,0.255006001,0.216424908,Web/CloudOps,7 -bizitobs_l2c/H/medium,Kairos_23m,64.85084853,64.89934256,4.740805435,0.49038535,0.501237908,0.656779709,9.271107074,8.053002455,0.487621542,0.287062977,0.253086961,Web/CloudOps,7 -bizitobs_l2c/H/long,Kairos_23m,78.84433918,78.94470137,5.141076927,0.555519392,0.618180873,0.692736589,13.19774405,8.879433494,0.542381448,0.31403183,0.285876414,Web/CloudOps,7 -bizitobs_service/10S/short,Kairos_23m,3733.905852,3732.507719,21.3423794,0.83937801,0.081588223,0.076264087,11.14286213,61.10569411,0.045269053,0.015811117,0.013379555,Web/CloudOps,2 -bizitobs_service/10S/medium,Kairos_23m,48475.35173,48416.88268,49.18446468,1.195938998,0.095605106,0.100061225,23.61456548,220.1711873,0.165706014,0.037017385,0.034242579,Web/CloudOps,2 -bizitobs_service/10S/long,Kairos_23m,130500.5745,130369.5168,74.21120669,1.387789888,0.104107327,0.11587307,32.58120384,361.2486325,0.267624542,0.054978036,0.052709078,Web/CloudOps,2 -car_parts/M/short,Kairos_23m,1.35126107,1.31537328,0.55208667,0.984477066,0.691285403,1.78780268,17.95724707,1.162437555,2.787406014,1.323847202,1.067456747,Sales,1 -covid_deaths/D/short,Kairos_23m,4200571.492,4058449.428,290.3054806,41.01133791,0.120935699,0.34891478,1068.641889,2049.529578,0.771145442,0.109228845,0.092407369,Healthcare,1 -electricity/15T/short,Kairos_23m,151495.2357,152907.6878,59.37378781,1.114015098,0.193453193,0.17018331,19.21801586,389.2238889,0.767237016,0.117037441,0.100976263,Energy,1 -electricity/15T/medium,Kairos_23m,357419.005,359468.0345,63.07073961,0.906346837,0.147418971,0.136715352,14.85963536,597.8453019,1.030147651,0.108677235,0.095116077,Energy,1 -electricity/15T/long,Kairos_23m,393679.1096,394906.6041,66.31459815,0.952142182,0.157162891,0.140069446,16.85484711,627.4385305,0.989861214,0.10461941,0.091762582,Energy,1 -electricity/H/short,Kairos_23m,1558370.562,1589332.451,181.873561,0.993847504,0.226693101,0.126667516,13.82499369,1248.347132,0.589308999,0.08585731,0.071976246,Energy,1 -electricity/H/medium,Kairos_23m,6075265.137,6118089.187,258.0729869,1.138975944,0.248377239,0.134842464,18.77937102,2464.805294,0.962094718,0.10073439,0.086806012,Energy,1 -electricity/H/long,Kairos_23m,9848452.691,9908715.864,299.7382446,1.287702513,0.32363146,0.14865293,24.87294107,3138.224449,1.237714828,0.118216678,0.104681863,Energy,1 -electricity/D/short,Kairos_23m,1315988843,1307053715,4336.847313,1.449698347,0.355762507,0.097909129,15.86737479,36276.56052,0.596144367,0.071268804,0.057508934,Energy,1 -electricity/W/short,Kairos_23m,41207838979,41641755155,26355.03477,1.417538512,0.240266199,0.091990789,15.01738318,202997.1403,0.463172493,0.060133493,0.048222479,Energy,1 -ett1/15T/short,Kairos_23m,5.046287597,5.040945261,1.105387792,0.710783211,0.504222102,0.240302058,9.503843586,2.246394355,0.424662022,0.208964296,0.177053945,Energy,7 -ett1/15T/medium,Kairos_23m,9.37870697,9.409024851,1.60489475,1.014918936,0.785989691,0.390423498,18.93467566,3.062467464,0.583709293,0.305894509,0.26927411,Energy,7 -ett1/15T/long,Kairos_23m,8.976488554,8.984341912,1.569445922,1.010603201,0.787634474,0.389888382,20.04390289,2.996078863,0.571055564,0.299137929,0.265214963,Energy,7 -ett1/H/short,Kairos_23m,105.937825,106.9329282,5.14991562,0.852810101,0.519497721,0.269973462,8.610663579,10.29261021,0.480398227,0.240367631,0.197163041,Energy,7 -ett1/H/medium,Kairos_23m,140.8053483,141.0363157,7.060981946,1.316249527,3.4834E+12,0.440387104,24.12348267,11.86614294,0.568360145,0.338204313,0.289715033,Energy,7 -ett1/H/long,Kairos_23m,154.8883624,155.4902709,7.111122511,1.335753115,2.06196E+12,0.457200531,28.455959,12.44541532,0.595983973,0.340536249,0.299249152,Energy,7 -ett1/D/short,Kairos_23m,38758.73978,38879.20092,130.4698161,1.63771442,1.332695709,0.45825865,14.15551161,196.8723947,0.517091894,0.342683313,0.27365841,Energy,7 -ett1/W/short,Kairos_23m,1173650.819,1152782.474,804.0648717,1.425272753,0.516807305,0.516429872,9.4166173,1083.35166,0.43109149,0.319956609,0.253953737,Energy,7 -ett2/15T/short,Kairos_23m,9.203477002,9.268239052,1.884437798,0.798972716,0.113856208,0.135158918,10.22236821,3.03372329,0.144021641,0.089460968,0.074564323,Energy,7 -ett2/15T/medium,Kairos_23m,13.15442562,13.20147098,2.237063072,0.909482641,0.13588952,0.161862105,17.5250791,3.626903035,0.193284218,0.119217134,0.104911461,Energy,7 -ett2/15T/long,Kairos_23m,12.32623053,12.32898447,2.21113724,0.907549653,0.135350556,0.162084741,20.0203573,3.510873186,0.187100775,0.117835498,0.105881471,Energy,7 -ett2/H/short,Kairos_23m,109.576251,109.52402,6.467573708,0.746650059,0.122458384,0.111828774,7.857717911,10.46786755,0.13276168,0.082026826,0.067473941,Energy,7 -ett2/H/medium,Kairos_23m,224.5714778,224.8240202,9.381193697,1.007837385,0.168440071,0.164329425,18.93446311,14.98570912,0.201942051,0.126417607,0.110914282,Energy,7 -ett2/H/long,Kairos_23m,222.4406173,222.0433279,9.194034282,1.008696567,0.174330261,0.170523555,24.13914005,14.91444325,0.204241824,0.125905225,0.114217777,Energy,7 -ett2/D/short,Kairos_23m,112864.7907,107753.8629,196.8557583,1.299417389,0.441465764,0.134760083,11.70359315,335.9535544,0.192275072,0.112665738,0.092540734,Energy,7 -ett2/W/short,Kairos_23m,4150899.216,3851434.076,1378.458293,0.974422298,0.172392256,0.201068219,6.133161686,2037.375571,0.170904557,0.115631505,0.090797113,Energy,7 -hierarchical_sales/D/short,Kairos_23m,27.63740127,28.0194047,2.320944333,0.752799204,0.630752444,1.012782492,8.883299434,5.257128614,1.61282884,0.712040018,0.593441178,Sales,1 -hierarchical_sales/W/short,Kairos_23m,477.5471216,475.1658983,9.405112953,0.739019721,0.582120064,0.469097267,8.142246176,21.85285157,1.003275104,0.431793336,0.365047483,Sales,1 -hospital/M/short,Kairos_23m,3520.236374,3482.872011,19.22710103,0.778595956,0.199949906,0.177208595,7.11413443,59.33157991,0.215387413,0.069798842,0.056159187,Healthcare,1 -jena_weather/10T/short,Kairos_23m,665.6996914,738.7743629,5.319821034,0.287800469,0.371741177,0.286308668,3.175860949,25.80115678,0.160257528,0.033042757,0.028078195,Nature,21 -jena_weather/10T/medium,Kairos_23m,1459.735745,1471.256187,9.794295435,0.631514464,0.76686471,0.468485839,14.7559298,38.20648826,0.234472569,0.060107425,0.054855219,Nature,21 -jena_weather/10T/long,Kairos_23m,1474.013604,1474.98976,9.818192732,0.657502959,0.786090625,0.446823014,17.09893396,38.39288481,0.234932637,0.060079203,0.055633265,Nature,21 -jena_weather/H/short,Kairos_23m,1050.920584,1061.345807,8.200854975,0.529682405,1.375343121,0.391644545,5.918081082,32.4179053,0.198719685,0.050270716,0.043463064,Nature,21 -jena_weather/H/medium,Kairos_23m,1413.997261,1415.30579,10.94789852,0.838833883,3.356717372,0.499567536,16.15376344,37.60315493,0.230100368,0.066992131,0.060483964,Nature,21 -jena_weather/H/long,Kairos_23m,1213.065642,1208.287068,11.93862578,1.067859211,3.607675903,0.502165502,27.99286492,34.82909189,0.209709732,0.071883758,0.065411956,Nature,21 -jena_weather/D/short,Kairos_23m,414.3082141,419.1556883,10.64303979,1.192655032,0.775180214,0.465483169,11.02148995,20.35456249,0.122567733,0.064088494,0.05127791,Nature,21 -kdd_cup_2018/H/short,Kairos_23m,4427.067033,4547.675895,22.57937472,0.946478975,0.886918433,0.497183501,9.870102896,66.53620844,1.392723345,0.472627206,0.387048407,Nature,1 -kdd_cup_2018/H/medium,Kairos_23m,5132.194615,5165.174831,25.18841888,1.037104069,0.949265505,0.552893546,22.98073436,71.63933707,1.499541069,0.527239225,0.480136893,Nature,1 -kdd_cup_2018/H/long,Kairos_23m,4109.150866,4160.615973,24.32751484,1.022176443,1.001448125,0.609091317,26.55714712,64.1026588,1.504107408,0.570821803,0.528577145,Nature,1 -kdd_cup_2018/D/short,Kairos_23m,2729.877292,2882.415445,21.042573,1.183348289,0.526683939,0.458743711,12.03411154,52.24822765,1.170415776,0.471375978,0.389363814,Nature,1 -loop_seattle/5T/short,Kairos_23m,41.21857391,41.74000211,3.5585549,0.565187368,0.095963707,0.075800575,6.04550471,6.420169305,0.110091729,0.06102136,0.049906874,Transport,1 -loop_seattle/5T/medium,Kairos_23m,76.41336493,77.20204798,5.108584848,0.809479209,0.165790482,0.116921678,13.41299602,8.741473842,0.155528782,0.090892222,0.078314894,Transport,1 -loop_seattle/5T/long,Kairos_23m,90.81446073,91.30525049,5.567232231,0.877176524,0.177444604,0.12563399,16.65596941,9.529662152,0.168486386,0.09842981,0.086398042,Transport,1 -loop_seattle/H/short,Kairos_23m,58.2232892,58.66340734,4.262083519,0.863428695,0.121602632,0.098258336,10.86460728,7.630418678,0.135066171,0.075443213,0.063286668,Transport,1 -loop_seattle/H/medium,Kairos_23m,66.37172741,66.25331435,4.786359657,0.978355234,0.132591807,0.108508555,16.08485316,8.146884522,0.143995274,0.084598372,0.073006138,Transport,1 -loop_seattle/H/long,Kairos_23m,59.68090767,60.23684426,4.54657852,0.944957048,0.145220616,0.105705796,17.28279314,7.725341913,0.13806218,0.081253432,0.070962569,Transport,1 -loop_seattle/D/short,Kairos_23m,18.36945918,18.43300042,2.926311011,0.879414473,0.054250849,0.054045953,14.76922716,4.285960706,0.076595843,0.052297087,0.045369863,Transport,1 -m4_daily/D/short,Kairos_23m,396646.5527,401988.9526,180.3247399,3.315577651,0.04465909,0.030930723,35.33684798,629.7988193,0.097286512,0.027855189,0.022857668,Econ/Fin,1 -m4_hourly/H/short,Kairos_23m,1166720.87,1146693.283,223.9427119,1.052534835,0.108654735,0.103211934,10.0369363,1080.148541,0.147464134,0.030573127,0.024380711,Econ/Fin,1 -m4_monthly/M/short,Kairos_23m,1896907.858,1912070.585,568.781396,0.971246858,0.160140477,0.132688364,11.40348175,1377.282781,0.286246644,0.1182123,0.097844071,Econ/Fin,1 -m4_quarterly/Q/short,Kairos_23m,1903249.605,1903418.522,575.5485976,1.191176832,0.120744574,0.104665653,13.7372899,1379.583127,0.230917922,0.0963367,0.079967133,Econ/Fin,1 -m4_weekly/W/short,Kairos_23m,338007.3029,335784.8361,292.2761162,2.476020196,0.071899634,0.070616361,18.93165696,581.3839548,0.10591914,0.053248176,0.042922655,Econ/Fin,1 -m4_yearly/A/short,Kairos_23m,3226780.754,3218642.586,818.7294448,2.983811947,0.163574173,0.13771485,39.06809475,1796.324234,0.288048916,0.131287061,0.109863485,Econ/Fin,1 -m_dense/H/short,Kairos_23m,47900.96206,48249.68888,97.748952,0.848947814,0.323038216,0.241225236,11.92782338,218.8628841,0.388208256,0.1733823,0.145967115,Transport,1 -m_dense/H/medium,Kairos_23m,41303.09042,41285.4816,95.0202017,0.80189121,0.334465831,0.239759647,11.87380326,203.2316177,0.353597352,0.165323152,0.138948911,Transport,1 -m_dense/H/long,Kairos_23m,43088.12026,43001.65535,99.22641762,0.839948397,0.353919713,0.249938733,13.38339459,207.5767816,0.359346113,0.171775606,0.145812086,Transport,1 -m_dense/D/short,Kairos_23m,8286.111559,8278.529345,45.90504214,0.67853632,0.12367634,0.097950838,9.493810009,91.02808116,0.157583073,0.079468418,0.066168307,Transport,1 -restaurant/D/short,Kairos_23m,141.5972164,142.202737,7.269481804,0.696364645,0.714720607,0.399619607,6.176499711,11.89946286,0.547640852,0.334558396,0.269602297,Sales,1 -saugeen/D/short,Kairos_23m,1108.032001,1281.533603,13.75392074,3.055252873,0.31216837,0.352331438,38.2413383,33.28711464,1.078177558,0.445492763,0.376255288,Nature,1 -saugeen/W/short,Kairos_23m,893.4954509,904.9346477,15.29253114,1.240646422,0.450981859,0.42216417,16.8360356,29.89139426,0.902296589,0.461617767,0.386462357,Nature,1 -saugeen/M/short,Kairos_23m,354.9838039,358.2287118,11.91686403,0.698269366,0.341086229,0.340606738,7.901661459,18.84101388,0.565663722,0.357779985,0.300805173,Nature,1 -solar/10T/short,Kairos_23m,34.241154,34.19272553,2.689900692,1.184199488,7.545181486,1.273942407,25.47895426,5.851594142,1.702228089,0.782491814,0.702745494,Energy,1 -solar/10T/medium,Kairos_23m,18.62466252,18.58357906,1.720628738,0.752775746,3.794913401,0.752194433,18.02185436,4.315630025,0.957655453,0.381814355,0.348417012,Energy,1 -solar/10T/long,Kairos_23m,19.25185393,19.3903283,1.842976035,0.806333465,3.450628129,0.785524389,18.97302332,4.387693463,0.948329758,0.39832979,0.362447306,Energy,1 -solar/H/short,Kairos_23m,706.1653341,704.473123,11.24235991,0.868485586,5.023929993,1.050632902,14.78353493,26.57377154,0.979518069,0.41439713,0.364527023,Energy,1 -solar/H/medium,Kairos_23m,708.4717261,704.3536115,11.01345669,0.841780452,5.048762216,0.807035163,15.26293817,26.61713219,0.958811226,0.396730415,0.348079802,Energy,1 -solar/H/long,Kairos_23m,725.9884084,723.2922657,12.07100678,0.917516877,6.628130214,0.920937438,16.50732944,26.94417207,0.934203671,0.418523858,0.36584082,Energy,1 -solar/D/short,Kairos_23m,133078.1359,139097.899,254.3133683,0.992591175,1.130621915,0.425223858,8.188429179,364.7987609,0.527009977,0.367396211,0.303081549,Energy,1 -solar/W/short,Kairos_23m,1419433.958,1438369.51,963.0668386,1.050541519,0.215749582,0.189172881,4.989217959,1191.399999,0.243194516,0.196586011,0.145785997,Energy,1 -sz_taxi/15T/short,Kairos_23m,16.89509476,16.93092615,2.782575109,0.555987108,1.11501E+12,0.40057374,5.448553214,4.11036431,0.384336654,0.260182681,0.210691447,Transport,1 -sz_taxi/15T/medium,Kairos_23m,17.05363022,17.0297247,2.8496483,0.557001753,1.05379E+13,0.413602269,11.52436232,4.129604123,0.384328584,0.265207333,0.235704128,Transport,1 -sz_taxi/15T/long,Kairos_23m,16.25944957,16.18806209,2.807774724,0.530414765,6.56588E+12,0.411599253,12.0531867,4.032300779,0.373222555,0.259882611,0.234653979,Transport,1 -sz_taxi/H/short,Kairos_23m,7.307025407,7.258550881,1.871615388,0.570611393,1.167082243,0.298827038,5.309991265,2.703151015,0.251815199,0.174352524,0.141253212,Transport,1 -temperature_rain/D/short,Kairos_23m,168.7283624,177.8638801,5.980755653,1.405348839,37.6737983,1.409521261,24.60785402,12.9895482,1.529213256,0.704093066,0.581370193,Nature,1 -us_births/D/short,Kairos_23m,206202.3625,204998.1265,296.5690299,0.436686197,0.02805545,0.027780127,5.594592874,454.0951029,0.042567731,0.02780094,0.023262481,Healthcare,1 -us_births/W/short,Kairos_23m,3422206.951,3427590.342,1366.115165,1.241663925,0.018325138,0.018463339,12.02327143,1849.920796,0.025112493,0.018544879,0.014988401,Healthcare,1 -us_births/M/short,Kairos_23m,81163490.67,81312970.67,7400.020833,0.836776873,0.022714441,0.023072151,6.103282184,9009.078236,0.027982136,0.022984414,0.019080748,Healthcare,1 diff --git a/results/Kairos_23m/config.json b/results/Kairos_23m/config.json deleted file mode 100644 index 9b57583c16a6939f870956630883e722fc029ee6..0000000000000000000000000000000000000000 --- a/results/Kairos_23m/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "Kairos_23m", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/mldi-lab/Kairos_23m", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/kairos.ipynb", - "org": "ShanghaiTech University", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/Kairos_50m/all_results.csv b/results/Kairos_50m/all_results.csv deleted file mode 100644 index 21969fbaf98e411b2e671faf97b61997331d3da8..0000000000000000000000000000000000000000 --- a/results/Kairos_50m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/5T/short,Kairos_50m,1859393.025,1895906.573,191.4840931,0.777266114,3.075856404,0.556909967,15.09210526,1363.595624,4.281353028,0.601212697,0.474591749,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Kairos_50m,3617116.576,3786168.797,356.9805528,1.053425503,5.828812637,0.69568294,29.02928915,1901.871861,5.778411984,1.084605513,0.940152242,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Kairos_50m,5347065.231,5491082.498,472.626398,0.971915712,6.514752641,0.721556412,26.95761055,2312.372209,6.11090933,1.249010455,1.133669183,Web/CloudOps,2 -bitbrains_fast_storage/H/short,Kairos_50m,3790466.007,3773475.471,384.7313619,1.086932016,5.230961917,0.520531875,21.77684911,1946.911915,5.549331401,1.096609359,0.820407234,Web/CloudOps,2 -bitbrains_rnd/5T/short,Kairos_50m,1533813.825,1707646.62,133.5053577,1.717693392,1.41809104,0.524111448,54.78831385,1238.472376,5.067471458,0.546265386,0.457716991,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Kairos_50m,2273023.479,2377314.928,184.45203,4.456124703,1.910063072,0.618703098,167.2992549,1507.65496,6.34347296,0.776083716,0.727873337,Web/CloudOps,2 -bitbrains_rnd/5T/long,Kairos_50m,2547494.005,2616006.238,216.6253947,3.38159377,3.234311451,0.590594225,124.7980449,1596.087092,6.114250098,0.829843088,0.784425776,Web/CloudOps,2 -bitbrains_rnd/H/short,Kairos_50m,1820958.458,1770432.862,179.2651649,5.741018387,2.43819341,0.52154564,200.0163663,1349.428938,5.905761322,0.784552078,0.650610652,Web/CloudOps,2 -bizitobs_application/10S/short,Kairos_50m,474754.098,478055.7495,369.6837068,1.540312786,0.020862356,0.02120089,23.92358334,689.0240184,0.026572904,0.014257224,0.011454249,Web/CloudOps,2 -bizitobs_application/10S/medium,Kairos_50m,3102011.245,3048442.928,976.253023,2.274304038,0.032161208,0.032683953,55.00321834,1761.252749,0.069159213,0.038334584,0.035262935,Web/CloudOps,2 -bizitobs_application/10S/long,Kairos_50m,4840797.506,4772801.644,1321.560514,2.907980515,0.040301668,0.041303781,77.6355305,2200.181244,0.084852203,0.050967311,0.048136903,Web/CloudOps,2 -bizitobs_l2c/5T/short,Kairos_50m,20.73574656,20.84141728,2.674121444,0.274380122,0.153384109,0.157241041,4.210190709,4.553652003,0.156390837,0.091840152,0.077865696,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Kairos_50m,151.1522163,152.7400397,8.321643368,0.826131405,0.602234835,0.79061562,22.90675997,12.29439776,0.646866138,0.437840829,0.409104329,Web/CloudOps,7 -bizitobs_l2c/5T/long,Kairos_50m,325.0179643,329.4567715,12.7999909,1.224763161,0.913108588,1.101895414,39.36157586,18.02825461,1.012259547,0.718700355,0.688543969,Web/CloudOps,7 -bizitobs_l2c/H/short,Kairos_50m,57.91809892,58.5556723,4.736005423,0.480059553,0.45162486,0.553507846,4.586802967,7.610394137,0.410219768,0.255282842,0.20851936,Web/CloudOps,7 -bizitobs_l2c/H/medium,Kairos_50m,63.73390488,63.67394802,4.684195818,0.474470152,0.485989131,0.708163446,6.840211445,7.983351732,0.483404085,0.283635179,0.238645077,Web/CloudOps,7 -bizitobs_l2c/H/long,Kairos_50m,76.49806179,76.72765939,5.158500381,0.556074308,0.603111698,0.714666628,11.13733155,8.746317042,0.53425031,0.315096105,0.277948554,Web/CloudOps,7 -bizitobs_service/10S/short,Kairos_50m,3121.050634,3168.797398,21.07442242,0.909076806,0.092010181,0.063097711,12.77227131,55.86636407,0.041387589,0.015612606,0.012906506,Web/CloudOps,2 -bizitobs_service/10S/medium,Kairos_50m,31113.11522,30839.08518,43.24961181,1.108338697,0.086811396,0.081545835,22.79641562,176.3891018,0.132754587,0.032550675,0.029779048,Web/CloudOps,2 -bizitobs_service/10S/long,Kairos_50m,132030.3125,131965.1704,73.82499945,1.405237841,0.102216737,0.108052115,33.73356756,363.3597562,0.26918853,0.054691921,0.052381197,Web/CloudOps,2 -car_parts/M/short,Kairos_50m,1.414383305,1.334951029,0.558874637,0.999470277,0.681762091,1.782883487,17.05439855,1.189278481,2.851767801,1.34012405,1.085418217,Sales,1 -covid_deaths/D/short,Kairos_50m,3580828.426,3557598.145,290.0575531,42.00239176,0.123081455,0.157623721,1220.94577,1892.307699,0.711989947,0.109135561,0.096761458,Healthcare,1 -electricity/15T/short,Kairos_50m,152185.9318,153507.6004,57.34764455,1.067538813,0.184793556,0.160299234,17.79196962,390.1101535,0.768984019,0.113043513,0.097225397,Energy,1 -electricity/15T/medium,Kairos_50m,277731.7063,278646.5793,58.35341668,0.883097662,0.139763041,0.130736294,14.27574336,527.0025676,0.908078487,0.100548812,0.087194536,Energy,1 -electricity/15T/long,Kairos_50m,356994.5982,358996.9872,64.06950792,0.936887595,0.154274704,0.135775034,16.58496097,597.4902494,0.942614128,0.101077504,0.088594224,Energy,1 -electricity/H/short,Kairos_50m,1518109.226,1562907.483,174.8803717,0.949474205,0.221146524,0.120495848,13.63652791,1232.115752,0.581646629,0.082556025,0.06915336,Energy,1 -electricity/H/medium,Kairos_50m,5571500.482,5588592.121,245.2659689,1.097297265,0.25350753,0.129798362,18.9808748,2360.40261,0.92134291,0.095735389,0.08292229,Energy,1 -electricity/H/long,Kairos_50m,9082957.177,9101245.658,288.5171285,1.248374157,0.308570919,0.142595428,25.39900142,3013.794481,1.188639685,0.113791073,0.101266118,Energy,1 -electricity/D/short,Kairos_50m,1248351582,1251751137,4251.995652,1.43649799,0.321572368,0.097051711,15.87298451,35332.01922,0.580622417,0.06987441,0.056563829,Energy,1 -electricity/W/short,Kairos_50m,42773897882,42563774916,26643.75409,1.42843091,0.246369676,0.09290708,15.16564683,206818.5144,0.471891607,0.060792255,0.049238666,Energy,1 -ett1/15T/short,Kairos_50m,5.165115501,5.264231797,1.141816796,0.719104212,0.523534821,0.248611826,8.234283115,2.272689046,0.429632813,0.215850894,0.178675105,Energy,7 -ett1/15T/medium,Kairos_50m,10.10533617,10.16933795,1.63775785,1.022493869,0.774998435,0.395545879,18.25644554,3.17888914,0.605899379,0.312158248,0.273001623,Energy,7 -ett1/15T/long,Kairos_50m,8.853108631,8.880000675,1.579386801,1.012680489,0.793426423,0.389877963,19.0526992,2.975417388,0.567117466,0.30103267,0.264197838,Energy,7 -ett1/H/short,Kairos_50m,107.0877701,108.5887301,5.09058598,0.837936409,0.507895703,0.261678663,8.594493947,10.34832209,0.482998529,0.237598474,0.193503729,Energy,7 -ett1/H/medium,Kairos_50m,135.0111678,136.2789804,6.800880085,1.275834324,2.60577E+12,0.426364182,23.91029885,11.61943062,0.556543209,0.32574605,0.27976782,Energy,7 -ett1/H/long,Kairos_50m,146.0603763,147.2046833,6.867089016,1.281688514,3.06197E+12,0.435697724,28.4452898,12.0855441,0.57875052,0.328850013,0.290574828,Energy,7 -ett1/D/short,Kairos_50m,41755.00513,41961.04904,139.5849564,1.785173239,1.433147215,0.485441241,14.67657218,204.3404148,0.536706898,0.366624532,0.289489663,Energy,7 -ett1/W/short,Kairos_50m,2153185.844,2085116.145,944.0636379,1.485292829,0.616860242,0.511878408,13.13188023,1467.373791,0.583903065,0.375665461,0.298061675,Energy,7 -ett2/15T/short,Kairos_50m,8.424481214,8.431180281,1.784239681,0.768928799,0.111529115,0.132711843,8.622286122,2.902495687,0.137791799,0.084704207,0.06998294,Energy,7 -ett2/15T/medium,Kairos_50m,13.8397496,13.82079922,2.273352646,0.929703183,0.138929158,0.167057506,17.92224457,3.720181394,0.19825519,0.121151071,0.107114016,Energy,7 -ett2/15T/long,Kairos_50m,12.82068073,12.78033339,2.260576486,0.931296981,0.141632767,0.168528531,19.96997419,3.580597818,0.190816526,0.120470205,0.108143276,Energy,7 -ett2/H/short,Kairos_50m,114.2573788,114.2044238,6.57644948,0.7502733,0.122897048,0.111602583,7.906534851,10.68912432,0.135567832,0.083407674,0.068441677,Energy,7 -ett2/H/medium,Kairos_50m,231.2604804,233.4488848,9.848766636,1.045914985,0.17425909,0.167788134,19.59925391,15.20725092,0.204927468,0.132718453,0.11629653,Energy,7 -ett2/H/long,Kairos_50m,220.8514104,220.8830466,9.262856604,1.022710757,0.177085177,0.176331057,24.00075498,14.8610703,0.203510922,0.126847693,0.114722352,Energy,7 -ett2/D/short,Kairos_50m,112943.3102,109548.9818,200.7072229,1.340029417,0.437446402,0.136124482,11.61837716,336.0703946,0.192341943,0.114870032,0.093753119,Energy,7 -ett2/W/short,Kairos_50m,5338131.795,5203102.11,1609.780385,1.164997698,0.194499319,0.228787634,7.208596958,2310.439741,0.193810452,0.13503588,0.107803016,Energy,7 -hierarchical_sales/D/short,Kairos_50m,27.28709216,28.09181395,2.290182335,0.744619214,0.606006926,1.021866525,9.134164445,5.223704831,1.602574793,0.702602578,0.587669947,Sales,1 -hierarchical_sales/W/short,Kairos_50m,458.3533465,461.3533303,9.157517712,0.729670587,0.570236492,0.462583864,8.227431206,21.40918837,0.98290631,0.420426117,0.356151975,Sales,1 -hospital/M/short,Kairos_50m,3344.590542,3287.376472,18.70543868,0.773676409,0.198966637,0.178374902,7.232407981,57.83243504,0.209945169,0.067905086,0.054445767,Healthcare,1 -jena_weather/10T/short,Kairos_50m,588.5397406,647.8586392,5.198390945,0.284913144,0.349235101,0.313635662,2.676367663,24.25983802,0.150684006,0.032288524,0.02655814,Nature,21 -jena_weather/10T/medium,Kairos_50m,1511.311017,1512.724232,9.978306859,0.627679752,0.774745832,0.4849722,13.92840437,38.87558382,0.238578797,0.0612367,0.054970218,Nature,21 -jena_weather/10T/long,Kairos_50m,1546.16145,1549.417083,9.980570664,0.645286124,0.801083221,0.451684113,16.38695187,39.32125952,0.240613521,0.061072821,0.056167538,Nature,21 -jena_weather/H/short,Kairos_50m,1086.585766,1089.305577,8.460113015,0.560436148,1.449401524,0.455677833,6.979475606,32.9634004,0.202063536,0.051859952,0.044385481,Nature,21 -jena_weather/H/medium,Kairos_50m,1505.286199,1506.090058,11.22069069,0.857690675,2.54889791,0.517302779,17.41095061,38.79801797,0.237411947,0.068661395,0.061741582,Nature,21 -jena_weather/H/long,Kairos_50m,1192.895535,1201.217889,11.42615049,1.035467669,2.670123256,0.481627523,27.26896082,34.5383198,0.207958962,0.068798089,0.061903489,Nature,21 -jena_weather/D/short,Kairos_50m,430.3252873,427.8099919,10.56173814,1.122478737,0.776305575,0.463632688,9.369381566,20.74428324,0.124914489,0.063598925,0.050842244,Nature,21 -kdd_cup_2018/H/short,Kairos_50m,4423.00621,4521.612552,22.35811906,0.936497383,0.865647907,0.495410356,10.3724855,66.50568555,1.392084446,0.467995924,0.387665232,Nature,1 -kdd_cup_2018/H/medium,Kairos_50m,5101.574262,5145.10676,25.18947189,1.042339793,0.977360496,0.553608629,21.30156337,71.42530548,1.495061001,0.527261267,0.473833851,Nature,1 -kdd_cup_2018/H/long,Kairos_50m,4086.297501,4139.237693,24.23591298,1.020022283,1.039520966,0.606979002,25.08223864,63.92415428,1.499918971,0.568672453,0.52287054,Nature,1 -kdd_cup_2018/D/short,Kairos_50m,2715.186615,2834.747051,20.78310564,1.176038485,0.529615013,0.454773322,12.19857362,52.10745259,1.167262265,0.465563634,0.385130423,Nature,1 -loop_seattle/5T/short,Kairos_50m,39.50931554,40.3244091,3.461945374,0.54921742,0.093988561,0.073542008,5.598254631,6.285643606,0.10778491,0.05936472,0.048348954,Transport,1 -loop_seattle/5T/medium,Kairos_50m,77.25818295,79.11513807,5.113389333,0.811052493,0.170814678,0.11671389,13.50493768,8.789663415,0.156386173,0.090977703,0.078503912,Transport,1 -loop_seattle/5T/long,Kairos_50m,91.6434602,93.12734209,5.531505158,0.873930664,0.180681217,0.124585919,16.76076789,9.573059083,0.169253653,0.097798148,0.085987464,Transport,1 -loop_seattle/H/short,Kairos_50m,54.33541909,54.70702461,4.089501762,0.830537226,0.115226523,0.094187465,10.97928907,7.371256276,0.130478733,0.07238834,0.060876962,Transport,1 -loop_seattle/H/medium,Kairos_50m,61.44919655,61.15202069,4.565432201,0.9380787,0.123629269,0.103425326,15.93332211,7.838953792,0.138552633,0.080693504,0.06989731,Transport,1 -loop_seattle/H/long,Kairos_50m,55.99017993,56.41888305,4.337624523,0.90673465,0.132348799,0.100673378,17.1753599,7.482658614,0.133725105,0.077519145,0.068042935,Transport,1 -loop_seattle/D/short,Kairos_50m,18.52714292,18.56323141,2.889923523,0.873032537,0.053492156,0.053302621,14.72432294,4.304316777,0.07692389,0.051646794,0.044789456,Transport,1 -m4_daily/D/short,Kairos_50m,404480.1803,408458.6218,180.7881428,3.30995315,0.045401151,0.03120442,38.02919375,635.987563,0.098242502,0.027926772,0.023009329,Econ/Fin,1 -m4_hourly/H/short,Kairos_50m,955905.4409,960364.1019,210.2623817,1.019491024,0.102742231,0.095853622,10.33854687,977.7041684,0.133478215,0.028705459,0.022277235,Econ/Fin,1 -m4_monthly/M/short,Kairos_50m,1888605.507,1905221.489,566.016827,0.973541534,0.15798614,0.132389552,12.49722163,1374.265443,0.285619537,0.117637728,0.098244516,Econ/Fin,1 -m4_quarterly/Q/short,Kairos_50m,1851646.548,1851589.755,578.9015635,1.206708185,0.122731953,0.105711689,15.47908243,1360.752199,0.227765956,0.096897928,0.081090363,Econ/Fin,1 -m4_weekly/W/short,Kairos_50m,350282.7948,342247.6443,303.2645621,2.431395293,0.076217278,0.07408593,20.03375849,591.8469353,0.107825333,0.0552501,0.044314932,Econ/Fin,1 -m4_yearly/A/short,Kairos_50m,3281788.821,3281350.4,833.4983668,3.044331239,0.16515814,0.140944585,47.48120055,1811.570816,0.290493776,0.133655326,0.11400675,Econ/Fin,1 -m_dense/H/short,Kairos_50m,48021.62194,48189.77842,95.69494292,0.837500594,0.334651014,0.231534621,13.30495925,219.1383625,0.388696886,0.169738999,0.14469597,Transport,1 -m_dense/H/medium,Kairos_50m,39322.67852,39322.3823,90.65505082,0.771879837,0.329843712,0.232506273,12.67312661,198.2994668,0.345016032,0.157728341,0.134242315,Transport,1 -m_dense/H/long,Kairos_50m,40222.16097,40198.09126,93.80588077,0.802050862,0.343733928,0.237319099,14.20688609,200.5546334,0.347189736,0.162391855,0.139522713,Transport,1 -m_dense/D/short,Kairos_50m,7916.943304,7938.128376,44.29911886,0.671495166,0.123345113,0.096759587,9.663077388,88.97720666,0.154032706,0.076688328,0.064397125,Transport,1 -restaurant/D/short,Kairos_50m,140.1237592,141.2470529,7.228977495,0.692931762,0.704488942,0.398910108,6.620843457,11.83738819,0.544784031,0.332694294,0.269871618,Sales,1 -saugeen/D/short,Kairos_50m,936.8498176,1119.693293,12.52292173,2.781401227,0.286591548,0.321401168,34.67216938,30.60800251,0.991400479,0.405620413,0.332404681,Nature,1 -saugeen/W/short,Kairos_50m,902.9603745,923.9052747,14.91398416,1.209895312,0.42131126,0.399116245,15.43171781,30.04929907,0.907063077,0.450191012,0.375871821,Nature,1 -saugeen/M/short,Kairos_50m,373.6656669,376.9704953,11.9940023,0.702779869,0.341025159,0.339837744,9.544651426,19.3304337,0.580357572,0.360095907,0.306651125,Nature,1 -solar/10T/short,Kairos_50m,31.05741108,31.65870506,2.532276401,1.11778936,6.795422657,1.309459949,22.72021849,5.572917646,1.621161127,0.736638925,0.648568364,Energy,1 -solar/10T/medium,Kairos_50m,18.68836707,18.67212164,1.711743919,0.74922253,3.651725607,0.785634443,17.02272027,4.323004403,0.959291857,0.379842778,0.341880425,Energy,1 -solar/10T/long,Kairos_50m,18.87930919,18.97296966,1.786490282,0.781561492,3.356449826,0.672876267,17.71496927,4.345032703,0.939109317,0.386121298,0.346967491,Energy,1 -solar/H/short,Kairos_50m,701.370366,711.6082536,10.93042915,0.844024776,4.763235367,0.863297421,13.92895042,26.48339793,0.976186868,0.402899259,0.348278462,Energy,1 -solar/H/medium,Kairos_50m,704.5910831,714.5251452,11.13871129,0.852412238,4.885715744,0.854922327,15.23863931,26.54413463,0.956181683,0.401242378,0.348896747,Energy,1 -solar/H/long,Kairos_50m,718.9796879,723.1976859,11.98387975,0.912655434,6.906881422,1.01813983,17.12603183,26.8137966,0.929683315,0.415503005,0.364192226,Energy,1 -solar/D/short,Kairos_50m,130743.0957,139400.5272,254.9196083,0.995196489,1.123354658,0.426924223,7.349796348,361.5841475,0.522365955,0.368272021,0.30062322,Energy,1 -solar/W/short,Kairos_50m,1387663.955,1450200.784,928.6871969,1.013478148,0.205651775,0.180864472,5.228532047,1177.991492,0.240457504,0.189568267,0.144666621,Energy,1 -sz_taxi/15T/short,Kairos_50m,16.69964544,16.77801924,2.758025874,0.550346844,1.00246E+12,0.396922753,5.315816891,4.086519967,0.382107106,0.257887223,0.208488519,Transport,1 -sz_taxi/15T/medium,Kairos_50m,16.77671943,16.77064254,2.802017281,0.546180246,1.06112E+13,0.409229129,9.106818545,4.095939383,0.381195518,0.260774472,0.22627041,Transport,1 -sz_taxi/15T/long,Kairos_50m,16.08016981,16.04621383,2.748618542,0.519517046,7.78138E+12,0.402883812,10.2055382,4.010008704,0.37115924,0.254407221,0.226165568,Transport,1 -sz_taxi/H/short,Kairos_50m,7.225723914,7.241094466,1.863642401,0.567262774,1.063674053,0.298543224,5.448591703,2.688070668,0.250410372,0.173609791,0.140658008,Transport,1 -temperature_rain/D/short,Kairos_50m,168.3354179,179.2370264,5.900764702,1.381027354,34.0699801,1.429881937,24.24066362,12.97441397,1.527431557,0.694676017,0.575360018,Nature,1 -us_births/D/short,Kairos_50m,189073.1956,188141.2082,272.5702897,0.401353423,0.026126218,0.025713137,5.566167923,434.8254772,0.04076136,0.025551253,0.021406951,Healthcare,1 -us_births/W/short,Kairos_50m,3006552.911,3010076.897,1313.646484,1.194214344,0.017668725,0.017781145,11.75990854,1733.941438,0.023538085,0.017832622,0.014462162,Healthcare,1 -us_births/M/short,Kairos_50m,60816298.67,60781960,6681.496094,0.756184961,0.020494023,0.020749506,5.042984677,7798.480536,0.024222027,0.020752681,0.016539774,Healthcare,1 diff --git a/results/Kairos_50m/config.json b/results/Kairos_50m/config.json deleted file mode 100644 index b654ac2d58bcb3a1d1bcc6d5cc8edd45fbf690f7..0000000000000000000000000000000000000000 --- a/results/Kairos_50m/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "Kairos_50m", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/mldi-lab/Kairos_50m", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/kairos.ipynb", - "org": "ShanghaiTech University", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/Lag-Llama/config.json b/results/Lag-Llama/config.json index 8bedf149fa22a08727d4273b6a2a8a000a96feb5..0a1fe2a9369b4e0a96c9b2162114f1f6abf2db1e 100644 --- a/results/Lag-Llama/config.json +++ b/results/Lag-Llama/config.json @@ -2,8 +2,5 @@ "model": "Lag-Llama", "model_type": "pretrained", "model_dtype": "float32", - "model_link": "https://huggingface.co/time-series-foundation-models/Lag-Llama", - "org": "Morgan Stanley & Service Now", - "testdata_leakage": "Yes", - "replication_code_available": "No" + "model_link": "https://huggingface.co/time-series-foundation-models/Lag-Llama" } \ No newline at end of file diff --git a/results/Lingjiang/all_results.csv b/results/Lingjiang/all_results.csv deleted file mode 100644 index 525284fcd42a680f5cf08ccd41f0b5d4eb6e3c38..0000000000000000000000000000000000000000 --- a/results/Lingjiang/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -sz_taxi/15T/short,Lingjiang,17.09323329017276,17.386402977837456,2.8361871821830853,0.5671442573738626,1174271885554.2747,0.41391644502276864,4.4164362710667655,4.134396363457761,0.3865837599108391,0.26519564364706477,0.2088630352872326,Transport,1 -sz_taxi/15T/medium,Lingjiang,17.21445792434562,17.514251552483973,2.9012683835803954,0.5629482017337738,10647864730398.818,0.420859646594417,7.359556932369601,4.149030962085679,0.38613655459189805,0.27001142864910843,0.22888953052411917,Transport,1 -sz_taxi/15T/long,Lingjiang,16.0140947738604,16.15976395566239,2.7994916519208513,0.5266544729112937,7095435715620.177,0.41183234812600794,8.351346494999227,4.001761458890372,0.37039587407490804,0.2591159338283382,0.22517247657610712,Transport,1 -ett1/W/short,Lingjiang,934043.8571428572,947159.0,790.3670479910714,1.4737426040551591,0.6101730210440499,0.5210852862135302,9.820853035160889,966.4594441273039,0.38457728002962416,0.3145059126779166,0.24913932511091272,Energy,7 -jena_weather/H/short,Lingjiang,1193.8501848493304,1208.3259932056703,9.578534883962837,0.6138052619383408,1.863594383941177,0.6316858060715952,5.381475698617071,34.55213719655168,0.21180240260988445,0.05871580939741089,0.04919520614165182,Nature,21 -jena_weather/H/medium,Lingjiang,1297.6031746031747,1307.8561507936508,11.358327132936507,0.9872609969998994,3.3926952205216447,0.742350333050968,14.921345951635312,36.022259432234044,0.2204265941250247,0.06950361816090492,0.059438166565082017,Nature,21 -jena_weather/H/long,Lingjiang,1811.4849206349206,1837.1756613756613,14.870024181547619,1.7036503568567323,2.949063884971797,0.8200739450141611,36.74916733824333,42.561542742655845,0.25626764233661453,0.08953402045444432,0.0788971210232101,Nature,21 -m_dense/H/short,Lingjiang,56631.89229166666,56800.576736111114,112.90510443793403,0.9664262184532758,0.377370882363476,0.2814634009501656,9.238169080076567,237.9745622785483,0.42210761464237084,0.2002655403961221,0.16331943563923426,Transport,1 -m_dense/H/medium,Lingjiang,45420.64666666667,45423.653333333335,104.41352430555555,0.8804006605805744,0.36356739349609785,0.2614305755106678,8.759647794720259,213.12120182343818,0.37080397737034576,0.18166634653191666,0.14737409299407392,Transport,1 -m_dense/H/long,Lingjiang,49752.48987654321,49573.60592592593,111.23660493827161,0.9419306312193415,0.41018671183221617,0.2705995697473848,9.47996512382366,223.05266166657418,0.3861371511555014,0.1925670171974051,0.15693379160316637,Transport,1 -solar/W/short,Lingjiang,1960827.0218978103,1931086.401459854,1143.7363138686133,1.2745701544246,0.26682154105527556,0.22508404991847766,7.160781461501847,1400.2953338127677,0.28583528674232206,0.23346517647965948,0.1795946046722768,Energy,1 -ett2/D/short,Lingjiang,127917.14285714286,125213.21904761904,217.22093253968254,1.409814638887418,0.41028955853174603,0.14565096545073425,12.304443440561512,357.65506127712337,0.20469542797939055,0.12432127087344728,0.10254958679143258,Energy,7 -saugeen/M/short,Lingjiang,821.346912202381,812.7466517857143,18.786626906622026,1.1000416552043357,0.7276654016403925,0.5518138153362566,12.341764029571204,28.659150584104562,0.860433677181315,0.5640308990896521,0.4597692693518234,Nature,1 -us_births/D/short,Lingjiang,2826943.5733333332,2767791.5733333332,1238.2691666666667,1.8224119325817063,0.12258313496907552,0.11965751761143395,26.734568368331193,1681.3517101824154,0.1576130794474118,0.11607768640027098,0.10097962706084157,Healthcare,1 -saugeen/D/short,Lingjiang,1119.9961458333332,1123.9998958333333,15.4671484375,3.4350743055523565,0.46237904866536456,0.42182195556561747,49.05696560926704,33.46634347868517,1.0839828391469435,0.5009846231772188,0.43951157433521065,Nature,1 -bizitobs_application/10S/short,Lingjiang,9553505.848888889,9514224.071111111,1451.8597222222222,6.286112010241561,0.09175194634331597,0.0917911630043546,62.00851907684117,3090.874609053057,0.11920267909227933,0.055992426236948833,0.04477444122233195,Web/CloudOps,2 -bizitobs_application/10S/medium,Lingjiang,38072316.586666666,38252482.56,3302.0416666666665,14.387804728830615,0.2129423395792643,0.2000370956039799,158.41181211429534,6170.27686466877,0.2422886122656452,0.12966145776068874,0.11724607413195814,Web/CloudOps,2 -bizitobs_application/10S/long,Lingjiang,43412484.55111111,43363701.19111111,3864.6633333333334,19.528228877502624,0.29758646647135417,0.2568691437691168,196.7394898977901,6588.815109798658,0.25410426253827,0.14904461726584523,0.13530558714350346,Web/CloudOps,2 -us_births/W/short,Lingjiang,4952630.285714285,4914044.571428572,1619.2280970982142,1.4719038040047356,0.021840855479240417,0.021891268192348306,11.998193407821105,2225.450580380136,0.030210273413381852,0.021980862645627184,0.017531634559094456,Healthcare,1 -hierarchical_sales/W/short,Lingjiang,474.56658521749205,485.61599990068856,11.611765522067829,0.8532678474854984,0.9547392777979034,0.5492071498136553,7.406421840175275,21.784549231450534,1.0001393108314858,0.5331018348551086,0.41534321791432344,Sales,1 -ett2/15T/short,Lingjiang,7.304725138346354,7.388636634463356,1.7280328659784225,0.755450170575806,0.1109275996181732,0.13233627908806683,5.027568057151543,2.7027255018492635,0.12830799794028813,0.0820358698125295,0.06468501154439725,Energy,7 -ett2/15T/medium,Lingjiang,9.417513640873016,9.406885540674603,1.9176648530505953,0.8102434393985944,0.12510297214776597,0.15249707628553727,9.903711351508937,3.0687967741238613,0.16354173501807104,0.10219583124418731,0.0863454473480228,Energy,7 -ett2/15T/long,Lingjiang,9.139844680059523,9.112584790426586,1.9217139253162203,0.8189941522786657,0.12683853355144326,0.15527791717027067,11.138921354033528,3.0232176038220477,0.16111273320475183,0.10241161025058418,0.08812990126301982,Energy,7 -electricity/D/short,Lingjiang,3690691163.269766,3749371098.125838,6297.337263513514,1.825122512966233,0.6999188601849841,0.13103219884045708,15.254432410985517,60751.05894772342,0.9983416598539446,0.10348616543007227,0.08229310686538788,Energy,1 -solar/10T/short,Lingjiang,7.971274427427863,8.137727385657838,1.0961543851235198,0.462802415535418,1.9123008104950558,1.4585079224571553,3.6283600508181397,2.8233445463541753,0.8213106059319867,0.3188711853830772,0.2561423437975909,Energy,1 -solar/10T/medium,Lingjiang,7.30529797576587,7.445224747618751,1.1026903179044667,0.4762752678565837,1.5468000155097679,1.4105413662397628,3.5098566111266103,2.702831473800368,0.5997690350163539,0.2446913595250988,0.19642906234134852,Energy,1 -solar/10T/long,Lingjiang,5.771575383219599,5.832640472931874,0.970011247200204,0.4178728686754567,1.3346674182704028,1.398590857213463,2.934618715582179,2.402410327820707,0.5192425663451602,0.20965241597042147,0.16677663458486577,Energy,1 -bitbrains_rnd/5T/short,Lingjiang,1454676.3283585014,1494937.345909339,134.2604387692385,4.896481511820496,1.4175650688905905,0.7594272120753072,147.95764243714765,1206.0996345072415,4.935011534888143,0.5493549579558862,0.4637727868272927,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Lingjiang,2273789.872997734,2313939.890607318,170.1899805013223,12.308572831484428,1.8745901569497507,0.7930135085822246,520.4164828195159,1507.9091063448534,6.344542271617249,0.7160760028260043,0.6592322518507768,Web/CloudOps,2 -bitbrains_rnd/5T/long,Lingjiang,2914635.2722429223,2955469.1726222956,213.55429364347947,15.358232684244904,3.434647996477651,0.7831883359010346,556.0979504987027,1707.2302926796146,6.540014637915582,0.8180783883736876,0.7395778802399846,Web/CloudOps,2 -m4_yearly/A/short,Lingjiang,3936182.7263573315,3940952.357534604,926.3724553435405,3.4868375752066854,0.17393223924764972,0.15295901769111292,48.18305387001156,1983.9815337742766,0.31814063371167794,0.14854811649147528,0.1275370953666668,Econ/Fin,1 -ett2/H/short,Lingjiang,89.04362444196428,89.76956031436012,6.018736049107143,0.7099558829055521,0.11579877961710715,0.10795171174971493,4.9516478976436655,9.436292939601032,0.11967844475334054,0.07633431627743249,0.060648775917624805,Energy,7 -ett2/H/medium,Lingjiang,162.27760416666666,162.34384300595238,7.973710123697916,0.907039326082194,0.15990741471564449,0.1551258809372724,10.553095233885985,12.738822715096818,0.17166382811626876,0.10745087161792942,0.09012610337599822,Energy,7 -ett2/H/long,Lingjiang,203.03371362433862,203.1843253968254,8.661400462962963,0.9753604010915986,0.1753028890534151,0.1665938242831186,15.46816916513478,14.248989915932238,0.19512895006611786,0.11861121303415395,0.10311387342623744,Energy,7 -sz_taxi/H/short,Lingjiang,7.773222311949119,7.7306772867838545,1.952469198112814,0.5932086240553479,0.8541937934027778,0.312534350797163,5.0204712814012,2.788049912026167,0.25972405404837795,0.1818845542725299,0.14649103854196774,Transport,1 -m_dense/D/short,Lingjiang,19804.687407407408,18806.54,87.40711226851852,1.2007316328104565,0.20896806992629088,0.1764104723064206,8.72343824704361,140.72912778599678,0.24362292699974167,0.1513146344787508,0.11454439973549187,Transport,1 -solar/D/short,Lingjiang,111434.15377128954,118397.84817518249,256.7090556569343,1.001682357126927,1.0344546872623936,0.4374496389920945,5.980117795658627,333.81754563127674,0.48225268020663853,0.37085716956476394,0.2769877237352596,Energy,1 -ett1/D/short,Lingjiang,37461.76190476191,37928.276190476194,129.57366071428572,1.6221098555549,1.2298961821056547,0.4697289031690134,10.953606401513925,193.55041179176527,0.5083665767038218,0.3403295178683078,0.2699848904414602,Energy,7 -kdd_cup_2018/H/short,Lingjiang,4209.314949408824,4216.908606780375,23.61135662573143,0.984102320635578,0.9781430452791183,0.5141915555291138,8.099185907929957,64.87923357599736,1.3580398573445713,0.49422845518296776,0.39632909174029174,Nature,1 -kdd_cup_2018/H/medium,Lingjiang,4845.177616249731,4861.564490791731,26.19779684160988,1.1098394040784967,1.1810039223819655,0.5766375241276097,13.33649079388805,69.60731007767598,1.457007068819303,0.5483673358893288,0.46181373879234333,Nature,1 -kdd_cup_2018/H/long,Lingjiang,3777.00968693059,3803.853777661251,23.727969318494736,1.0484847535977073,1.1259852954507164,0.6109661612949059,17.113950961780642,61.45738106143631,1.4420385148776678,0.556754047213631,0.48712247821121907,Nature,1 -electricity/15T/short,Lingjiang,106979.23249725068,107181.7422578499,46.46277012043171,0.8784644069041138,0.13553034983860776,0.15343672736125977,8.811451409216483,327.0767990812719,0.6447328516706035,0.09158727968601092,0.07531089291314144,Energy,1 -electricity/15T/medium,Lingjiang,255490.10428068694,255875.68903265765,54.502327594928914,0.8566204719397901,0.1281595765067794,0.13442819134255826,8.821427516852031,505.46028951905504,0.8709589721345719,0.09391299812333029,0.07809937535060471,Energy,1 -electricity/15T/long,Lingjiang,334741.3973020364,335076.2592161693,60.12559493917746,0.9148428045415958,0.1344224968796469,0.13701520395166672,10.056061198522244,578.5684033042562,0.9127626100349774,0.09485549963178525,0.07962718340403302,Energy,1 -bizitobs_l2c/H/short,Lingjiang,87.81773933531746,89.68966238839286,6.0466109987289185,0.6132358461152132,0.5974742422858588,0.7479992598736441,4.581464839162259,9.371111958317298,0.5051269343858105,0.32592781846989066,0.25796571862790607,Web/CloudOps,7 -bizitobs_l2c/H/medium,Lingjiang,124.88249627976191,125.34395461309524,6.777444312686012,0.6772106649665172,0.5648209363825983,0.8958428998502589,7.0525878583762385,11.175083725850197,0.6766682935776718,0.41038454747989217,0.3411844013367673,Web/CloudOps,7 -bizitobs_l2c/H/long,Lingjiang,89.15849454365079,87.8366691468254,5.973435174851191,0.6403397953288085,0.7194376691083268,0.7969727165870376,6.905730267625824,9.44237758955078,0.5767676755497171,0.36487466087550813,0.2997970122577858,Web/CloudOps,7 -covid_deaths/D/short,Lingjiang,1105261.6408521303,740747.3333724937,137.14504082471805,46.97034764025341,0.15950974217228742,0.43316459058217893,1879.9299471650304,1051.3142445777714,0.39556208215522815,0.051601486601826514,0.054492841692399346,Healthcare,1 -saugeen/W/short,Lingjiang,1263.2958984375,1263.604296875,19.51348876953125,1.5823706309584327,0.735014820098877,0.5518413174729557,17.640407083308567,35.54287408802924,1.0728912091471896,0.5890308844684464,0.4824969935136547,Nature,1 -bizitobs_l2c/5T/short,Lingjiang,24.677030581519716,24.987733532133557,3.3545733497256323,0.3469033280871791,0.19827775474083162,0.22295488926795595,2.977660203881629,4.967598069642885,0.17060741821913902,0.1152096225177417,0.09272611009401345,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Lingjiang,129.2798469387755,131.56347789115645,8.081776147959184,0.7769746660786438,0.6045963468467317,0.8976405112532411,11.664473778347682,11.370129591995665,0.5982360215357628,0.4252202730478402,0.36594187170290926,Web/CloudOps,7 -bizitobs_l2c/5T/long,Lingjiang,223.84626984126984,231.2563888888889,10.753251488095238,1.006255620896709,0.8809193260013586,1.113815891635273,16.574954822937777,14.9614929014878,0.840065458946472,0.6037789949166943,0.5294317796593851,Web/CloudOps,7 -bitbrains_fast_storage/H/short,Lingjiang,3727547.588392926,3791171.796155395,377.3934709330137,10.801860834345261,6.207630168792159,0.6849123241399845,460.00229166739234,1930.6857818901879,5.503081569150704,1.0756939703445767,0.8207449978427294,Web/CloudOps,2 -temperature_rain/D/short,Lingjiang,176.29479995097697,174.49793604630133,6.85677124670614,1.7758821742923057,116.20552978809594,1.4046897334015713,26.967362774168745,13.277605203912977,1.5631251811050066,0.8072232629453759,0.6562921590914577,Nature,1 -car_parts/M/short,Lingjiang,1.8932626458679012,1.8726193467443393,0.6411106109112339,1.1685522892965359,0.6084963390294428,1.7623544992125955,20.072465496027167,1.375958809655253,3.299408079093689,1.5373174795568199,1.2669739090728933,Sales,1 -ett2/W/short,Lingjiang,4426755.428571428,4369387.428571428,1364.62890625,0.8382627374642126,0.1382666059902736,0.14333329986210291,12.311190433214493,2103.9856055998644,0.17649213624456936,0.11447144419815805,0.11005800035604434,Energy,7 -electricity/W/short,Lingjiang,293216462548.1802,295802462886.0541,55618.44344031531,2.05324014036354,0.27171204641973346,0.12206950498191843,22.060561717235288,541494.6560661335,1.23551211068611,0.1269029337191233,0.1028292775747762,Energy,1 -m4_hourly/H/short,Lingjiang,49552281.860504836,50028761.21694263,1165.5019900065497,3.5781276609341783,0.20328312718733882,0.18213568786364756,31.88400326014204,7039.338169210571,0.9610251650543133,0.15911676856444965,0.14132054518393555,Econ/Fin,1 -solar/H/short,Lingjiang,403.2013216921181,407.78835768424256,8.668212499824921,0.6708210896810207,3.1421810864188666,1.3852154919689164,5.949208250515579,20.07987354771235,0.7401508249717682,0.3195131990015417,0.2630590271714113,Energy,1 -solar/H/medium,Lingjiang,331.3477493917275,329.5756576946472,8.693236770072993,0.6676443209120652,2.7253151548540204,1.3853930825626415,4.587427010147563,18.20295990743614,0.6557130961698584,0.31315067589164935,0.2503951025798989,Energy,1 -solar/H/long,Lingjiang,673.3169606650446,676.609058191403,12.09902026941403,0.9209384654596641,4.7493711059971115,1.4154859835191538,8.69386646391294,25.948351790914284,0.8996767745888563,0.41949514248080394,0.34892784690362766,Energy,1 -kdd_cup_2018/D/short,Lingjiang,2703.469975979121,2773.758597438327,20.306674391374095,1.1497899485946192,0.5179411246822714,0.44369057369143095,10.851214597383303,51.99490336541766,1.1647410635822066,0.4548910758079765,0.3796386481450549,Nature,1 -jena_weather/D/short,Lingjiang,473.0492063492064,411.16262400793653,11.217914496527778,1.4728260202685992,1.3635249775274958,0.4644392777350744,46.922873965526826,21.749694396685356,0.1309687084952244,0.06755018010018583,0.056511961170245484,Nature,21 -bitbrains_rnd/H/short,Lingjiang,2202512.1140639754,2260655.0138066253,212.19918226834977,13.35019690042479,5.980002028465694,0.6937300674779288,493.5711218791996,1484.086289291824,6.49508785045146,0.9286874628324797,0.7650499489657955,Web/CloudOps,2 -hierarchical_sales/D/short,Lingjiang,38.006511076051574,37.00677162728658,2.706984823920008,0.863367504569755,0.8717738846921772,1.0006339814958891,10.424223068046993,6.164942098353526,1.8913359631596063,0.8304729659301323,0.6946249814745438,Sales,1 -jena_weather/10T/short,Lingjiang,977.3738037896535,1015.139794843159,7.531115359351748,0.37825047090228486,0.46671172549086337,0.5697162743324519,3.394748151401456,31.262978165709892,0.19418227685150435,0.04677766526140715,0.03774732755741364,Nature,21 -jena_weather/10T/medium,Lingjiang,1559.6394683441558,1587.0593795093796,10.57711024869228,0.764288819186884,1.1371565525463427,0.6901022479536222,10.098053444720671,39.49227099502073,0.24236339531671758,0.06491154567524171,0.055233552370179545,Nature,21 -jena_weather/10T/long,Lingjiang,1649.9367466517858,1682.0053664434524,11.055315548528439,0.9397411592430446,1.2492042783485764,0.7165659701217734,14.352642834267925,40.61941342082362,0.2485571589359742,0.06764937236819439,0.058566091098292535,Nature,21 -us_births/M/short,Lingjiang,290976896.0,278728234.6666667,13686.470052083334,1.5482366052848597,0.04185976584752401,0.042657789917493916,10.642255753936716,17058.044905557024,0.052982171176379204,0.04251008266859687,0.034070572629962194,Healthcare,1 -electricity/H/short,Lingjiang,1476958.854649845,1496657.4412938133,168.618973663605,0.9268181713686989,0.21461331510676238,0.13056361738499564,10.325464849490274,1215.301960275653,0.5737093134382268,0.07960019713229399,0.06595918627209568,Energy,1 -electricity/H/medium,Lingjiang,4832241.587778716,4832438.3344453825,236.6403076171875,1.09338040135783,0.2203902037909806,0.13760611188733513,11.456428467859483,2198.236017305402,0.8580439399674773,0.0923685083423678,0.0759866724604843,Energy,1 -electricity/H/long,Lingjiang,7462470.25157958,7462616.077933934,271.33462443693696,1.2508563444428233,0.31756775432533596,0.15122960311419073,15.006629317815262,2731.752231001117,1.07740227511646,0.10701429594114113,0.08966975033431074,Energy,1 -m4_daily/D/short,Lingjiang,329778.9041535706,329276.8219819866,179.72874353578928,3.3390770455173326,0.03956626303122064,0.031533893375144395,32.544817631462834,574.2637931765946,0.08870788516277912,0.02776312372721809,0.02269625702322976,Econ/Fin,1 -m4_monthly/M/short,Lingjiang,2039266.7754814816,2019260.9005138888,606.0031261664496,1.1226536765536765,0.17505683936013117,0.14117489530719476,12.328738993471791,1428.0289827176064,0.2967934456053558,0.12594825317920366,0.10535289873829468,Econ/Fin,1 -m4_weekly/W/short,Lingjiang,438319.71030640666,431951.546603814,334.4961665684594,2.754998052518397,0.08141426379046655,0.08592870751588935,31.990570079580458,662.0571805413839,0.12061654816449499,0.060940012693090316,0.05065628874458569,Econ/Fin,1 -restaurant/D/short,Lingjiang,190.70837601860933,188.50204050212477,8.80071393860325,0.8240286415384662,0.8861624333909037,0.4547895594580048,6.821462891597235,13.80972034541646,0.6355553270966182,0.4050292465038384,0.32166372383285985,Sales,1 -loop_seattle/H/short,Lingjiang,66.54251516506989,67.2236101766819,5.007142026604021,1.0090175995760635,0.13925489204269575,0.11299635656371422,9.209791822772557,8.157359570661937,0.14439355959368416,0.08863150561767132,0.07358521044869959,Transport,1 -loop_seattle/H/medium,Lingjiang,76.99957793150155,77.71184570816402,5.4727257081967755,1.1136856506759663,0.1404986118753628,0.1214428414386179,12.099502781988765,8.77494033777447,0.155096089495088,0.09672981508109388,0.08105321799367299,Transport,1 -loop_seattle/H/long,Lingjiang,70.79840334382794,71.36419579893361,5.203341549878929,1.0755114882183647,0.15542734892129653,0.11751095357153543,13.30539704902671,8.414178708812164,0.15037261467675547,0.09299066504162157,0.07834189026002741,Transport,1 -bizitobs_service/10S/short,Lingjiang,35265.94363715278,34056.36448536706,61.2626712859623,2.6223915545703065,0.4595425358525029,0.1380988889196401,22.613725021528364,187.79228854549055,0.139122534885189,0.045385346699561374,0.03639845287624151,Web/CloudOps,2 -bizitobs_service/10S/medium,Lingjiang,193462.79111111112,194218.79873015874,156.6311681547619,5.9955915470430785,0.7422196742466518,0.25725528334621645,97.64690266645428,439.8440531723842,0.3310369895041594,0.1178843046632047,0.10660746620190892,Web/CloudOps,2 -bizitobs_service/10S/long,Lingjiang,253671.4835978836,253741.34857142856,188.32271164021165,6.784637070568266,0.8733079117063492,0.299932154213709,122.38880218805424,503.6581018884573,0.373126023423768,0.13951548530883168,0.12738002349704713,Web/CloudOps,2 -m4_quarterly/Q/short,Lingjiang,1907992.1736171874,1885455.4473020833,597.3419918823242,1.317592379302041,0.12494343821704387,0.10819207996411273,14.695396455754674,1381.3008990141095,0.23120544634977488,0.09998453049237148,0.08444549378418674,Econ/Fin,1 -ett1/H/short,Lingjiang,75.08721342540923,75.77322533017113,4.548006875174386,0.7860549040833174,0.4154739879426502,0.24878916943412938,5.9106727904619065,8.665287844348232,0.40444444034314075,0.21227409041078596,0.16958609057392596,Energy,7 -ett1/H/medium,Lingjiang,112.00694754464286,111.83585379464286,6.249769810267857,1.1492538539997257,2449041071080.1504,0.42695534965027,12.927616230242444,10.5833334797994,0.506916585978865,0.29934915888474967,0.24726110557639414,Energy,7 -ett1/H/long,Lingjiang,144.31777447089948,143.885582010582,7.321917886078042,1.407624351485582,5745437891059.979,0.4952164807372616,23.525579877290475,12.013233306271026,0.5752877241803844,0.35063078939111864,0.3014529425723179,Energy,7 -ett1/15T/short,Lingjiang,5.069843056088402,5.253259386335101,1.0913109643118721,0.7011456306428773,0.5423508031230515,0.23254345565111398,5.320126779644499,2.251631198950752,0.4256520211031162,0.20630319824481128,0.16474795691969796,Energy,7 -ett1/15T/medium,Lingjiang,7.2045667472718256,7.302609592013889,1.455413566468254,0.938675437736884,0.7068375110976982,0.36035814622461165,10.192483819569587,2.6841324012186556,0.5115982593580167,0.27740324840650293,0.2299298580590507,Energy,7 -ett1/15T/long,Lingjiang,7.722519221230159,7.829583333333333,1.4931809973338295,0.9729193255919298,0.7450789178175449,0.3710287804470433,11.835645389025277,2.778942104692028,0.5296691128128234,0.2846017744635329,0.23855391412204563,Energy,7 -bitbrains_fast_storage/5T/short,Lingjiang,1954053.6534129835,1999442.6899339228,201.41602733258273,2.444770665010118,3.084083823438249,0.7988736351806309,62.558692740494266,1397.8746915989943,4.388980808138862,0.6323965114520241,0.5076265340917465,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Lingjiang,3729072.868248793,3802328.3051358406,340.5113410739228,5.6708881935007796,5.7487469436829794,0.8372298529267018,251.82308947685195,1931.0807513537059,5.867156562612315,1.034567481460983,0.8260142958672874,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Lingjiang,5361708.249168937,5447638.381534424,439.83621263937414,7.11267701991123,5.660593612997303,0.8467724288118024,252.1054136753133,2315.5362767983006,6.119271025766552,1.1623557873204784,0.9661501434349373,Web/CloudOps,2 -loop_seattle/D/short,Lingjiang,31.224680453557347,30.775288499767196,4.392894287424314,1.285936906558313,0.0818310491071766,0.08014658430309597,10.135276966701587,5.587904835764237,0.09986332280961768,0.07850688821431616,0.06165947831056865,Transport,1 -loop_seattle/5T/short,Lingjiang,43.63356840371839,44.16203817361649,3.7965655804172513,0.5980905252220723,0.10697673070787522,0.08048186692263883,5.303860601208523,6.60557101269212,0.11327095886670523,0.06510271751948467,0.05258145275089023,Transport,1 -loop_seattle/5T/medium,Lingjiang,97.04370509223426,98.38700033055986,6.10260813489664,0.9613854475761041,0.20460375692330154,0.1377187937535835,9.255888837051833,9.851076341813329,0.17527089061615334,0.10857793867099694,0.08888026521949656,Transport,1 -loop_seattle/5T/long,Lingjiang,110.97194815316477,112.31845896399496,6.511690450851393,1.0222475428434075,0.21798529059033153,0.14519023734942602,10.731233432727178,10.534322386995985,0.18624898575435347,0.11512802603369222,0.09536266556169985,Transport,1 -hospital/M/short,Lingjiang,5807.473737641243,5310.54653886897,21.607252642363374,0.8321956108572173,0.22188464334248356,0.18458302524987624,18.56413557902935,76.20678275351376,0.27664831769002246,0.0784393446020066,0.08745533368176907,Healthcare,1 \ No newline at end of file diff --git a/results/Lingjiang/config.json b/results/Lingjiang/config.json deleted file mode 100644 index a665d6a99ea5889d62a3b92306d7856f62723422..0000000000000000000000000000000000000000 --- a/results/Lingjiang/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "Lingjiang", - "model_type": "zero-shot", - "model_dtype": "bf16", - "model_link": "", - "code_link": "", - "org": "Alibaba Cloud", - "testdata_leakage": "No", - "replication_code_available": "No" -} \ No newline at end of file diff --git a/results/Migas-1.0/all_results.csv b/results/Migas-1.0/all_results.csv deleted file mode 100644 index 47227a9ee5d76ff663404fd2233af37705b2fb82..0000000000000000000000000000000000000000 --- a/results/Migas-1.0/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -,dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -0,bizitobs_l2c/H/short,Migas-1.0,67.76839095637912,67.76839095637912,5.009414623654078,0.5022733834226804,0.4793521524619141,0.6395102516408004,20.09093534697701,8.23215591181187,0.4437343228145305,0.2700202996076525,0.2700203015146806,Web/CloudOps,7 -1,ett1/15T/short,Migas-1.0,5.306348299909206,5.306348299909206,1.0909823749746594,0.6783365755142878,0.501195515902572,0.2366594876384451,27.13346301659078,2.3035512366581314,0.4354670511844889,0.2062410725509608,0.2062410708331307,Energy,7 -2,loop_seattle/5T/short,Migas-1.0,41.47205902523178,41.47205902523178,3.489898709714474,0.5550519916361527,0.0935554249529314,0.0744429245059665,22.20207966496013,6.439880357990495,0.1104297302638297,0.0598440579231682,0.0598440578596965,Transport,1 -3,m_dense/D/short,Migas-1.0,8459.816347384982,8459.816347384982,44.71176086425781,0.6598702795112841,0.1187783441692038,0.0956550211818129,26.39481119810435,91.97725994714662,0.1592262423494718,0.077402671870579,0.0774026725126156,Transport,1 -4,m_dense/H/short,Migas-1.0,43554.68589326223,43554.68589326223,85.78731758382585,0.7671199965802804,0.2884761630204521,0.2042029148919714,30.684799865618604,208.69759436385996,0.3701775636388744,0.1521653390931193,0.1521653390531793,Transport,1 -5,saugeen/D/short,Migas-1.0,1232.8392111555736,1232.8392111555736,12.916662642161052,2.8700569175506434,0.2711084647973378,0.3225906809171041,114.80227674663196,35.111810137837864,1.137279879432168,0.418373774370366,0.4183737779741687,Nature,1 -6,ett2/W/short,Migas-1.0,3125085.0909598214,3125085.0909598214,1215.8835721697126,0.8125884956884671,0.1261013919221503,0.1378701725708586,32.50353985134533,1767.7910201604209,0.1482905486509351,0.1019939800329699,0.1019939810386672,Energy,7 -7,electricity/15T/short,Migas-1.0,122828.08130247508,122828.08130247508,45.80764403239967,0.8895529982434434,0.1489005167933964,0.1481435798177564,35.58211993069854,350.46837418299964,0.6908422566532103,0.0902958968810858,0.0902958968908497,Energy,1 -8,m4_daily/D/short,Migas-1.0,356566.8243499921,356566.8243499921,168.3290862531732,3.142947306349373,0.0373566069648811,0.0292169523462131,125.71789226001334,597.1321665678313,0.0922404171071156,0.0260021918036944,0.0260021918025791,Econ/Fin,1 -9,kdd_cup_2018/H/short,Migas-1.0,4507.389056657808,4507.389056657808,22.387264040026224,0.936685942429756,0.8820281105911753,0.4930063159340872,37.46743769773799,67.13709151175532,1.4053009163269776,0.4686059815980297,0.4686059816803452,Nature,1 -10,car_parts/M/short,Migas-1.0,1.3533870743906176,1.3533870743906176,0.4569626225792569,0.8352137406325942,0.8145454849144085,1.8867476047711556,33.40854963152422,1.163351655515484,2.7895979328018634,1.0957494935183392,1.095749490568756,Sales,1 -11,bizitobs_application/10S/short,Migas-1.0,311668.51268759783,311668.51268759783,310.5035489484999,1.0148212617952204,0.0145882193578614,0.0146695295307371,40.59285047180883,558.2727941495966,0.0215303514143063,0.0119748814456093,0.0119748814451189,Web/CloudOps,2 -12,hierarchical_sales/D/short,Migas-1.0,29.101012274988825,29.101012274988825,2.299111049862231,0.7419043206916037,0.6415052106119464,1.0763982781500498,29.676172811496308,5.394535408632408,1.6549837224925863,0.7053418089787743,0.7053418103808433,Sales,1 -13,electricity/W/short,Migas-1.0,40736454739.87946,40736454739.87946,26613.067773946128,1.4180225340595478,0.130853166465629,0.092931130958033,56.72090135471426,201832.73951437973,0.4605157141876901,0.0607222393261477,0.0607222393302134,Energy,1 -14,us_births/M/short,Migas-1.0,52507404.0,52507404.0,6178.21484375,0.6977779759494185,0.0190304666757583,0.0192727943261464,27.911119037976743,7246.199279622387,0.0225066455585445,0.0191894931821508,0.0191894931821508,Healthcare,1 -15,saugeen/M/short,Migas-1.0,396.4324006580171,396.4324006580171,12.338201341174898,0.7229378014319574,0.348202455611456,0.3547525462650117,28.91751215505913,19.91061025327996,0.59777621195335,0.3704297942766609,0.370429792061077,Nature,1 -16,ett2/15T/short,Migas-1.0,7.905865891069975,7.905865891069975,1.709707535803318,0.7334786860152273,0.1052803108457417,0.1238771218030403,29.339147463409947,2.8117371660718886,0.1334831692174201,0.0811659009482601,0.0811659009844728,Energy,7 -17,us_births/D/short,Migas-1.0,129661.49307291668,129661.49307291668,218.12482666015623,0.3211106809830259,0.0209289260208606,0.0208372311790784,12.844427239321035,360.08539691706005,0.0337550840365124,0.0204474325185122,0.0204474325185122,Healthcare,1 -18,bizitobs_l2c/5T/short,Migas-1.0,19.04098795023872,19.04098795023872,2.511150352443968,0.2614861159954598,0.1384228702642644,0.2029559666568058,10.459444642526446,4.363598050948175,0.1498636150002648,0.0862430648360631,0.0862430648653068,Web/CloudOps,7 -19,bitbrains_fast_storage/H/short,Migas-1.0,2951013.797367886,2951013.797367886,312.7932485650886,0.9752025616305824,3.711985458914432,0.5485283163354276,39.00810247334038,1717.8515062041558,4.896434826057503,0.8915623673515254,0.8915623611914132,Web/CloudOps,2 -20,loop_seattle/H/short,Migas-1.0,51.49528513636648,51.49528513636648,3.821495207781432,0.7797770503794993,0.1054925616175206,0.0876833858078421,31.191082018719037,7.176021539569573,0.1270228795664434,0.0676443517992669,0.0676443518243598,Transport,1 -21,m4_hourly/H/short,Migas-1.0,2093795.6341603703,2093795.6341603703,243.2031891050834,0.6891953678612321,0.088772833260712,0.0790879686929933,27.56781472253949,1446.9953815269662,0.1975468309563048,0.0332026072090685,0.0332026071478287,Econ/Fin,1 -22,m4_yearly/A/short,Migas-1.0,3305269.588524174,3305269.588524174,849.0509265922793,3.168430451364711,0.1637160751820891,0.1389799737223793,126.73721805346798,1818.040040407299,0.2915311459772832,0.1361492509081746,0.136149250818028,Econ/Fin,1 -23,temperature_rain/D/short,Migas-1.0,173.76696928367366,173.76696928367366,5.628950057348068,1.3103254342693558,22.194486015696167,1.5009931574003892,52.41301737774159,13.182069992367422,1.551878160879882,0.662676246427151,0.6626762464345192,Nature,1 -24,solar/H/short,Migas-1.0,834.3518415441401,834.3518415441401,11.561373296613226,0.8916078050280686,5.9285421714348105,1.4095480295802405,35.6643122019761,28.88514915218788,1.064716217827055,0.4261560701799031,0.4261560695062678,Energy,1 -25,electricity/D/short,Migas-1.0,1214167725.408834,1214167725.408834,4109.197000560313,1.3579297486535695,0.2646136470400649,0.0913194949020136,54.31718995528783,34844.909605404835,0.5726175882261352,0.067527753771011,0.0675277539026973,Energy,1 -26,hierarchical_sales/W/short,Migas-1.0,464.9657967711909,464.9657967711909,8.889073700344158,0.7141019539240872,0.5150542356477084,0.4551623651158001,28.564078117080463,21.56306556988572,0.9899708875933284,0.4081017215521483,0.4081017220724851,Sales,1 -27,electricity/H/short,Migas-1.0,1595674.6365738146,1595674.6365738146,162.26891320532036,0.8805457965733049,0.1554048549841485,0.1152273846604687,35.22183185693411,1263.2001569718927,0.5963206879848646,0.0766025157827034,0.0766025160715647,Energy,1 -28,ett1/W/short,Migas-1.0,1216463.9529854911,1216463.9529854911,886.0552673339844,1.603269880916382,0.6305486740810531,0.5997300105435508,64.13079548238782,1102.9342469002816,0.4388838690131237,0.3525825451334957,0.3525825409048953,Energy,7 -29,hospital/M/short,Migas-1.0,3196.9933659169324,3196.9933659169324,18.33753624070162,0.7642296454637582,0.1935917449138025,0.1740902049199025,30.56918581035713,56.54196110780853,0.2052604486658856,0.0665695147894957,0.0665695147578993,Healthcare,1 -30,bizitobs_service/10S/short,Migas-1.0,1935.9845940465264,1935.9845940465264,17.680996731073453,0.7270076939585101,0.0588525227331137,0.0524567128545432,29.08030774393231,43.99982493199861,0.0325964776723826,0.0130986479164549,0.0130986478990794,Web/CloudOps,2 -31,jena_weather/H/short,Migas-1.0,1075.4824023834856,1075.4824023834856,8.22083675634942,0.522421859980838,1.3181338798162827,0.609640715548161,20.89687438609983,32.7945483637675,0.2010284833703333,0.0503932033712617,0.0503932036393799,Nature,21 -32,m4_monthly/M/short,Migas-1.0,1834118.996764256,1834118.996764256,551.490991030225,0.9144070732837974,0.1573834074086815,0.1300209767893713,36.57628293315568,1354.2964951458216,0.2814693043020879,0.1146187604638469,0.1146187604371496,Econ/Fin,1 -33,ett1/D/short,Migas-1.0,39554.89821661086,39554.89821661086,135.39450165279328,1.7138136837306872,1.070365625714499,0.4920332276631915,68.55254745946515,198.88413264162344,0.5223757911101468,0.3556181630652616,0.3556181649816762,Energy,7 -34,ett2/H/short,Migas-1.0,105.68491889124824,105.68491889124824,6.333841402757735,0.7302320710630512,0.1191886888564583,0.1089032747028839,29.20928285194677,10.28031706180545,0.1303830182495259,0.0803307285408113,0.0803307281691345,Energy,7 -35,bitbrains_fast_storage/5T/short,Migas-1.0,2141993.1398959034,2141993.1398959034,173.39397331723637,0.6664024367536718,2.2949611125996694,0.7385465810614297,26.656097451470384,1463.554966475774,4.595200643527671,0.5444141942204079,0.5444141943897776,Web/CloudOps,2 -36,ett1/H/short,Migas-1.0,91.91580637324424,91.91580637324424,4.789374753974733,0.7964461149388368,0.4452036355400369,0.2511551908439114,31.8578446134522,9.587273145855615,0.447477260923512,0.2235397139353714,0.2235397133599294,Energy,7 -37,kdd_cup_2018/D/short,Migas-1.0,2891.8936432065207,2891.8936432065207,21.20372079339653,1.200569405635661,0.5337890400770172,0.4668241714621107,48.022776191827646,53.776329766975735,1.2046468856675427,0.4749858595596329,0.4749858607139228,Nature,1 -38,solar/W/short,Migas-1.0,1620841.3524749088,1620841.3524749088,1018.8470797573564,1.1099454609971078,0.2284319842383809,0.1987499896518505,44.39781846152969,1273.12267770035,0.2598761566234685,0.2079721521830476,0.2079721521716801,Energy,1 -39,covid_deaths/D/short,Migas-1.0,284543.2041749239,284543.2041749239,89.8512155269184,32.357190042209034,0.0849226800040846,0.1225092299132897,1294.2876029137644,533.4259125454292,0.200704086081099,0.0338069555143146,0.0338069552012592,Healthcare,1 -40,bitbrains_rnd/5T/short,Migas-1.0,1635458.22427014,1635458.22427014,120.70988489785196,1.623902795988607,1.089419817858844,0.657004913223982,64.95611183529356,1278.850352570675,5.232686483999105,0.4939100043413175,0.4939100028560019,Web/CloudOps,2 -41,solar/10T/short,Migas-1.0,22.1851231478387,22.1851231478387,1.9872746427246617,0.8725932711089113,5.418489149182527,1.5154800592674682,34.903730838613185,4.710108613167928,1.3701700747043477,0.5780979738063264,0.5780979720120226,Energy,1 -42,saugeen/W/short,Migas-1.0,1008.759186410904,1008.759186410904,14.896779882907868,1.208453940795026,0.3517409026622772,0.3932497665286064,48.33815767271116,31.760969544566866,0.9587312729934196,0.449671686520859,0.4496716902992121,Nature,1 -43,ett2/D/short,Migas-1.0,105638.46390438988,105638.46390438988,193.10420948815724,1.2672659048952086,0.423116128690659,0.1321143412400805,50.69063618000478,325.0207130390152,0.1860179186451422,0.1105186275506405,0.1105186283684935,Energy,7 -44,m4_weekly/W/short,Migas-1.0,227638.43862793903,227638.43862793903,244.76915367425624,1.956377051683399,0.0596917244334924,0.05964945607308,78.25508208805994,477.114701752041,0.0869228995528345,0.044593143913467,0.044593143832756,Econ/Fin,1 -45,jena_weather/D/short,Migas-1.0,375.9053472787464,375.9053472787464,9.541148790950697,0.9913058738955652,0.5702506750317797,0.44802408471467,39.6522349103001,19.388278605351907,0.1167491244044441,0.0574533092818528,0.0574533090409228,Nature,21 -46,bitbrains_rnd/H/short,Migas-1.0,2075609.7725232616,2075609.7725232616,185.6195064037285,5.80491717235226,2.716465395795177,0.5459502204616469,232.19669000958132,1440.6976686741957,6.305197948182749,0.8123617857982075,0.8123617838595786,Web/CloudOps,2 -47,sz_taxi/15T/short,Migas-1.0,16.61627431385039,16.61627431385039,2.719343078801447,0.5426504688154726,1071727247042.426,0.3946893663840398,21.706018757322443,4.076306454849831,0.3811520991741947,0.2542702160252995,0.2542702155392077,Transport,1 -48,jena_weather/10T/short,Migas-1.0,986.8081050001985,986.8081050001985,6.286482201555352,0.2778234302990672,0.3410393516834363,0.5496226620391502,11.112937215482209,31.41350195378093,0.1951172267980626,0.0390469351455171,0.0390469352306203,Nature,21 -49,restaurant/D/short,Migas-1.0,137.35283071331264,137.35283071331264,7.055709096194538,0.6771059178639187,0.6781232262304587,0.3903189632692646,27.08423672972001,11.719762400036643,0.5393706200663334,0.324720080520593,0.3247200817641926,Sales,1 -50,m4_quarterly/Q/short,Migas-1.0,1779758.9104318037,1779758.9104318037,555.4432852094968,1.1669607452761237,0.1128954662002373,0.1012704342420778,46.678429810833805,1334.0760512173972,0.2233008384818926,0.0929714248323658,0.092971424797221,Econ/Fin,1 -51,loop_seattle/D/short,Migas-1.0,17.421374277286116,17.421374277286116,2.8663807879783545,0.8679986061902566,0.0537240504649524,0.0528546621986463,34.71994425307893,4.173891981985892,0.0745930251725961,0.0512260535717519,0.0512260537089437,Transport,1 -52,us_births/W/short,Migas-1.0,1637938.6517857143,1637938.6517857143,979.7848074776786,0.8912715134852082,0.0133232812929366,0.0133735231523002,35.65086053940833,1279.819773165626,0.0173734279287305,0.0133004826892729,0.0133004826892729,Healthcare,1 -53,sz_taxi/H/short,Migas-1.0,7.198654377212127,7.198654377212127,1.8420700711699636,0.5607765806687299,1.1167570382802403,0.2959272311045191,22.43106322352115,2.6830308192810843,0.2499408789749059,0.1716001953510747,0.171600195617653,Transport,1 -54,solar/D/short,Migas-1.0,124632.77216012774,124632.77216012774,247.015595076438,0.9628569502983718,1.068637354356529,0.4208773470853077,38.5142779908695,353.0336700091476,0.5100134272140122,0.3568534134916323,0.3568534121762478,Energy,1 -55,ett2/15T/medium,Migas-1.0,11.509069886586024,11.509069886586024,2.0905353636968704,0.8580587100503677,0.1285027873931954,0.1559533062813774,34.32234840004277,3.3925020098131147,0.180792563385955,0.1114084077646773,0.1114084079609359,Energy,7 -56,ett2/15T/long,Migas-1.0,11.463524088481115,11.463524088481115,2.1162549809047158,0.8723057694577218,0.1323912177774623,0.1603278015530298,34.8922307873619,3.385782640466029,0.1804344735109531,0.1127790510621544,0.1127790502743471,Energy,7 -57,loop_seattle/H/medium,Migas-1.0,52.97706385908728,52.97706385908728,4.040250850437349,0.8340629171915586,0.1115759540083976,0.0917510105022089,33.36251668753303,7.278534458191928,0.1286472836772297,0.0714109825637204,0.0714109825175344,Transport,1 -58,loop_seattle/H/long,Migas-1.0,52.39794758369228,52.39794758369228,3.9661828832688673,0.8331927520076586,0.1190985104657958,0.0917662197939915,33.3277100788355,7.238642661693715,0.1293642141129145,0.0708809863536098,0.0708809864220545,Transport,1 -59,bizitobs_l2c/5T/medium,Migas-1.0,87.05864830860476,87.05864830860476,6.193035409888442,0.623511045616352,0.4224511253947698,0.6801275300330856,24.940441822209014,9.330522402770638,0.4909227036019123,0.3258447443437908,0.3258447413874479,Web/CloudOps,7 -60,bizitobs_l2c/5T/long,Migas-1.0,84.60085026090107,84.60085026090107,6.021703187246171,0.6064923854746656,0.4510890628805291,0.7598989462474036,24.25969542204889,9.197872050692,0.5164467664885384,0.3381096326049718,0.3381096301696746,Web/CloudOps,7 -61,sz_taxi/15T/medium,Migas-1.0,16.335364486009645,16.335364486009645,2.74319380776495,0.5354055486713524,9841071528236.752,0.4046837877004574,21.416221946327365,4.0417031664893015,0.3761479327989113,0.2552999657700842,0.2552999648901282,Transport,1 -62,sz_taxi/15T/long,Migas-1.0,15.653788915454832,15.653788915454832,2.6988208292556286,0.5088091922462166,6909949951098.135,0.400426020309796,20.352367692171264,3.956486941145495,0.3662053613010493,0.2497980333477892,0.2497980332784227,Transport,1 -63,ett1/15T/medium,Migas-1.0,9.514118940167958,9.514118940167958,1.5471230984491016,0.9716578459644328,0.8107757181194033,0.366317107904525,38.86631383253582,3.084496545656674,0.5879080577893083,0.2948831754248394,0.2948831745934117,Energy,7 -64,ett1/15T/long,Migas-1.0,9.218223182322488,9.218223182322488,1.5303059870099265,0.9779601276501768,0.8266003531213665,0.3680910936234489,39.1184051019901,3.03615269417111,0.5786936751612096,0.2916778189200215,0.2916778213258872,Energy,7 -65,m_dense/H/medium,Migas-1.0,37154.66851888021,37154.66851888021,79.37441232893202,0.6911595427883805,0.2734789884544799,0.1921849634912278,27.64638171415161,192.75546300657783,0.3353701654771137,0.1381014544656209,0.1381014536495813,Transport,1 -66,m_dense/H/long,Migas-1.0,34604.21305531443,34604.21305531443,78.16825997811776,0.6788404132320769,0.271994361191247,0.19035823215673,27.153616525814567,186.0220767955095,0.3220317307657802,0.1353208204388265,0.1353208198928278,Transport,1 -67,kdd_cup_2018/H/medium,Migas-1.0,5203.671635191357,5203.671635191357,26.6779962851947,1.0942553523790686,1.136272247528093,0.5745578983881852,43.77021409189434,72.13647922647291,1.5099471550820396,0.5584187782113987,0.558418777961172,Nature,1 -68,kdd_cup_2018/H/long,Migas-1.0,4145.881406076919,4145.881406076919,25.064391914486716,1.0562447324083994,1.142778040442103,0.6308476903119091,42.24978929527145,64.38851921015826,1.5108148483738553,0.5881119170681895,0.5881119174516758,Nature,1 -69,solar/H/medium,Migas-1.0,894.7889034670345,894.7889034670345,13.54190968330179,1.033087514768356,3.695045712912529,1.444535513119035,41.32350059194646,29.91302230579576,1.0775368803581895,0.4878111935686999,0.4878111960263955,Energy,1 -70,solar/H/long,Migas-1.0,840.1633811239102,840.1633811239102,12.70827762080902,0.967848005377142,4.527621281633469,1.4192543983280608,38.71392021896654,28.985571947503644,1.0049827344364264,0.440619202424875,0.4406192008030075,Energy,1 -71,loop_seattle/5T/medium,Migas-1.0,107.42258194969658,107.42258194969658,6.096134604718786,0.9581653276076684,0.2205349013024808,0.1331818555152453,38.32661310445517,10.364486574341084,0.1844055135124861,0.1084627611952763,0.1084627612126597,Transport,1 -72,loop_seattle/5T/long,Migas-1.0,107.77953447966328,107.77953447966328,6.230377596223208,0.9778064301630996,0.2193870149614504,0.1363740683050279,39.11225720585818,10.38169227436757,0.183550453826505,0.110154356830705,0.1101543568407007,Transport,1 -73,jena_weather/10T/medium,Migas-1.0,1507.6620965502464,1507.6620965502464,9.691627323105031,0.6040823324290929,0.7402687138682937,0.6501803237756696,24.163293300698275,38.82862470588221,0.238290609718815,0.0594773521205958,0.0594773526127156,Nature,21 -74,jena_weather/10T/long,Migas-1.0,1416.1354973752009,1416.1354973752009,9.844957298218894,0.6690524610298093,0.8186941759442563,0.6439758713953394,26.762098443244263,37.63157580244549,0.2302740568190083,0.0602429796767491,0.0602429798315487,Nature,21 -75,electricity/H/medium,Migas-1.0,5130942.793905815,5130942.793905815,232.6547902980984,1.0392075198115247,0.2299269504070957,0.1212892065213382,41.5683007941759,2265.15844785874,0.8841659757001467,0.0908128302722862,0.0908128302498635,Energy,1 -76,electricity/H/long,Migas-1.0,6905445.22063516,6905445.22063516,261.13082473875164,1.1539008302476184,0.2856901378396458,0.1324908817220043,46.15603321153579,2627.821382939708,1.036412004558257,0.1029899228602364,0.1029899224675225,Energy,1 -77,bizitobs_application/10S/medium,Migas-1.0,2690137.900464554,2690137.900464554,913.0550760142008,2.0022628770998026,0.0284663490454355,0.0289972422520319,80.09051508399207,1640.1639858454866,0.0644044139766699,0.0358529864127001,0.0358529852306545,Web/CloudOps,2 -78,bizitobs_application/10S/long,Migas-1.0,6038219.094881727,6038219.094881727,1497.670241631402,3.075290185405175,0.0446167394849989,0.0456595717536078,123.011607416207,2457.2787987694287,0.0947674286682864,0.0577591593853721,0.0577591599721262,Web/CloudOps,2 -79,bizitobs_l2c/H/medium,Migas-1.0,73.66053873697916,73.66053873697916,5.051676432291667,0.5087521655211131,0.5658799612100607,0.8073558989025298,20.350086604612365,8.582571802028758,0.5196877713664674,0.3058866884331329,0.3058866826837325,Web/CloudOps,7 -80,bizitobs_l2c/H/long,Migas-1.0,101.72183334108382,101.72183334108382,5.81146993485708,0.6160174705019227,0.7091944789241765,0.8160629575214688,24.64069881205705,10.085724234832313,0.6160651709764573,0.3549813712611184,0.354981377366632,Web/CloudOps,7 -81,bitbrains_rnd/5T/medium,Migas-1.0,2051063.219697872,2051063.219697872,139.14773800040834,4.351451754995677,0.4666779361587485,0.7369215944419996,174.05807036261194,1432.1533506220178,6.025799200913054,0.5854654657530562,0.5854654645397397,Web/CloudOps,2 -82,bitbrains_rnd/5T/long,Migas-1.0,2211673.325289293,2211673.325289293,152.1655749578905,3.2745940395356627,0.9458300937258992,0.7086650462719241,130.98376165394694,1487.1695684384122,5.697011600903938,0.582912039212575,0.5829120421819396,Web/CloudOps,2 -83,bitbrains_fast_storage/5T/medium,Migas-1.0,3451995.487205172,3451995.487205172,258.7699905768139,0.957507713367123,2.501966424623635,0.8148775972377993,38.30030858915207,1857.95465154701,5.644979372101459,0.7862146999705947,0.7862146996060186,Web/CloudOps,2 -84,bitbrains_fast_storage/5T/long,Migas-1.0,5037535.070207264,5037535.070207264,360.8481000479623,0.8679540125506047,3.311077517093307,0.827821601901677,34.71816050904581,2244.4453814266158,5.931399007422117,0.9536137881395894,0.9536137853105444,Web/CloudOps,2 -85,jena_weather/H/medium,Migas-1.0,1110.98398648155,1110.98398648155,9.538080062545715,0.7646243713194735,1.843682585163896,0.682453987640994,30.584974860282934,33.33142640934453,0.2039609046111737,0.0583652020744371,0.0583652026360851,Nature,21 -86,jena_weather/H/long,Migas-1.0,1159.714754385035,1159.714754385035,10.639683098815103,0.9135132358137096,1.7634913527898304,0.663015451885405,36.54052943112237,34.05458492457418,0.2050463420643673,0.0640626836288884,0.0640626831514936,Nature,21 -87,electricity/15T/medium,Migas-1.0,246472.83138909825,246472.83138909825,50.47550105704568,0.7946476059106433,0.1196695627697146,0.120811463839626,31.785904235667218,496.4603019266478,0.8554510874851036,0.0869743705650507,0.0869743705384092,Energy,1 -88,electricity/15T/long,Migas-1.0,309269.9492847028,309269.9492847028,56.06374499633595,0.8442964603654705,0.121411040569716,0.1227628812822043,33.771858414301235,556.1204449439912,0.8773481891103969,0.0884474318368861,0.0884474315446764,Energy,1 -89,bizitobs_service/10S/medium,Migas-1.0,9536.99710428784,9536.99710428784,31.68423638332931,0.9710657290744256,0.0682710242366033,0.0654162736106959,38.84262914276357,97.65755016529872,0.0734993691250821,0.0238463015040098,0.0238463016459156,Web/CloudOps,2 -90,bizitobs_service/10S/long,Migas-1.0,130393.74902528874,130393.74902528874,76.21497271261518,1.3126320345493696,0.0859763518275407,0.0886442199333635,52.50528138051342,361.1007463649012,0.2675149834386251,0.0564624896742453,0.0564624918316794,Web/CloudOps,2 -91,solar/10T/medium,Migas-1.0,19.28303655913593,19.28303655913593,1.7907610159905394,0.7828363883609444,3.585723994205194,1.445882866910677,31.313455535759758,4.391245445102782,0.9744348153027796,0.3973769860243638,0.3973769848451963,Energy,1 -92,solar/10T/long,Migas-1.0,19.040676687736383,19.040676687736383,1.8154775176508489,0.7943141717656826,3.490084853583276,1.4475720922314346,31.772566873193316,4.363562384994213,0.9431142110047708,0.392386425491302,0.3923864246930272,Energy,1 -93,ett1/H/medium,Migas-1.0,149.8045224780128,149.8045224780128,7.174136273066202,1.3090815425249192,3302090822118.268,0.4454975721381959,52.36326173214562,12.239465775842213,0.5862414244003958,0.3436241372451138,0.3436241385930232,Energy,7 -94,ett1/H/long,Migas-1.0,149.20867934857728,149.20867934857728,7.4953584015054044,1.4127199451045032,3141416336823.4604,0.4870810412856006,56.50879779346757,12.215100464121337,0.5849546933958798,0.3589364728127277,0.3589364784135151,Energy,7 -95,ett2/H/medium,Migas-1.0,241.2125319526309,241.2125319526309,9.583782277788435,1.0325837120117285,0.1773123444939934,0.1669127471390224,41.30334850580631,15.531018381053798,0.2092904428900654,0.1291476185442648,0.1291476196905136,Energy,7 -96,ett2/H/long,Migas-1.0,226.25396996553613,226.25396996553613,9.114234197707404,1.0151421918383703,0.1826299412288919,0.1710562968380236,40.605687660194185,15.041740922032137,0.2059850672257801,0.1248124238848163,0.1248124245982704,Energy,7 diff --git a/results/Migas-1.0/config.json b/results/Migas-1.0/config.json deleted file mode 100644 index a92fd1c1b00541393f8f00b27c2274a6faca6f98..0000000000000000000000000000000000000000 --- a/results/Migas-1.0/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "model": "Migas-1.0", - "model_type": "pretrained", - "model_dtype": "float32", - "model_link": "To your HF model link", - "org": "Synthefy", - "replication_code_available": "No", - "testdata_leakage": "No" -} diff --git a/results/Moirai2/all_results.csv b/results/Moirai2/all_results.csv deleted file mode 100644 index 9464cb03c82869e4b034430bed97ce79ecb59b07..0000000000000000000000000000000000000000 --- a/results/Moirai2/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,Moirai2,3594920.75,3594920.75,890.2982177734375,3.3204338550567627,0.1615607738494873,0.14871330559253693,31.004405975341797,1896.027587890625,0.30403682589530945,0.14276345074176788,0.11582298576831818,Econ/Fin,1 -m4_quarterly/Q/short,Moirai2,1823365.0,1823365.0,563.21240234375,1.1834304332733154,0.11163925379514694,0.10256534814834595,9.126250267028809,1350.3203125,0.22601984441280365,0.09427183121442795,0.0754113420844078,Econ/Fin,1 -m4_monthly/M/short,Moirai2,1941434.375,1941434.375,570.535400390625,0.9396260380744934,0.1640520691871643,0.1344050168991089,7.7516584396362305,1393.3536376953125,0.28958672285079956,0.11857683956623077,0.0947224348783493,Econ/Fin,1 -m4_weekly/W/short,Moirai2,320634.75,320634.75,278.2564697265625,2.12347149848938,0.0672045573592186,0.06834465265274048,14.761430740356445,566.2462158203125,0.1031612828373909,0.050694018602371216,0.04001309722661972,Econ/Fin,1 -m4_daily/D/short,Moirai2,370240.09375,370240.09375,162.80653381347656,3.0710153579711914,0.0357334166765213,0.028526095673441887,25.159442901611328,608.4735717773438,0.09399235248565674,0.025149110704660416,0.02005612850189209,Econ/Fin,1 -m4_hourly/H/short,Moirai2,1835445.0,1835445.0,234.19720458984375,0.8085440993309021,0.09139207750558853,0.0857313722372055,6.641612529754639,1354.7860107421875,0.18495818972587585,0.03197309002280235,0.023612583056092262,Econ/Fin,1 -electricity/15T/short,Moirai2,126713.3984375,126713.3984375,46.910743713378906,0.8698201179504395,0.15082401037216187,0.15161579847335815,9.704551696777344,355.96826171875,0.7016835808753967,0.09247032552957535,0.07707162946462631,Energy,1 -electricity/H/short,Moirai2,1653415.0,1653415.0,166.158447265625,0.8867659568786621,0.17141255736351013,0.1225140392780304,10.239829063415527,1285.851806640625,0.6070138812065125,0.0784386545419693,0.06450968980789185,Energy,1 -electricity/D/short,Moirai2,1252630400.0,1252630400.0,4144.4931640625,1.4215503931045532,0.3179091513156891,0.09655529260635376,11.699943542480469,35392.51953125,0.5816166400909424,0.06810778379440308,0.054104726761579514,Energy,1 -electricity/W/short,Moirai2,119910146048.0,119910146048.0,38409.51171875,1.6595454216003418,0.2552073895931244,0.10663959383964539,12.825586318969727,346280.4375,0.790097713470459,0.08763783425092697,0.06640905886888504,Energy,1 -solar/10T/short,Moirai2,16.64216423034668,16.64216423034668,1.5266270637512207,0.6575393676757812,3.457381010055542,1.4877198934555054,4.72634220123291,4.07948112487793,1.186720609664917,0.444095641374588,0.35037222504615784,Energy,1 -solar/H/short,Moirai2,802.4385986328125,802.4385986328125,11.403338432312012,0.8793642520904541,5.135405540466309,1.4044291973114014,7.138167858123779,28.327346801757812,1.044155478477478,0.42033085227012634,0.34237420558929443,Energy,1 -solar/D/short,Moirai2,170813.640625,170813.640625,271.90545654296875,1.0638502836227417,1.260561227798462,0.43576809763908386,5.254066467285156,413.2960510253906,0.5970720648765564,0.39281076192855835,0.302970290184021,Energy,1 -solar/W/short,Moirai2,1808328.625,1808328.625,1091.556396484375,1.1966747045516968,0.24777232110500336,0.2126481980085373,6.803486347198486,1344.7410888671875,0.27449527382850647,0.2228139340877533,0.16269271075725555,Energy,1 -hospital/M/short,Moirai2,2882.404541015625,2882.404541015625,17.830530166625977,0.7649638056755066,0.19130390882492065,0.17565345764160156,5.211375713348389,53.68803024291992,0.1949000209569931,0.06472896039485931,0.051829639822244644,Healthcare,1 -covid_deaths/D/short,Moirai2,252106.03125,252106.03125,90.99989318847656,36.958065032958984,0.09614738076925278,0.37670475244522095,903.7923583984375,502.10162353515625,0.18891817331314087,0.03423915058374405,0.028077920898795128,Healthcare,1 -us_births/D/short,Moirai2,190028.15625,190028.15625,253.57337951660156,0.3733060956001282,0.024424130097031593,0.024021783843636513,3.786815881729126,435.92218017578125,0.04086416959762573,0.023770447820425034,0.019569363445043564,Healthcare,1 -us_births/M/short,Moirai2,78014528.0,78014528.0,7187.57177734375,0.811284065246582,0.022174924612045288,0.022547760978341103,4.347311973571777,8832.5830078125,0.02743394300341606,0.02232454903423786,0.016870463266968727,Healthcare,1 -us_births/W/short,Moirai2,1552936.875,1552936.875,957.8921508789062,0.8713688850402832,0.013035018928349018,0.01307821087539196,8.680670738220215,1246.1688232421875,0.01691661961376667,0.013003292493522167,0.010818051174283028,Healthcare,1 -saugeen/D/short,Moirai2,1027.0377197265625,1027.0377197265625,12.161438941955566,2.7013986110687256,0.27170529961586,0.31519150733947754,28.517648696899414,32.047428131103516,1.038023829460144,0.3939118981361389,0.32843467593193054,Nature,1 -saugeen/M/short,Moirai2,395.2047119140625,395.2047119140625,12.477724075317383,0.7310303449630737,0.34958547353744507,0.36250320076942444,4.56439733505249,19.879756927490234,0.5968499183654785,0.3746187090873718,0.291035532951355,Nature,1 -saugeen/W/short,Moirai2,1292.568603515625,1292.568603515625,17.306808471679688,1.4041545391082764,0.43147262930870056,0.4447954297065735,11.917930603027344,35.952308654785156,1.0852503776550293,0.5224204063415527,0.41932234168052673,Nature,1 -temperature_rain/D/short,Moirai2,182.9368438720703,182.9368438720703,5.879575729370117,1.3483240604400635,26.145727157592773,1.5040239095687866,20.553316116333008,13.52541446685791,1.5922988653182983,0.6921814680099487,0.5607564449310303,Nature,1 -kdd_cup_2018/H/short,Moirai2,5302.97900390625,5302.97900390625,25.09421157836914,1.006497859954834,0.7897746562957764,0.5716216564178467,8.732624053955078,72.8215560913086,1.5242869853973389,0.5252673029899597,0.4258558750152588,Nature,1 -kdd_cup_2018/D/short,Moirai2,3038.3671875,3038.3671875,22.03321647644043,1.241451621055603,0.5251110196113586,0.49087435007095337,9.494770050048828,55.12138366699219,1.234777569770813,0.49356743693351746,0.38876861333847046,Nature,1 -car_parts/M/short,Moirai2,1.3793753385543823,1.3793753385543823,0.452247679233551,0.8265860080718994,0.84610915184021,1.9076740741729736,15.108853340148926,1.1744681596755981,2.816254138946533,1.0844435691833496,0.9362376928329468,Sales,1 -restaurant/D/short,Moirai2,145.2991180419922,145.2991180419922,7.219489574432373,0.6959659457206726,0.5734684467315674,0.4184328019618988,4.903660774230957,12.054008483886719,0.5547534227371216,0.33225762844085693,0.26036787033081055,Sales,1 -hierarchical_sales/D/short,Moirai2,29.79330825805664,29.79330825805664,2.318131446838379,0.7479611039161682,0.6207244992256165,1.0917775630950928,7.330531597137451,5.458324432373047,1.6745535135269165,0.7111770510673523,0.5769974589347839,Sales,1 -hierarchical_sales/W/short,Moirai2,479.5635070800781,479.5635070800781,9.145384788513184,0.7380942106246948,0.5495446920394897,0.4673403799533844,6.649598598480225,21.898937225341797,1.005391001701355,0.4198690950870514,0.35205039381980896,Sales,1 -loop_seattle/5T/short,Moirai2,39.24714279174805,39.24714279174805,3.421999216079712,0.5417709946632385,0.08640541136264801,0.0729915127158165,3.9730300903320312,6.264753818511963,0.1074267029762268,0.0586797334253788,0.04626261815428734,Transport,1 -loop_seattle/H/short,Moirai2,61.74736022949219,61.74736022949219,4.191839694976807,0.8503301739692688,0.11133775115013123,0.0967826396226883,9.166641235351562,7.8579487800598145,0.13909368216991425,0.07419982552528381,0.06182778254151344,Transport,1 -loop_seattle/D/short,Moirai2,18.762725830078125,18.762725830078125,2.959958553314209,0.8935706615447998,0.054861217737197876,0.05470937117934227,7.584428787231445,4.3315958976745605,0.07741141319274902,0.05289841443300247,0.04312709718942642,Transport,1 -sz_taxi/15T/short,Moirai2,16.780986785888672,16.780986785888672,2.739116907119751,0.5463942885398865,1337784270848.0,0.39846450090408325,3.9365270137786865,4.096460342407227,0.3830365538597107,0.25611916184425354,0.20139019191265106,Transport,1 -sz_taxi/H/short,Moirai2,7.243575096130371,7.243575096130371,1.8513975143432617,0.5632746815681458,1.0763623714447021,0.2980678081512451,4.026472091674805,2.6913890838623047,0.25071948766708374,0.1724691092967987,0.13598862290382385,Transport,1 -m_dense/H/short,Moirai2,46547.97265625,46547.97265625,89.18578338623047,0.7929656505584717,0.2878399193286896,0.22047041356563568,9.276103973388672,215.74978637695312,0.3826863765716553,0.15819334983825684,0.1318719983100891,Transport,1 -m_dense/D/short,Moirai2,9495.8916015625,9495.8916015625,49.772308349609375,0.6916432976722717,0.1258518397808075,0.10052505135536194,7.0636701583862305,97.44686889648438,0.16869494318962097,0.086163230240345,0.07009396702051163,Transport,1 -ett1/15T/short,Moirai2,5.021040439605713,5.021040439605713,1.079026699066162,0.6870396733283997,0.508618950843811,0.2342115193605423,5.1140007972717285,2.24076771736145,0.42359837889671326,0.203980952501297,0.1601320505142212,Energy,7 -ett1/H/short,Moirai2,110.48771667480469,110.48771667480469,5.065814971923828,0.8362465500831604,0.5074601173400879,0.2602593004703522,5.890653133392334,10.511313438415527,0.49060600996017456,0.23644231259822845,0.1851363480091095,Energy,7 -ett1/D/short,Moirai2,45395.75390625,45395.75390625,142.25717163085938,1.7477439641952515,1.1709531545639038,0.4903564453125,11.691117286682129,213.0627899169922,0.5596165060997009,0.373643159866333,0.28650546073913574,Energy,7 -ett1/W/short,Moirai2,1070556.875,1070556.875,834.5147094726562,1.4887369871139526,0.6270838379859924,0.5270519852638245,8.900157928466797,1034.67724609375,0.41172274947166443,0.3320733308792114,0.2488435059785843,Energy,7 -ett2/15T/short,Moirai2,8.050238609313965,8.050238609313965,1.7549543380737305,0.7664317488670349,0.11144769191741943,0.1304120570421219,5.862476825714111,2.837294101715088,0.13469645380973816,0.08331392705440521,0.06601111590862274,Energy,7 -ett2/H/short,Moirai2,111.35823822021484,111.35823822021484,6.3867340087890625,0.7354120016098022,0.11914452165365219,0.10967425256967545,5.209497928619385,10.552640914916992,0.13383685052394867,0.08100155740976334,0.06421768665313721,Energy,7 -ett2/D/short,Moirai2,112720.8125,112720.8125,200.43377685546875,1.3028600215911865,0.42615124583244324,0.13562987744808197,11.184418678283691,335.73919677734375,0.19215241074562073,0.11471354216337204,0.09300631284713745,Energy,7 -ett2/W/short,Moirai2,3018818.25,3018818.25,1220.6822509765625,0.8076348304748535,0.13835936784744263,0.15684010088443756,8.835420608520508,1737.4747314453125,0.14574748277664185,0.10239652544260025,0.08512651920318604,Energy,7 -jena_weather/10T/short,Moirai2,1263.07080078125,1263.07080078125,7.481215953826904,0.3154330551624298,0.3750856816768646,0.5614975690841675,2.2623369693756104,35.53970718383789,0.22074612975120544,0.04646772891283035,0.03595806285738945,Nature,21 -jena_weather/H/short,Moirai2,1116.116455078125,1116.116455078125,8.453888893127441,0.536453366279602,1.3488954305648804,0.6119290590286255,4.829719066619873,33.408329010009766,0.2047909051179886,0.05182179436087608,0.04214661940932274,Nature,21 -jena_weather/D/short,Moirai2,341.45184326171875,341.45184326171875,9.141987800598145,1.0152313709259033,0.49779126048088074,0.4462507367134094,6.8829264640808105,18.47841453552246,0.11127026379108429,0.05504970625042915,0.043448418378829956,Nature,21 -bitbrains_fast_storage/5T/short,Moirai2,2142125.25,2142125.25,171.67364501953125,0.6864748001098633,2.1051151752471924,0.7426697015762329,13.77940845489502,1463.60009765625,4.59534215927124,0.5390127897262573,0.42746859788894653,Web/CloudOps,2 -bitbrains_fast_storage/H/short,Moirai2,2942650.5,2942650.5,292.0852355957031,1.0675305128097534,3.8830220699310303,0.5814351439476013,22.434349060058594,1715.41552734375,4.889491558074951,0.8325377702713013,0.6148646473884583,Web/CloudOps,2 -bitbrains_rnd/5T/short,Moirai2,1660531.75,1660531.75,115.86862182617188,1.6520637273788452,1.0863659381866455,0.6535171270370483,53.418670654296875,1288.6162109375,5.272645473480225,0.4741009473800659,0.4042632579803467,Web/CloudOps,2 -bitbrains_rnd/H/short,Moirai2,2235573.0,2235573.0,196.98313903808594,5.809325695037842,3.0100300312042236,0.5586869716644287,197.04698181152344,1495.1832275390625,6.54365348815918,0.8620945811271667,0.6697196960449219,Web/CloudOps,2 -bizitobs_application/10S/short,Moirai2,621851.5625,621851.5625,406.2403259277344,1.438624620437622,0.019857333973050117,0.020211493596434593,18.09433364868164,788.57568359375,0.030412212014198303,0.015667065978050232,0.01300627738237381,Web/CloudOps,2 -bizitobs_service/10S/short,Moirai2,4797.98974609375,4797.98974609375,22.429920196533203,0.8738541007041931,0.07667911797761917,0.05649945139884949,9.500022888183594,69.26752471923828,0.05131559818983078,0.016616802662611008,0.01355437096208334,Web/CloudOps,2 -bizitobs_l2c/5T/short,Moirai2,20.624032974243164,20.624032974243164,2.863663911819458,0.2905579209327698,0.1645745486021042,0.20925605297088623,4.3934006690979,4.5413689613342285,0.15596899390220642,0.09834981709718704,0.08443012833595276,Web/CloudOps,7 -bizitobs_l2c/H/short,Moirai2,67.85011291503906,67.85011291503906,5.013020038604736,0.5027103424072266,0.4784625768661499,0.636954665184021,7.536383152008057,8.237117767333984,0.4440017640590668,0.27021461725234985,0.23466992378234863,Web/CloudOps,7 -electricity/15T/medium,Moirai2,283338.0,283338.0,55.66805648803711,0.8413034677505493,0.13227048516273499,0.13056059181690216,9.948001861572266,532.2950439453125,0.9171979427337646,0.09592166543006897,0.07995456457138062,Energy,1 -electricity/H/medium,Moirai2,6361778.5,6361778.5,248.01693725585938,1.0840357542037964,0.23050330579280853,0.13177354633808136,13.117870330810547,2522.256591796875,0.9845198392868042,0.09680917859077454,0.0803634449839592,Energy,1 -solar/10T/medium,Moirai2,29.05948257446289,29.05948257446289,2.3172357082366943,0.9995384812355042,3.0454554557800293,1.504119634628296,8.617097854614258,5.390684604644775,1.1962143182754517,0.5142037868499756,0.41236650943756104,Energy,1 -solar/H/medium,Moirai2,714.8151245117188,714.8151245117188,11.081192016601562,0.8482193350791931,4.51237154006958,1.3964426517486572,9.920930862426758,26.736026763916016,0.9630941152572632,0.3991703987121582,0.3328731954097748,Energy,1 -kdd_cup_2018/H/medium,Moirai2,5786.36328125,5786.36328125,28.03462028503418,1.1240187883377075,0.9898953437805176,0.6367859840393066,13.927753448486328,76.06815338134766,1.5922441482543945,0.586815357208252,0.4954785704612732,Nature,1 -loop_seattle/5T/medium,Moirai2,91.64151000976562,91.64151000976562,5.445966720581055,0.864562451839447,0.1969592571258545,0.12233935296535492,10.540971755981445,9.572957038879395,0.17032258212566376,0.09689493477344513,0.08025791496038437,Transport,1 -loop_seattle/H/medium,Moirai2,71.71796417236328,71.71796417236328,4.58095121383667,0.9398095607757568,0.11857419461011887,0.10557787865400314,13.948471069335938,8.468646049499512,0.14968235790729523,0.08096779882907867,0.06914147734642029,Transport,1 -sz_taxi/15T/medium,Moirai2,17.261032104492188,17.261032104492188,2.8276829719543457,0.551992654800415,10671182839808.0,0.41678428649902344,6.010798931121826,4.154639720916748,0.38665857911109924,0.26316311955451965,0.21361444890499115,Transport,1 -m_dense/H/medium,Moirai2,39792.32421875,39792.32421875,81.75311279296875,0.7038857936859131,0.2560286521911621,0.2057977169752121,10.532341957092285,199.48013305664062,0.34707024693489075,0.14224009215831757,0.12004674971103668,Transport,1 -ett1/15T/medium,Moirai2,10.665780067443848,10.665780067443848,1.6653506755828857,1.0389811992645264,0.8133009672164917,0.3889414370059967,11.519312858581543,3.26585054397583,0.6224743723869324,0.3174174726009369,0.2604987323284149,Energy,7 -ett1/H/medium,Moirai2,167.32998657226562,167.32998657226562,7.205399036407471,1.3042583465576172,3247734521856.0,0.4289178252220154,16.902162551879883,12.935608863830566,0.6195850372314453,0.3451215326786041,0.2872616946697235,Energy,7 -ett2/15T/medium,Moirai2,12.762186050415039,12.762186050415039,2.2088661193847656,0.9195600748062134,0.14112639427185059,0.1747804433107376,11.09008502960205,3.572420120239258,0.1903807371854782,0.11771447211503983,0.0978662297129631,Energy,7 -ett2/H/medium,Moirai2,253.32736206054688,253.32736206054688,10.055752754211426,1.0852832794189453,0.18437251448631287,0.1749981790781021,11.153921127319336,15.916260719299316,0.21448184549808502,0.135507732629776,0.11071720719337463,Energy,7 -jena_weather/10T/medium,Moirai2,1942.8153076171875,1942.8153076171875,11.685406684875488,0.6510283946990967,0.8314501047134399,0.6661898493766785,9.578142166137695,44.07737731933594,0.27050212025642395,0.07171314209699631,0.058668095618486404,Nature,21 -jena_weather/H/medium,Moirai2,1340.6402587890625,1340.6402587890625,10.796601295471191,0.8172689080238342,2.6193902492523193,0.6811408400535583,10.437860488891602,36.61475372314453,0.22405217587947845,0.06606631726026535,0.05496005341410637,Nature,21 -bitbrains_fast_storage/5T/medium,Moirai2,3831930.5,3831930.5,260.03118896484375,0.9918146729469299,2.2075326442718506,0.8067377805709839,25.72391128540039,1957.53173828125,5.947522163391113,0.7900465726852417,0.6938542127609253,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Moirai2,2298316.0,2298316.0,147.26242065429688,4.3996782302856445,0.5052017569541931,0.7080380916595459,163.8717498779297,1516.019775390625,6.378668308258057,0.6196081042289734,0.5960801839828491,Web/CloudOps,2 -bizitobs_application/10S/medium,Moirai2,3613374.25,3613374.25,1043.581787109375,2.478975534439087,0.03553047776222229,0.036169908940792084,53.43565368652344,1900.8878173828125,0.07464227080345154,0.040978383272886276,0.03691835328936577,Web/CloudOps,2 -bizitobs_service/10S/medium,Moirai2,56963.7421875,56963.7421875,49.91252899169922,1.131983995437622,0.10594205558300018,0.08129119127988815,18.115341186523438,238.67079162597656,0.17962925136089325,0.03756534680724144,0.03391871601343155,Web/CloudOps,2 -bizitobs_l2c/5T/medium,Moirai2,77.68477630615234,77.68477630615234,5.3431549072265625,0.5360136032104492,0.37753960490226746,0.6504175662994385,14.239152908325195,8.813896179199219,0.46374058723449707,0.28112852573394775,0.2606326937675476,Web/CloudOps,7 -bizitobs_l2c/H/medium,Moirai2,73.60548400878906,73.60548400878906,5.040341854095459,0.5085751414299011,0.561661422252655,0.7950536608695984,10.486610412597656,8.579363822937012,0.5194935202598572,0.30520036816596985,0.27381497621536255,Web/CloudOps,7 -electricity/15T/long,Moirai2,391376.21875,391376.21875,63.17216110229492,0.9070706963539124,0.13897426426410675,0.134074404835701,10.943906784057617,625.6007080078125,0.9869617819786072,0.09966183453798294,0.08316168934106827,Energy,1 -electricity/H/long,Moirai2,9811671.0,9811671.0,291.53839111328125,1.262234091758728,0.30803436040878296,0.1474093645811081,17.574865341186523,3132.35888671875,1.2354013919830322,0.11498266458511353,0.09748584777116776,Energy,1 -solar/10T/long,Moirai2,29.88235855102539,29.88235855102539,2.3448755741119385,1.01083505153656,3.0021917819976807,1.503761887550354,9.01179027557373,5.466475963592529,1.1814912557601929,0.5068073272705078,0.40836524963378906,Energy,1 -solar/H/long,Moirai2,695.760498046875,695.760498046875,11.615360260009766,0.8833699226379395,5.532365322113037,1.3997015953063965,10.741609573364258,26.37727165222168,0.9145480990409851,0.4027256965637207,0.3319021165370941,Energy,1 -kdd_cup_2018/H/long,Moirai2,4539.05078125,4539.05078125,25.811946868896484,1.0698148012161255,0.9702631831169128,0.6511303782463074,15.044140815734863,67.37247467041016,1.5808305740356445,0.6056525707244873,0.5183228254318237,Nature,1 -loop_seattle/5T/long,Moirai2,102.15283966064453,102.15283966064453,5.852677822113037,0.928242027759552,0.20338046550750732,0.12979671359062195,12.898492813110352,10.10706901550293,0.17869503796100616,0.10347653925418854,0.08709994703531265,Transport,1 -loop_seattle/H/long,Moirai2,63.72977828979492,63.72977828979492,4.314446926116943,0.9038061499595642,0.12193074822425842,0.1005469411611557,13.63674259185791,7.98309326171875,0.14266857504844666,0.07710494101047516,0.06565489619970322,Transport,1 -sz_taxi/15T/long,Moirai2,17.477622985839844,17.477622985839844,2.822140693664551,0.5302703380584717,7329904328704.0,0.4138064384460449,6.833168029785156,4.180624961853027,0.38695117831230164,0.2612123191356659,0.21665456891059875,Transport,1 -m_dense/H/long,Moirai2,37458.58203125,37458.58203125,82.49683380126953,0.7182016968727112,0.26045969128608704,0.2135668247938156,10.146082878112793,193.54220581054688,0.33505016565322876,0.14281421899795532,0.11958861351013184,Transport,1 -ett1/15T/long,Moirai2,10.191858291625977,10.191858291625977,1.6781684160232544,1.0669130086898804,0.8645601868629456,0.3900018632411957,14.423590660095215,3.1924688816070557,0.6084877252578735,0.3198605477809906,0.2684479057788849,Energy,7 -ett1/H/long,Moirai2,203.1143341064453,203.1143341064453,7.873347282409668,1.4649949073791504,2660186456064.0,0.47378653287887573,24.58342933654785,14.251818656921387,0.682488739490509,0.37703752517700195,0.3230729103088379,Energy,7 -ett2/15T/long,Moirai2,13.122858047485352,13.122858047485352,2.279362916946411,0.959116518497467,0.14799295365810394,0.17831845581531525,13.057952880859375,3.6225485801696777,0.19305217266082764,0.12147137522697449,0.10234040766954422,Energy,7 -ett2/H/long,Moirai2,237.56210327148438,237.56210327148438,9.41628360748291,1.0515315532684326,0.18637529015541077,0.17694972455501556,14.501702308654785,15.413049697875977,0.21106985211372375,0.12894874811172485,0.10860944539308548,Energy,7 -jena_weather/10T/long,Moirai2,1825.4752197265625,1825.4752197265625,11.603591918945312,0.7330641150474548,0.9823510646820068,0.6760526895523071,13.818408012390137,42.725582122802734,0.26144516468048096,0.07100436836481094,0.05961557850241661,Nature,21 -jena_weather/H/long,Moirai2,1278.8531494140625,1278.8531494140625,11.639805793762207,1.0389355421066284,2.1917943954467773,0.6806634664535522,18.089033126831055,35.76105499267578,0.21532118320465088,0.07008452713489532,0.05799703299999237,Nature,21 -bitbrains_fast_storage/5T/long,Moirai2,5638371.5,5638371.5,367.75579833984375,0.9175259470939636,3.341670274734497,0.8233354091644287,21.859882354736328,2374.525390625,6.275162220001221,0.9718688130378723,0.8065826892852783,Web/CloudOps,2 -bitbrains_rnd/5T/long,Moirai2,2340055.5,2340055.5,154.8313751220703,3.3333687782287598,0.7227475643157959,0.6934060454368591,121.20797729492188,1529.7239990234375,5.860028266906738,0.5931240916252136,0.5697691440582275,Web/CloudOps,2 -bizitobs_application/10S/long,Moirai2,6483073.0,6483073.0,1533.4085693359375,3.4278934001922607,0.050070058554410934,0.05103800445795059,101.53668975830078,2546.18798828125,0.09819629043340683,0.05913744121789932,0.056454192847013474,Web/CloudOps,2 -bizitobs_service/10S/long,Moirai2,128119.78125,128119.78125,76.06694030761719,1.37122642993927,0.11781994998455048,0.09284570068120956,30.744794845581055,357.938232421875,0.2651720941066742,0.05635282024741173,0.053895868360996246,Web/CloudOps,2 -bizitobs_l2c/5T/long,Moirai2,85.3919677734375,85.3919677734375,5.707359790802002,0.5799223184585571,0.40167975425720215,0.7719800472259521,16.739368438720703,9.240777015686035,0.5188558101654053,0.3204597234725952,0.30009692907333374,Web/CloudOps,7 -bizitobs_l2c/H/long,Moirai2,99.5688705444336,99.5688705444336,5.755020618438721,0.6107828617095947,0.7120028138160706,0.8073437213897705,15.002673149108887,9.97842025756836,0.6095107197761536,0.35153326392173767,0.3212907910346985,Web/CloudOps,7 diff --git a/results/Moirai2/config.json b/results/Moirai2/config.json deleted file mode 100644 index 91f641de227e65ed9c42e091713223cbc599e290..0000000000000000000000000000000000000000 --- a/results/Moirai2/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "Moirai2", - "model_type": "pretrained", - "model_dtype": "float32", - "model_link": "https://huggingface.co/Salesforce/moirai-2.0-R-small", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai2.ipynb", - "org": "Salesforce AI Research", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/MoiraiAgent-leaking/all_results.csv b/results/MoiraiAgent-leaking/all_results.csv deleted file mode 100644 index d9311921cb89222a8220dbe2d67f9e41734fdb91..0000000000000000000000000000000000000000 --- a/results/MoiraiAgent-leaking/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -ett1/W/short,MoiraiAgent-leaking,1922959.1008590756,1922959.1008590756,1023.2215715135846,1.7678688932404552,0.6523123413431963,0.6174924906567194,10.592081938419708,1386.708008507586,0.5518042141837641,0.4071642853008308,0.3176080006270214,Energy,7 -saugeen/M/short,MoiraiAgent-leaking,442.6092938609176,442.6092938609176,13.155999881880623,0.7708285039004029,0.4059140170041235,0.376905313222615,6.120241012309893,21.03828162804457,0.6316323287276547,0.3949825840840487,0.3164227285413208,Nature,1 -us_births/M/short,MoiraiAgent-leaking,36880588.98640951,36880588.98640951,5135.903645833333,0.5796021824032279,0.0158337987051696,0.0159895124982955,3.116167384532352,6072.93907316791,0.018862507356484,0.0159520817078096,0.0116857772064188,Healthcare,1 -bizitobs_application/10S/long,MoiraiAgent-leaking,5800082.536261333,5800082.536261333,1464.715584081014,3.035268031373664,0.0438796853778002,0.0449018324974507,51.59158666862463,2408.336051356067,0.092879895106491,0.056488225442494,0.0503907930633193,Web/CloudOps,2 -bizitobs_l2c/H/long,MoiraiAgent-leaking,82.31742262811916,82.31742262811916,5.29689035225837,0.5646743036295974,0.6109710678291644,0.7847466165328574,5.301016690912072,9.0728949419752,0.55419860904321,0.3235493504829356,0.2631600366379862,Web/CloudOps,7 -ett1/D/short,MoiraiAgent-leaking,41145.4073356658,41145.4073356658,134.282200280778,1.6076143842723154,1.417166829902787,0.454428059923752,12.384847189063422,202.84330734748383,0.5327746751294115,0.3526966532235528,0.2730852757697575,Energy,7 -bizitobs_application/10S/medium,MoiraiAgent-leaking,2828606.770918675,2828606.770918675,938.2000554243724,2.1229976863180413,0.0301435773739771,0.0307052242576038,19.857662706714244,1681.846238786018,0.0660411518278822,0.0368403548887631,0.0303350781574969,Web/CloudOps,2 -bizitobs_l2c/H/medium,MoiraiAgent-leaking,70.95556323788983,70.95556323788983,4.829649629224358,0.4854082092213864,0.4647784520085575,0.7602969887287452,3.552974919534388,8.42351252375693,0.510056477894272,0.2924426208660596,0.2310871259677409,Web/CloudOps,7 -ett2/W/short,MoiraiAgent-leaking,3344984.803398259,3344984.803398259,1254.9863880702428,0.8198858253264116,0.1308706122168081,0.1447069959259727,6.41122158920198,1828.929961315703,0.153419184549162,0.105274117845128,0.0838331148615692,Energy,7 -ett2/D/short,MoiraiAgent-leaking,104282.32720071592,104282.32720071592,189.30453249537757,1.237825686492234,0.4110022840284508,0.1285059927702507,10.336576525749434,322.9277430025422,0.184820067481461,0.1083439785787208,0.0888575059495718,Energy,7 -saugeen/W/short,MoiraiAgent-leaking,968.515666503706,968.515666503706,15.271628594398498,1.2386532215612605,0.3978034767671309,0.4018297114926435,12.194035216392564,31.120984343425032,0.9394127902929924,0.4609868078678916,0.3785246081972413,Nature,1 -us_births/W/short,MoiraiAgent-leaking,1657965.44241333,1657965.44241333,1001.2321428571428,0.910554396027256,0.0136668304001199,0.0136704300942968,5.715512214382463,1287.6200691249458,0.0174793161815247,0.0135916281640429,0.0105582246634151,Healthcare,1 -jena_weather/D/short,MoiraiAgent-leaking,394.2980810603372,394.2980810603372,9.61741770075196,0.9967864218742054,0.5853275745379929,0.4566469308242122,7.240984737859527,19.856940375101527,0.1195712357222849,0.0579125734988972,0.0454740933462453,Nature,21 -bizitobs_application/10S/short,MoiraiAgent-leaking,362448.7766435564,362448.7766435564,326.0790887811449,1.0322422842293868,0.014563968016641,0.0146926839794667,7.6236653057529455,602.0371887546121,0.0232181679588484,0.0125755670789214,0.0099328847339682,Web/CloudOps,2 -bizitobs_l2c/H/short,MoiraiAgent-leaking,51.698960453781126,51.698960453781126,4.349450548567499,0.4327348105109728,0.3943748259399773,0.5896353840467818,2.969148880223248,7.190198916148365,0.3875701359980726,0.2344465237031578,0.1851041294926999,Web/CloudOps,7 -us_births/D/short,MoiraiAgent-leaking,144092.22343805592,144092.22343805592,225.1733553059896,0.3314301858018498,0.0217307261985107,0.0215195303870016,3.065084319944368,379.5948148197706,0.0355839336550964,0.0211081748835481,0.0171657923453774,Healthcare,1 -ett2/H/long,MoiraiAgent-leaking,234.7012196580409,234.7012196580409,9.268702529536354,1.0287833021144752,0.1809058778208943,0.173364232429192,7.351493310922624,15.319961477041677,0.2097950813149224,0.126927747421686,0.1029193447296313,Energy,7 -saugeen/D/short,MoiraiAgent-leaking,1086.453311888044,1086.453311888044,11.955319752693176,2.6561451289717404,0.2553949752399402,0.3013758503640539,36.156892150153325,32.96139123107585,1.0676273155342193,0.3872356553289948,0.3267546725551236,Nature,1 -ett1/H/long,MoiraiAgent-leaking,144.37525942663015,144.37525942663015,7.034508810350523,1.3433637923963433,3150427836070.028,0.4680202093894685,9.205412287337252,12.015625636088624,0.5754022876739224,0.3368673912393686,0.2616734754415228,Energy,7 -jena_weather/H/long,MoiraiAgent-leaking,1106.5238511486352,1106.5238511486352,10.249544346933783,0.8453325167374546,1.79609527110441,0.6108534520222177,6.790415363364537,33.26445326694301,0.2002888632087222,0.0617136127018452,0.0499238745289775,Nature,21 -ett2/H/medium,MoiraiAgent-leaking,235.30514024330705,235.30514024330705,9.48963560395475,1.0330257011357225,0.1757913497232386,0.1679309155735952,7.251909134525472,15.339659065419513,0.206711770472137,0.1278789423195871,0.1030607913319248,Energy,7 -ett1/H/medium,MoiraiAgent-leaking,140.97813261414478,140.97813261414478,6.820466484707071,1.2513513770335738,3369069804995.08,0.4326347979727412,9.050658851082996,11.87342126828425,0.5687087329049733,0.3266841767586929,0.2549407068345508,Energy,7 -m_dense/D/short,MoiraiAgent-leaking,8762.381605938195,8762.381605938195,45.44365694116663,0.6709316438156029,0.1206921257646375,0.0982101740781094,7.932004904699334,93.60759374077615,0.1620485911421941,0.0786696921625643,0.0653627983208705,Transport,1 -jena_weather/H/medium,MoiraiAgent-leaking,1168.884994716826,1168.884994716826,9.67344134557375,0.7511120656178825,1.9190698241034452,0.6316595811251793,6.508329829883853,34.18896012921168,0.2092083105485261,0.0591935032083237,0.0474920323767152,Nature,21 -solar/W/short,MoiraiAgent-leaking,1482700.2404831452,1482700.2404831452,960.4683833992394,1.043678869855943,0.2131107776370573,0.1875504751764428,5.459400333652198,1217.6617923229526,0.2485552090738321,0.1960556053821608,0.1404323058515825,Energy,1 -bizitobs_l2c/5T/long,MoiraiAgent-leaking,88.81988518852675,88.81988518852675,5.938304780518732,0.6177959427352402,0.4901490008059811,0.776735331715351,6.102607049720008,9.424430231506134,0.5291676679505634,0.3334269356445322,0.2745323543842006,Web/CloudOps,7 -covid_deaths/D/short,MoiraiAgent-leaking,242705.70573679812,242705.70573679812,83.87179795183302,33.239184101665955,0.0876101806404636,0.1248095442844032,701.6649048043954,492.6517083465744,0.1853625940445299,0.0315571706585731,0.028651235209677,Healthcare,1 -bizitobs_l2c/5T/medium,MoiraiAgent-leaking,62.152304924536445,62.152304924536445,5.245451580836627,0.5503502362005954,0.4107753440196096,0.6907350538595017,4.24717078066682,7.883673314168748,0.4147971507621293,0.2759878908581025,0.2200715634238864,Web/CloudOps,7 -sz_taxi/15T/medium,MoiraiAgent-leaking,16.26584353405287,16.26584353405287,2.7432859414279607,0.534836458162746,9614899200899.854,0.4038453903786848,3.9433375324168223,4.033093543925416,0.3753466653680791,0.2553085414587048,0.2020051126646884,Transport,1 -hierarchical_sales/W/short,MoiraiAgent-leaking,464.1881975026845,464.1881975026845,8.938606926508374,0.7191409360244655,0.5280152514457227,0.4580353239245712,7.421791152111056,21.54502721053479,0.9891427376923094,0.4103758161938505,0.344678402667067,Sales,1 -bizitobs_service/10S/long,MoiraiAgent-leaking,129921.21888730965,129921.21888730965,75.65915844685028,1.3131301187280573,0.0786872796079503,0.0763868603599212,19.85008828981294,360.44586124314094,0.2670298171734636,0.0560507233399257,0.0515311539731114,Web/CloudOps,2 -solar/D/short,MoiraiAgent-leaking,121706.4160932687,121706.4160932687,248.42212083693548,0.9677288967397706,1.0561448218938685,0.4238682760182228,4.515579545076429,348.86446665326736,0.5039903406221526,0.3588853588322652,0.2703548529272707,Energy,1 -sz_taxi/15T/long,MoiraiAgent-leaking,15.455212343807252,15.455212343807252,2.6837548986810518,0.5061601466274933,6568357735729.298,0.39712348290551,3.5873166400619425,3.931311784100474,0.3638751877116986,0.2484035536126279,0.1960291123136082,Transport,1 -m4_weekly/W/short,MoiraiAgent-leaking,220386.8966151768,220386.8966151768,241.1860764615664,1.858585168765529,0.0594641682519109,0.0585660750529685,13.247784358335284,469.45382799075867,0.0855272054973445,0.0439403619583068,0.035039527537903,Econ/Fin,1 -ett1/H/short,MoiraiAgent-leaking,99.13028522490048,99.13028522490048,4.886843017502023,0.8076410086977762,0.4670246261471479,0.2563510184376263,5.740288531249164,9.956419297362908,0.4647067967904227,0.2280889441530953,0.1768892652838432,Energy,7 -bizitobs_service/10S/medium,MoiraiAgent-leaking,19560.353310614497,19560.353310614497,35.81813772769792,0.9921884018998464,0.0635273419636437,0.0567131624258045,9.695895273721227,139.85833300384536,0.1052606771956583,0.0269575745122114,0.0209736849782821,Web/CloudOps,2 -ett2/H/short,MoiraiAgent-leaking,107.90844776117396,107.90844776117396,6.385164782617774,0.7318255343840185,0.1193336995280915,0.1091390787750913,5.54881917778854,10.387899102377435,0.1317474639433783,0.0809816555474566,0.0648465254146567,Energy,7 -sz_taxi/H/short,MoiraiAgent-leaking,7.132598843609232,7.132598843609232,1.837800126860474,0.5597375526572784,1.137913795837233,0.2944005599432808,4.193710668341138,2.6706925775179053,0.2487914911173868,0.1712024205954399,0.1356767697536933,Transport,1 -kdd_cup_2018/D/short,MoiraiAgent-leaking,3078.232510680184,3078.232510680184,22.192841526974757,1.2387248595389642,0.5380777106260454,0.4859012938921304,10.357692304531176,55.48182144342581,1.2428517069707397,0.4971432129793569,0.4001246255639362,Nature,1 -m_dense/H/long,MoiraiAgent-leaking,33275.39462595018,33275.39462595018,76.69024538124785,0.6633716774678452,0.2648703596900888,0.186496713131142,7.305148955855769,182.4154451408931,0.3157881014450989,0.1327621516346807,0.1095644998487754,Transport,1 -loop_seattle/D/short,MoiraiAgent-leaking,18.468066837006607,18.468066837006607,2.937754229790655,0.885774281184359,0.0546682304801358,0.0541910881425739,8.640705751025932,4.2974488754383815,0.076801153435212,0.0525015933631274,0.0430062016079124,Transport,1 -m_dense/H/medium,MoiraiAgent-leaking,35796.37455213744,35796.37455213744,77.98605653602216,0.6768500768864981,0.2682997191560772,0.1884592107191146,8.317821293168087,189.19929849800565,0.3291828765949031,0.13568588588129,0.1123607476262086,Transport,1 -hospital/M/short,MoiraiAgent-leaking,3186.2245918136227,3186.2245918136227,18.404263072250096,0.76587996713784,0.1931911764511835,0.1744733258827311,5.165826464899864,56.44665261832293,0.2049144567170711,0.0668117486829324,0.0519486082776373,Healthcare,1 -m4_hourly/H/short,MoiraiAgent-leaking,1241505.6206661155,1241505.6206661155,214.67741793993972,0.6788343354056098,0.0868393482228086,0.0778341758176444,5.472877345296247,1114.2287111119133,0.1521168364398163,0.0293082105553569,0.0217615815324199,Econ/Fin,1 -bizitobs_l2c/5T/short,MoiraiAgent-leaking,19.21867740203917,19.21867740203917,2.537843653311332,0.2665844470986803,0.1480165085857754,0.2031408347986984,2.680562154908216,4.383911199150727,0.1505612493006792,0.087159819990509,0.0706035533707206,Web/CloudOps,7 -solar/H/medium,MoiraiAgent-leaking,600.5255129407659,600.5255129407659,10.172298594019257,0.7759985673179968,3.788771915294393,1.3576157924833727,4.035497267113435,24.5056220680228,0.8827497032920046,0.3664299376176856,0.2830659024458329,Energy,1 -jena_weather/H/short,MoiraiAgent-leaking,1095.598569307172,1095.598569307172,8.321559204081451,0.5172613682014809,1.2378954620962417,0.5695605894813986,4.785618543702605,33.09982733047367,0.2028998187923268,0.0510106242464674,0.0411788546111014,Nature,21 -electricity/W/short,MoiraiAgent-leaking,36092800415.65694,36092800415.65694,25948.885207406463,1.4202135280103632,0.1058227079521159,0.0902172530846871,12.024212164992706,189981.05278068373,0.4334740831947393,0.0592067948912308,0.0460692062479211,Energy,1 -ett1/15T/long,MoiraiAgent-leaking,8.991227395028174,8.991227395028174,1.5273126489462865,0.9765382704500768,0.8000395593895443,0.3723549572820061,7.369432331297573,2.998537542707807,0.5715242204220015,0.2911073010082861,0.2301857186651809,Energy,7 -solar/H/long,MoiraiAgent-leaking,599.7042825914968,599.7042825914968,10.698153308879933,0.8133307519736666,4.639376467594781,1.3521956245880704,4.057072931014464,24.48886037755732,0.8490735308877917,0.3709245209415354,0.2796492297528261,Energy,1 -restaurant/D/short,MoiraiAgent-leaking,137.63240083189052,137.63240083189052,7.062107270439989,0.678016519175151,0.6584484470146008,0.3902829708478726,4.6589857976088025,11.731683631597408,0.5399192627639758,0.3250145393237894,0.2554124505749786,Sales,1 -ett2/15T/long,MoiraiAgent-leaking,11.889467509118012,11.889467509118012,2.1591536020049973,0.890255619901942,0.1348756420333238,0.1608448022710511,7.699890880232282,3.448110715901972,0.1837560615834547,0.1150652038023039,0.0928460923382147,Energy,7 -hierarchical_sales/D/short,MoiraiAgent-leaking,28.89327839263237,28.89327839263237,2.312057612056498,0.7450733557706084,0.6342060536884567,1.063330113769468,7.999372259333705,5.3752468215545575,1.649066197585318,0.7093136708854482,0.5808777900911117,Sales,1 -bizitobs_service/10S/short,MoiraiAgent-leaking,2135.672394549348,2135.672394549348,18.226838727946635,0.723223785025364,0.0558043224962753,0.0450336083052,5.872363644096716,46.21333567866908,0.0342363179417515,0.0135030254059553,0.0106411772938972,Web/CloudOps,2 -ett1/15T/medium,MoiraiAgent-leaking,9.257671719036347,9.257671719036347,1.542459416512459,0.9714533238729788,0.7706487963148978,0.3739968314331537,7.066523659793911,3.0426422265912807,0.5799306339124488,0.293994298568728,0.2311450045472186,Energy,7 -ett2/15T/medium,MoiraiAgent-leaking,11.334073198617274,11.334073198617274,2.0943166991346884,0.8633323594950876,0.1301556993622524,0.1569237683263772,6.876641756063168,3.3666115306962983,0.1794128106229322,0.1116099205091781,0.0888799033407043,Energy,7 -jena_weather/10T/long,MoiraiAgent-leaking,1411.4909967526894,1411.4909967526894,9.616698463717212,0.6472233828365973,0.7977089640109153,0.6185700220762774,5.970667404026429,37.5698149683052,0.229896134127328,0.0588462254004157,0.0477285235329391,Nature,21 -loop_seattle/H/long,MoiraiAgent-leaking,53.83723740263109,53.83723740263109,3.999869223036408,0.8383294835462033,0.1193368716321157,0.0927078619296184,7.337945736874577,7.33738627868474,0.1311288947505264,0.0714830064061217,0.0573188110243204,Transport,1 -loop_seattle/H/medium,MoiraiAgent-leaking,59.07611561709591,59.07611561709591,4.191195134581735,0.8618598731158854,0.1122319039773527,0.0954522603633129,6.785559739901541,7.686098855537567,0.1358509380211795,0.0740789054583282,0.0585107628729103,Transport,1 -sz_taxi/15T/short,MoiraiAgent-leaking,16.570788246179074,16.570788246179074,2.713615720856091,0.541343904449962,1026926200035.912,0.3920738526027632,4.18019953812287,4.0707233074945135,0.380630049622624,0.2537346826261978,0.2009064787048499,Transport,1 -kdd_cup_2018/H/medium,MoiraiAgent-leaking,4377.495776316529,4377.495776316529,19.649335198091435,0.8141400962175239,0.7506871951548849,0.4347940531649916,7.555753436484032,66.16264638235482,1.3849040268428403,0.4112961758415845,0.3406220726609609,Nature,1 -kdd_cup_2018/H/long,MoiraiAgent-leaking,3352.225861948829,3352.225861948829,17.699599378355167,0.754144402047447,0.6688743226394185,0.449927874475496,6.527752036118429,57.89840983955284,1.3585307087219898,0.4153041396854681,0.3411435264504319,Nature,1 -ett2/15T/short,MoiraiAgent-leaking,7.699720499513117,7.699720499513117,1.6798881159030965,0.7260620959089167,0.1049230648719417,0.1238508746902464,6.058336669766258,2.7748370221533945,0.1317313957076078,0.0797502715920969,0.0642960478328363,Energy,7 -bitbrains_rnd/H/short,MoiraiAgent-leaking,1830237.7388572784,1830237.7388572784,144.0952470391579,5.805481271955453,1.1014774761453825,0.5055499661667071,208.3938026491316,1352.862793803303,5.920789498800784,0.6306313023052907,0.6168246016091092,Web/CloudOps,2 -ett1/15T/short,MoiraiAgent-leaking,4.570573538356436,4.570573538356436,1.0557503806183204,0.672161226538084,0.468461707618226,0.2367501389081841,4.770646742176777,2.137889973398172,0.4041502060801065,0.1995807731947193,0.1559861644073163,Energy,7 -m_dense/H/short,MoiraiAgent-leaking,42575.00205910077,42575.00205910077,85.12043382046951,0.757566311892297,0.2845861172071204,0.2036550518040063,9.56965571199351,206.33710780928564,0.3659906556060106,0.1509824563801495,0.125644344007529,Transport,1 -electricity/D/short,MoiraiAgent-leaking,1376024314.6556034,1376024314.6556034,4356.532716627356,1.383151148354362,0.2374848183037236,0.0889351669357248,11.933170241617075,37094.80172012789,0.6095908133275252,0.0715923040121236,0.0569352737249552,Energy,1 -jena_weather/10T/medium,MoiraiAgent-leaking,1447.1449215696714,1447.1449215696714,9.54355435575667,0.6126390130883194,0.7427771026741966,0.6212686116066237,5.653290667652542,38.04135804055464,0.2334591768869023,0.0585686331727467,0.0475902665359285,Nature,21 -car_parts/M/short,MoiraiAgent-leaking,1.3712891236144362,1.3712891236144362,0.4454835633693085,0.8243541012699945,0.8436202817784486,1.8826219878971917,16.154873936551763,1.171020547904449,2.8079871500722486,1.0682238870598233,0.9377930518310624,Sales,1 -jena_weather/10T/short,MoiraiAgent-leaking,858.288823557019,858.288823557019,5.774348040532098,0.2743923194800031,0.3471210258267757,0.5057820253920888,2.288540366828305,29.29656675375152,0.1819684131036287,0.0358659415103348,0.0287590539495772,Nature,21 -bitbrains_fast_storage/H/short,MoiraiAgent-leaking,4710294.1306892745,4710294.1306892745,368.6220884098037,0.9704924741677424,3.350517179106832,0.5391573042515456,19.34355072422629,2170.321204496992,6.1861205010729625,1.0506927055475754,0.8211485157407529,Web/CloudOps,2 -solar/H/short,MoiraiAgent-leaking,603.5866565715182,603.5866565715182,10.188234415364231,0.7865458014370584,4.024865043272097,1.3496179036589395,4.896978849421916,24.56800066288501,0.9055846907206506,0.3755417153649517,0.2931557017925997,Energy,1 -m4_daily/D/short,MoiraiAgent-leaking,381714.3416147495,381714.3416147495,169.3211172917965,3.150790890483381,0.0382864091538235,0.0290736550938306,25.854586458178687,617.8303501890705,0.0954377140511578,0.0261554330731272,0.020900875015841,Econ/Fin,1 -bitbrains_rnd/5T/medium,MoiraiAgent-leaking,2309249.3766866284,2309249.3766866284,148.7534943958135,4.354969717385149,0.6745840486026569,0.6563139580611497,159.93464776340124,1519.621458352911,6.393822021503173,0.6258817701050262,0.655730522595201,Web/CloudOps,2 -bitbrains_rnd/5T/long,MoiraiAgent-leaking,2390128.7436598563,2390128.7436598563,164.61913903575265,3.2710403459027115,1.1952702965853588,0.6411810709017152,115.53050312512713,1546.0041214886382,5.922393632691715,0.6306188497775609,0.6249570910274973,Web/CloudOps,2 -solar/10T/long,MoiraiAgent-leaking,18.48530508500165,18.48530508500165,1.844432255377427,0.8077314817128574,3.196107120891229,1.4404321352288,4.0539420978601495,4.299454044992416,0.9292581941919216,0.3986445183517212,0.301211438678737,Energy,1 -electricity/H/long,MoiraiAgent-leaking,7941654.720915937,7941654.720915937,270.651409538232,1.1636586657798176,0.2880209942651526,0.1338301233228583,10.444912646142264,2818.09416466447,1.1114555504409094,0.1067448402320295,0.0853057705828549,Energy,1 -loop_seattle/H/short,MoiraiAgent-leaking,41.83198045939066,41.83198045939066,3.5146763169777127,0.7252731216258883,0.1011813330586836,0.081206490150456,6.497171440752356,6.467764718926522,0.1144860148854065,0.0622133461295235,0.0503988029618021,Transport,1 -solar/10T/medium,MoiraiAgent-leaking,19.410690062794465,19.410690062794465,1.839805845075798,0.8028349055702573,3.519398278224154,1.437968320132993,4.0691967534132125,4.405756468847826,0.977654881197956,0.408260233540614,0.310317738127033,Energy,1 -kdd_cup_2018/H/short,MoiraiAgent-leaking,3743.1493174646353,3743.1493174646353,16.443774578741152,0.6770786314312887,0.5786148762220655,0.3711718737136376,6.3918925505540285,61.18128241108252,1.280635042112117,0.3441979821317664,0.2885938116836737,Nature,1 -bitbrains_fast_storage/5T/medium,MoiraiAgent-leaking,3222058.5119395223,3222058.5119395223,291.8819936104378,0.9613594805425484,4.132512384745454,0.768922542236128,21.66248865775462,1795.009334777823,5.453734098558912,0.8868181075535352,0.7106437689628734,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,MoiraiAgent-leaking,4476090.869644248,4476090.869644248,375.56108774984233,0.8719455125603943,4.437202793429223,0.7826495034403658,16.12787046000301,2115.6774020734465,5.591103636573008,0.9924958131214008,0.79908114565413,Web/CloudOps,2 -electricity/H/medium,MoiraiAgent-leaking,5353877.008593891,5353877.008593891,234.74639760218935,1.0422547287235862,0.2302762041186331,0.1216231671239951,9.391840463702495,2313.8446379551697,0.9031698078047152,0.0916292543273713,0.073331805999729,Energy,1 -m4_yearly/A/short,MoiraiAgent-leaking,3516755.460513619,3516755.460513619,869.6197613247356,3.2344877796601037,0.1650581389494087,0.1447390793590613,32.22733623253641,1875.3014319073127,0.3007132764860076,0.1394475593499499,0.1140816815605301,Econ/Fin,1 -m4_quarterly/Q/short,MoiraiAgent-leaking,1758350.28115295,1758350.28115295,551.5888870245616,1.144955127800403,0.1140453732170217,0.1005360687052079,9.314091832099416,1326.0280091887012,0.2219537386032714,0.0923262667143985,0.0740449863272539,Econ/Fin,1 -solar/10T/short,MoiraiAgent-leaking,21.58929055122808,21.58929055122808,1.877066185608718,0.821064877237723,5.119796612755968,1.499489365114564,6.064855980361769,4.646427719359043,1.351645294108808,0.5460383395049954,0.4412185972369583,Energy,1 -bitbrains_rnd/5T/short,MoiraiAgent-leaking,1702753.7780737197,1702753.7780737197,119.24636908403642,1.6113625487668528,0.9565043700860956,0.5837912460744317,52.53284280882733,1304.8960794154143,5.339258080624209,0.4879217201742764,0.4139724340422097,Web/CloudOps,2 -m4_monthly/M/short,MoiraiAgent-leaking,1809005.8372003816,1809005.8372003816,549.1371842566176,0.9072368197886248,0.1569295508788732,0.1297795411211348,8.03107875188016,1344.9928762638044,0.2795356919922433,0.1141295582369595,0.0919525076549633,Econ/Fin,1 -electricity/H/short,MoiraiAgent-leaking,1703762.1760296577,1703762.1760296577,162.05968912801202,0.8677276145534056,0.1497353933223731,0.1142297063122895,10.68528836799473,1305.282412365101,0.6161865191116784,0.076503747224458,0.0633078101249869,Energy,1 -temperature_rain/D/short,MoiraiAgent-leaking,174.9461532522185,174.9461532522185,5.650777000078573,1.309692067775514,23.49794962106201,1.4846019362913354,22.59824556451883,13.226721182977228,1.557134784195671,0.6652458536647419,0.5477337723339576,Nature,1 -bitbrains_fast_storage/5T/short,MoiraiAgent-leaking,1871352.344541768,1871352.344541768,159.8617234659177,0.6521529904922871,1.9962647783722725,0.6823002326646034,13.26901703348176,1367.9738098888326,4.295099415098394,0.5019262722660474,0.4134415562173111,Web/CloudOps,2 -loop_seattle/5T/long,MoiraiAgent-leaking,88.06876240809983,88.06876240809983,5.4172781922252415,0.8605164811153628,0.17373773724089,0.1221241736587049,6.803290176602598,9.384495852633735,0.1659198153711317,0.0957785917946583,0.0771438316697811,Transport,1 -loop_seattle/5T/medium,MoiraiAgent-leaking,73.99705059631746,73.99705059631746,5.016062351226753,0.7988602332224258,0.1613098377107649,0.1145437652741581,6.297998533106341,8.602153834727526,0.153049991877322,0.0892460559135899,0.0716343078152939,Transport,1 -loop_seattle/5T/short,MoiraiAgent-leaking,44.62217712510064,44.62217712510064,3.5802928772730938,0.5669836442339283,0.0952770218366573,0.0765577464950278,4.935491572959085,6.679983317726223,0.1145469662019274,0.0613941184729783,0.0492957716663473,Transport,1 -electricity/15T/long,MoiraiAgent-leaking,295611.2374323296,295611.2374323296,55.91430838385571,0.8510185480364513,0.1252222113691885,0.1231474356275867,7.391626067003246,543.7014230552736,0.8577556378079997,0.0882116750419311,0.0708844731948339,Energy,1 -electricity/15T/medium,MoiraiAgent-leaking,227065.9224501587,227065.9224501587,50.583872111053125,0.8014910916163803,0.1219113695165609,0.1210731746257233,7.298422489486778,476.5143465313072,0.8210821913509518,0.0871611040932867,0.0705510395156446,Energy,1 -electricity/15T/short,MoiraiAgent-leaking,131880.39777924397,131880.39777924397,48.60602376284002,0.9176286302001752,0.1548712764834733,0.1487684250526304,10.938975568133582,363.1534080512586,0.7158469673619654,0.0958120561028618,0.0808094801655365,Energy,1 \ No newline at end of file diff --git a/results/MoiraiAgent-leaking/config.json b/results/MoiraiAgent-leaking/config.json deleted file mode 100644 index 7238b60e7f1bc2e3ae1adc3c5da53725d5769b47..0000000000000000000000000000000000000000 --- a/results/MoiraiAgent-leaking/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "MoiraiAgent-leaking", - "model_type": "agentic", - "model_dtype": "bf16", - "model_link": "https://huggingface.co/Salesforce/moirai-agent", - "code_link": "https://github.com/SalesforceAIResearch/uni2ts/tree/main/project/moirai-agent/gift_eval", - "org": "Salesforce AI Research", - "testdata_leakage": "Yes", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/MoiraiAgent/all_results.csv b/results/MoiraiAgent/all_results.csv deleted file mode 100644 index 7a6fd586f5ae64ee90d6393668e187a74bce7996..0000000000000000000000000000000000000000 --- a/results/MoiraiAgent/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -ett1/W/short,MoiraiAgent,1352765.7856364343,1352765.7856364343,925.8344933646066,1.618608939759112,0.6359818469834985,0.584643682236503,7.834702064371434,1163.0845995182096,0.462819122359509,0.3684116424950215,0.2729947667639454,Energy,7 -ett1/D/short,MoiraiAgent,37471.18491072942,37471.18491072942,129.08230155796286,1.5566275280936923,1.3726162636579862,0.4516571430793921,11.44452537614056,193.57475277197028,0.5084305090957287,0.339038946745676,0.2613360178621839,Energy,7 -us_births/M/short,MoiraiAgent,38466434.88366699,38466434.88366699,5063.61328125,0.5713265856247004,0.0156097670383255,0.0157715784210701,3.0027055899003563,6202.131479069675,0.0192637780883921,0.0157275483282832,0.0120038061573812,Healthcare,1 -bizitobs_application/10S/long,MoiraiAgent,6093266.163729536,6093266.163729536,1500.5113391113282,3.144567357947045,0.0455923608676251,0.0466076387501265,51.42855102278452,2468.454205313426,0.0951984120054974,0.0578687246342912,0.0517269306335458,Web/CloudOps,2 -ett2/W/short,MoiraiAgent,3372574.287506861,3372574.287506861,1233.998185294015,0.7782184281428035,0.1232854068137059,0.1297796232774148,6.230850513900975,1836.4569931002636,0.1540505871194562,0.1035135294009621,0.0834030901069498,Energy,7 -bizitobs_application/10S/medium,MoiraiAgent,2896830.038944903,2896830.038944903,948.5478690640132,2.1336761416309296,0.0303295118123068,0.0308841900318056,19.999294735585075,1702.0076494965888,0.066832831087909,0.0372466830749566,0.0303255314571911,Web/CloudOps,2 -saugeen/M/short,MoiraiAgent,418.0067241953927,418.0067241953927,12.781790682247706,0.748843312965724,0.3706422391829786,0.3637731085979867,4.861806725260907,20.4452127451732,0.6138266216738716,0.383747701293984,0.3039403723776294,Nature,1 -bizitobs_l2c/H/medium,MoiraiAgent,71.49381001060063,71.49381001060063,4.801446896820285,0.4834676911213507,0.4809710528099655,0.7523832145692969,3.537655274061299,8.455401232975325,0.5119873876736136,0.2907349025815024,0.2297339134520516,Web/CloudOps,7 -bizitobs_l2c/H/long,MoiraiAgent,81.36305677386217,81.36305677386217,5.266159118091067,0.5627149878438005,0.6185166461030641,0.782885825606479,5.402726941594578,9.020147270076148,0.5509766290044599,0.3216721980041929,0.2616038483114018,Web/CloudOps,7 -ett2/D/short,MoiraiAgent,100697.58484477576,100697.58484477576,187.7946618995969,1.2367586908193842,0.4058072733327699,0.1285819139689067,10.203785514356522,317.328827629599,0.1816156604911263,0.1074798398001636,0.0880240185114922,Energy,7 -us_births/W/short,MoiraiAgent,2157828.067363194,2157828.067363194,1108.154994419643,1.0075658189502523,0.015034289227291,0.0150864433367893,7.186989401908293,1468.954753341026,0.0199409167390897,0.0150430953897451,0.0119241987550481,Healthcare,1 -bizitobs_l2c/H/short,MoiraiAgent,59.604154625757474,59.604154625757474,4.530936563612952,0.4492353413976346,0.4023519555423127,0.5752554834618275,3.0804914539438903,7.720372699925663,0.4161478607421958,0.2442290847078252,0.1939171911345982,Web/CloudOps,7 -us_births/D/short,MoiraiAgent,157364.50620109678,157364.50620109678,230.93515380859372,0.3399361562939616,0.02217824249431,0.0220657883723069,3.3457535002733585,396.69195378920506,0.0371866516982003,0.0216482967388692,0.0176490207026225,Healthcare,1 -bizitobs_application/10S/short,MoiraiAgent,341587.4033823008,341587.4033823008,318.5398887930976,1.0211638056692272,0.0142322837559567,0.0143715490721235,7.669099641985366,584.4547915641558,0.0225400851780626,0.012284810270426,0.0096597729220868,Web/CloudOps,2 -saugeen/W/short,MoiraiAgent,925.0864911976036,925.0864911976036,14.762212866544724,1.1973237281931408,0.393585953589763,0.3857141271523241,11.8851586831272,30.41523452478385,0.918109145168459,0.4456096705305454,0.3676909190685906,Nature,1 -jena_weather/D/short,MoiraiAgent,347.5738341526048,347.5738341526048,9.303695782782446,1.021803728112439,0.6929608194968275,0.4538076797337613,7.392566130949666,18.643332163339384,0.1122633307368286,0.0560234547980214,0.0441988054769723,Nature,21 -ett1/H/long,MoiraiAgent,142.02542733740407,142.02542733740407,7.093178078946633,1.350921080433261,3832430809346.176,0.4701999242605449,9.459183334012405,11.917442147432649,0.5707004930528605,0.3396769354436225,0.2623340206157171,Energy,7 -ett2/H/long,MoiraiAgent,226.8490278693037,226.8490278693037,9.179426481515645,1.022521647582833,0.1793278661954687,0.1727388574068828,7.362789210135406,15.061508153877012,0.2062557619745508,0.1257051806559636,0.1017823638023186,Energy,7 -ett2/H/medium,MoiraiAgent,233.82900894743756,233.82900894743756,9.440233191111613,1.0294221451635657,0.1749895710607018,0.1675125075863896,7.522601591701592,15.291468501992789,0.2060623716397702,0.127213213037023,0.1023839838442453,Energy,7 -jena_weather/H/medium,MoiraiAgent,1191.0267603718796,1191.0267603718796,9.780318472896507,0.7466179132415781,2.0369253597855503,0.664567048766185,6.610473623691858,34.511255560641075,0.2111804934535347,0.059847503305401,0.0478965476603304,Nature,21 -saugeen/D/short,MoiraiAgent,1082.3818935053832,1082.3818935053832,12.204498295783996,2.711603328343001,0.2773404714519053,0.3111324092278479,33.1936581695358,32.89957284685294,1.065625003340056,0.3953066077103364,0.3314646700265691,Nature,1 -jena_weather/H/long,MoiraiAgent,1119.547376755506,1119.547376755506,10.470168261985936,0.8588687641560072,2.0158468188413217,0.6373308782191968,6.882826956612988,33.45963802487269,0.2014640917016628,0.0630420131053584,0.0506773646350672,Nature,21 -m_dense/D/short,MoiraiAgent,8842.001343573229,8842.001343573229,45.97818169487847,0.6846139569330977,0.1229454990742302,0.0998071500562728,8.062547573127155,94.03191662182172,0.16278315681486,0.0795950335778078,0.0662035696607738,Transport,1 -bizitobs_l2c/5T/long,MoiraiAgent,80.1826368188519,80.1826368188519,5.661460135795778,0.5981732821582612,0.4771609656580611,0.752693502060964,5.875505446003986,8.954475798105209,0.5027804290982414,0.3178825227268256,0.264263140882958,Web/CloudOps,7 -solar/W/short,MoiraiAgent,1494598.358810968,1494598.358810968,962.6622683671268,1.052154052439466,0.2132898552299666,0.1879707058115188,5.4925076460304565,1222.5376717348909,0.2495504979416392,0.1965034321435121,0.144418048867952,Energy,1 -ett1/H/medium,MoiraiAgent,142.02682610862587,142.02682610862587,6.909293941711075,1.260470380932714,3293538275996.3164,0.435428440815242,9.182945686392657,11.917500833170765,0.5708200395727973,0.3309388013844405,0.2562620040969263,Energy,7 -solar/D/short,MoiraiAgent,125421.58344939328,125421.58344939328,247.73921307531296,0.9663387213572382,1.0770709158385756,0.4213955212462256,4.539166014978126,354.149097767301,0.5116248327811702,0.3578987896964187,0.2748545778966779,Energy,1 -bizitobs_l2c/5T/medium,MoiraiAgent,102.8356535341232,102.8356535341232,6.091979077985349,0.6179668938003371,0.4265015101387533,0.7138545189631208,4.407241280697617,10.14079156348868,0.5335547630376744,0.3205276859340874,0.2536645769330768,Web/CloudOps,7 -sz_taxi/15T/long,MoiraiAgent,15.457887452767109,15.457887452767109,2.682040417661973,0.5058064327859539,6574931508316.631,0.396931676873654,3.5857764155939265,3.931652000465848,0.3639066775300116,0.2482448643157962,0.1960831512611438,Transport,1 -sz_taxi/15T/medium,MoiraiAgent,16.27079166543773,16.27079166543773,2.7410300020763905,0.5342650077472046,9614655378610.76,0.4037579995396392,3.912378937875328,4.033706938467113,0.3754037519675526,0.2550985886510988,0.2018256062488793,Transport,1 -sz_taxi/H/short,MoiraiAgent,7.166686088220611,7.166686088220611,1.8404209862425276,0.5604558851701957,1.0833992498144307,0.29495951020118,4.206606214052367,2.677066694765114,0.2493852794657125,0.1714465698169411,0.1358042784995731,Transport,1 -covid_deaths/D/short,MoiraiAgent,213982.2644382116,213982.2644382116,79.87039716483997,33.8008034515976,0.0862469077749362,0.1254148667841752,694.4651813887957,462.5821704715948,0.1740487846173121,0.0300516242103975,0.0271142249459932,Healthcare,1 -bizitobs_service/10S/long,MoiraiAgent,129820.06615492344,129820.06615492344,76.2003300948654,1.3197164792493763,0.0792434230934667,0.0766594112957767,19.449316660195937,360.3055177969433,0.2669258462618532,0.056451640597598,0.051683343269173,Web/CloudOps,2 -ett2/H/short,MoiraiAgent,106.52021611190116,106.52021611190116,6.308384767919779,0.7238682355831751,0.1176922293956289,0.1077685183660484,5.535480461111588,10.320863147620027,0.1308972615159894,0.0800078713281162,0.0638451716560979,Energy,7 -hierarchical_sales/W/short,MoiraiAgent,463.775892811971,463.775892811971,8.903886270251567,0.7164347305947641,0.5190882478694694,0.4550544609393098,7.441705768957756,21.53545664275478,0.9887033482442162,0.4087817738819643,0.343933119501022,Sales,1 -bizitobs_service/10S/medium,MoiraiAgent,11271.791894200249,11271.791894200249,33.3395987831813,0.9882254195837348,0.0632918120045229,0.0556346282844855,9.49408723554683,106.16869545303948,0.0799050620749161,0.025092167695526,0.0190760021088053,Web/CloudOps,2 -ett1/H/short,MoiraiAgent,100.14874707304207,100.14874707304207,4.886711712248091,0.8085546710607726,0.4617481134348747,0.2565582214673985,5.6321609324471575,10.007434589995684,0.467087889080598,0.2280828156000348,0.1773236578321797,Energy,7 -kdd_cup_2018/D/short,MoiraiAgent,2922.8139964494053,2922.8139964494053,21.465333965573112,1.2106190272511297,0.545445043287299,0.4707725420062064,10.182593880678436,54.06305574465252,1.2110698489740632,0.4808462711883572,0.3860798419261956,Nature,1 -m4_hourly/H/short,MoiraiAgent,1065283.3296271868,1065283.3296271868,196.79407577444584,0.691023957521275,0.0872446103571716,0.0778326729384961,5.582512716534011,1032.1256365516683,0.1409079528062008,0.0268667392415626,0.0208442051762901,Econ/Fin,1 -loop_seattle/D/short,MoiraiAgent,17.991199413222905,17.991199413222905,2.8977329674396968,0.87655324451319,0.0540376106704831,0.0534805057732329,8.801016790351719,4.2416034012178585,0.0758031201930282,0.0517863599305544,0.0424757897849006,Transport,1 -m_dense/H/long,MoiraiAgent,33223.679875705464,33223.679875705464,76.5711024846486,0.6614335075754502,0.2670904756625508,0.1862606226141603,7.521675104921542,182.2736401011004,0.3155426159586211,0.1325558976681446,0.1092041454719824,Transport,1 -m_dense/H/medium,MoiraiAgent,35762.746222780166,35762.746222780166,77.44898279728162,0.6705002427206382,0.2649787522385362,0.1866664620191043,8.49549896134553,189.1104074946172,0.3290282174791922,0.1347514454279388,0.1119555065731968,Transport,1 -m4_weekly/W/short,MoiraiAgent,223286.48595297665,223286.48595297665,242.1176576730856,1.8892778387829177,0.0590034242768474,0.0584446347809014,13.060230568936255,472.5319946342011,0.0860879997126043,0.044110081604765,0.0350956841794039,Econ/Fin,1 -solar/H/medium,MoiraiAgent,648.0676275933828,648.0676275933828,11.047900666012298,0.8436119544720273,3.7485605334170526,1.3460297395747725,4.101542185164354,25.45717241944562,0.917026767879644,0.3979711679161126,0.3015323598269475,Energy,1 -solar/H/long,MoiraiAgent,657.1375417682298,657.1375417682298,11.464085175157434,0.8719623643665272,4.656949458072464,1.3547831663390706,4.237806031685371,25.63469410327008,0.888801679617524,0.3974807781169696,0.2989987795767928,Energy,1 -bizitobs_l2c/5T/short,MoiraiAgent,19.02807456309803,19.02807456309803,2.50962542632506,0.2633960017345521,0.1445640307357481,0.2030030684733802,2.7065670493104625,4.36211812805408,0.1498127870574099,0.0861906918957323,0.0696905128733639,Web/CloudOps,7 -hospital/M/short,MoiraiAgent,3210.8135593058914,3210.8135593058914,18.37806001428623,0.7671233183115229,0.1969797306790924,0.1742584924502835,5.155504999322464,56.66404114873816,0.2057036275631754,0.0667166254978022,0.0525390578252888,Healthcare,1 -jena_weather/H/short,MoiraiAgent,1077.0554751142038,1077.0554751142038,8.268799286650014,0.5204620138813819,1.2418139357929376,0.579840413680533,4.794966774744499,32.81852335365203,0.2011754434548723,0.0506872093361884,0.0409843252284757,Nature,21 -hierarchical_sales/D/short,MoiraiAgent,28.870291686621457,28.870291686621457,2.3191254524337745,0.7468318981343345,0.6382011397760713,1.051065047082289,7.99552129325384,5.373108196065054,1.648410090968987,0.711482006041586,0.5824485433968072,Sales,1 -electricity/W/short,MoiraiAgent,35083258662.777435,35083258662.777435,25947.24747589508,1.411048546243956,0.1347660237114963,0.0891211348817585,12.008972425657277,187305.25529941075,0.4273687961514646,0.0592030581282483,0.0464722087335114,Energy,1 -restaurant/D/short,MoiraiAgent,137.63419478435353,137.63419478435353,7.056270440110553,0.6784764777862964,0.6676881454550713,0.3900825698628384,4.732296280896855,11.73176008893608,0.5399227815078514,0.3247459148682288,0.2558508734596628,Sales,1 -ett1/15T/long,MoiraiAgent,9.422389992794113,9.422389992794113,1.5470378183472222,0.981033857099885,0.7531562054576552,0.3792726665056002,7.523447563893045,3.0695911768172177,0.5850671133370122,0.2948669377992203,0.2329642053337012,Energy,7 -ett2/15T/long,MoiraiAgent,11.05902507518202,11.05902507518202,2.104863958521968,0.8745799197263837,0.1326890333710151,0.159253384286456,7.510771242918992,3.325511250196279,0.1772225141348714,0.1121720104297118,0.0905086495937666,Energy,7 -sz_taxi/15T/short,MoiraiAgent,16.558701200517916,16.558701200517916,2.713412677519949,0.541352420905126,1060919047909.5848,0.392490469085212,4.186093973386571,4.069238405465809,0.3804912049284071,0.2537156972053589,0.2008213361949844,Transport,1 -bizitobs_service/10S/short,MoiraiAgent,2013.6883889253145,2013.6883889253145,17.76859489197611,0.7232148011719008,0.0561148362491071,0.045569652013514,5.841026503107757,44.874139422670986,0.0332441984997456,0.0131635437080268,0.010401715241124,Web/CloudOps,2 -ett2/15T/medium,MoiraiAgent,11.216139165001795,11.216139165001795,2.082252240858617,0.8602960222935935,0.1294691975494473,0.1563953319707347,6.85638304139059,3.349050487078658,0.1784769508826012,0.1109669837318822,0.0882731211028097,Energy,7 -ett1/15T/medium,MoiraiAgent,9.211166373617363,9.211166373617363,1.5535133586687937,0.97516976528058,0.7575482724837574,0.3768693774949594,6.841411002774935,3.03499034160199,0.5784721770246549,0.2961011909354774,0.2316135939004746,Energy,7 -loop_seattle/H/medium,MoiraiAgent,59.95017397866822,59.95017397866822,4.199623861113876,0.8629634613624407,0.1126407214818518,0.0957095396856846,7.013925137658528,7.742749768568543,0.1368522365758383,0.074227882257513,0.0587577611976216,Transport,1 -loop_seattle/H/long,MoiraiAgent,54.728499583149045,54.728499583149045,3.9986275471903263,0.8384211436067731,0.1197238625477462,0.0928030737463196,7.73103779251201,7.397871287279135,0.1322098426555045,0.0714608160000083,0.0575171879156625,Transport,1 -kdd_cup_2018/H/long,MoiraiAgent,4105.566404065304,4105.566404065304,24.236422609833472,1.0275908584670568,1.0691830440404957,0.6092836788710563,8.35120906350066,64.07469394437483,1.5034512971361498,0.5686844332386076,0.4514615651130975,Nature,1 -kdd_cup_2018/H/medium,MoiraiAgent,5203.888577877176,5203.888577877176,26.065055544016637,1.0727131239975156,1.0707403520198455,0.5635987296308976,9.229457527299909,72.13798290690679,1.509978643217316,0.5455888232490341,0.4358203209390789,Nature,1 -jena_weather/10T/long,MoiraiAgent,1411.8308823147645,1411.8308823147645,9.569876721662808,0.6490229098861618,0.8228760370905064,0.6110301417857371,6.004536935489068,37.57433808219067,0.2299238118360781,0.058559715139439,0.0476223060729336,Nature,21 -ett2/15T/short,MoiraiAgent,7.697886461454878,7.697886461454878,1.6816792677821857,0.728616320945213,0.1057309039065743,0.1245312281064521,6.14778851806086,2.774506525754603,0.1317157058665319,0.0798353039507806,0.064420357783512,Energy,7 -ett1/15T/short,MoiraiAgent,4.841649423363355,4.841649423363355,1.0774285208367342,0.6838519269347753,0.4776944593229803,0.2390559149076567,4.812270758281873,2.200374837013765,0.4159624465701956,0.2036788441645641,0.1592722497489796,Energy,7 -electricity/D/short,MoiraiAgent,1382539227.4389296,1382539227.4389296,4280.645653167839,1.3769762133998855,0.2364769507687429,0.0882320757655221,11.985107678410165,37182.51238739698,0.611032190946995,0.0703452274787498,0.0558817033080569,Energy,1 -bitbrains_rnd/H/short,MoiraiAgent,1830718.1047719184,1830718.1047719184,144.83140278963333,5.793483950035641,1.1844216256196078,0.5035021397170135,207.29718743113952,1353.040318974981,5.921566435809517,0.6338530800471734,0.6124089749522702,Web/CloudOps,2 -m_dense/H/short,MoiraiAgent,43583.89351095005,43583.89351095005,85.78774213762748,0.7668042028810433,0.2859960120068759,0.2035590437221675,9.845856077590676,208.7675585692137,0.3703016701224414,0.1521660951888979,0.1271604736242572,Transport,1 -jena_weather/10T/medium,MoiraiAgent,1465.044121810083,1465.044121810083,9.6248185725881,0.6160587878697006,0.7595650945636003,0.6277965884459716,5.677193887332573,38.27589478784373,0.2348985249751979,0.0590673503098057,0.0478798992720037,Nature,21 -car_parts/M/short,MoiraiAgent,1.4025056181568585,1.4025056181568585,0.4450336703294434,0.8261044203942538,0.8595463400269872,1.8922814289742744,16.17866119584093,1.1842743002180105,2.83976828854443,1.0671450897004526,0.9345276438391568,Sales,1 -jena_weather/10T/short,MoiraiAgent,782.917292034779,782.917292034779,5.409718375575396,0.2689855130756008,0.3410574809361546,0.5135571717473996,2.2262182097441934,27.980659249466925,0.173794977548585,0.0336011340992691,0.027284411331707,Nature,21 -bitbrains_fast_storage/H/short,MoiraiAgent,4878404.276856472,4878404.276856472,380.5056931232042,0.973322929989456,3.705378308505334,0.5314349305613412,19.483639821014783,2208.710998944061,6.295543886868999,1.0845648395855654,0.8440802232476808,Web/CloudOps,2 -solar/H/short,MoiraiAgent,622.97924053864,622.97924053864,10.62125978975074,0.8182496694810134,4.008891458403775,1.3100783334664758,4.357436060353433,24.959552090104506,0.9200174067965444,0.3915031749529274,0.3032379136007312,Energy,1 -m4_daily/D/short,MoiraiAgent,385173.09420242615,385173.09420242615,173.63991968324504,3.217525327321453,0.039445056656975,0.030095037959494,26.6561346770643,620.6231499085626,0.0958691244228714,0.0268225686833354,0.021439903375178,Econ/Fin,1 -bitbrains_rnd/5T/medium,MoiraiAgent,2255536.29482422,2255536.29482422,143.97818051637063,4.353420753986056,0.4074655698968911,0.6524662182186739,160.04175927199788,1501.8442977966192,6.319024446080794,0.6057895906519503,0.6373970327921428,Web/CloudOps,2 -bitbrains_rnd/5T/long,MoiraiAgent,2446482.0417865925,2446482.0417865925,167.54995487757208,3.274085748188964,1.357909158003714,0.6335628892253924,115.43771155649571,1564.123410024475,5.991804546648539,0.6418461452542831,0.6310167903981543,Web/CloudOps,2 -solar/10T/long,MoiraiAgent,18.28894126091996,18.28894126091996,1.8223210662703535,0.7980520053034176,3.190862832731323,1.404419899486424,4.029003350046246,4.276557173816335,0.924309401871066,0.3938655386380047,0.2978518991941628,Energy,1 -electricity/H/long,MoiraiAgent,7819442.5542399,7819442.5542399,269.90377532486485,1.1607238844406762,0.2897195709071405,0.1313644640114194,10.45507756308273,2796.326617947177,1.102870436103079,0.1064499735073594,0.0850267454347041,Energy,1 -loop_seattle/H/short,MoiraiAgent,41.50817033689937,41.50817033689937,3.4901375251492714,0.7207826650116116,0.100702513930955,0.0805439017196264,6.549942908781301,6.442683473281872,0.1140420513234954,0.0617789845519726,0.0500934032335441,Transport,1 -kdd_cup_2018/H/short,MoiraiAgent,4498.544802038399,4498.544802038399,23.202175202457283,0.973623212120263,0.9418033089617764,0.503392393461108,8.334360566202426,67.07119204277198,1.403921517517034,0.4856635468646081,0.3907892166931287,Nature,1 -solar/10T/medium,MoiraiAgent,19.445901852140217,19.445901852140217,1.8222868522423377,0.7971827731567656,3.57885939940251,1.4121247114517472,4.000905955482016,4.409750769844053,0.9785412324735082,0.4043726993615441,0.3072792592929066,Energy,1 -bitbrains_fast_storage/5T/medium,MoiraiAgent,3459001.260873351,3459001.260873351,290.4120960594848,0.958875376203356,4.364740169399297,0.7649254353883703,21.552631358523332,1859.8390416574632,5.650704652504714,0.8823521528424902,0.7088765178306987,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,MoiraiAgent,4345897.869383571,4345897.869383571,358.91226953150715,0.8720846744357172,4.239064793033548,0.7781968237887354,16.148341735392567,2084.6817189642093,5.509191301365249,0.9484979578746844,0.7779812709018508,Web/CloudOps,2 -electricity/H/medium,MoiraiAgent,5208583.625229657,5208583.625229657,233.0335891886531,1.0439053049978122,0.2317242076712653,0.1197832044123217,9.53078610711172,2282.232158486436,0.8908304153763461,0.0909606887632516,0.0728234126838367,Energy,1 -m4_yearly/A/short,MoiraiAgent,4728377.849018028,4728377.849018028,867.2981508906784,3.2247219974126247,0.1642933680961573,0.1429771888427956,33.278309609647195,2174.483352205307,0.3486884840912626,0.1390752783563608,0.1141283056435453,Econ/Fin,1 -m4_quarterly/Q/short,MoiraiAgent,1728674.678475974,1728674.678475974,544.7411762065888,1.1330936812084451,0.1119620134933035,0.0990553691422616,9.322200066051035,1314.7907356214423,0.2200728168861753,0.0911800805053659,0.0732135629566889,Econ/Fin,1 -solar/10T/short,MoiraiAgent,25.21909661909571,25.21909661909571,2.077295875354469,0.9125814406386376,6.037282440396299,1.496408209751763,6.453731541827825,5.021861867783273,1.4608590451053176,0.6042851334361932,0.4844959349683279,Energy,1 -bitbrains_rnd/5T/short,MoiraiAgent,1610569.5688300766,1610569.5688300766,114.8521813808176,1.6110979500369793,1.0095199141373952,0.586210315455754,52.46989296343748,1269.0821757593465,5.192717925043372,0.4699419725359041,0.3989302913635819,Web/CloudOps,2 -electricity/H/short,MoiraiAgent,1859940.526659586,1859940.526659586,166.59905035054285,0.8857916474096268,0.1504940370493263,0.1136383141251817,10.749727380050793,1363.7963655398066,0.6438092839513999,0.0786466499129525,0.0645433779778622,Energy,1 -m4_monthly/M/short,MoiraiAgent,1831601.5699268072,1831601.5699268072,553.1967798390499,0.912164753268308,0.1598582693000557,0.1308008202229178,8.103887229520714,1353.366753665394,0.281276071183406,0.114973281560979,0.092575070475117,Econ/Fin,1 -temperature_rain/D/short,MoiraiAgent,174.4420753223385,174.4420753223385,5.637379459866857,1.3050658667583694,23.95387878900661,1.4831368082190672,22.49757861777002,13.207652150262684,1.5548898548794776,0.6636686089645835,0.5463483189595792,Nature,1 -bitbrains_fast_storage/5T/short,MoiraiAgent,1846933.933235437,1846933.933235437,156.9393220212747,0.6466780843540297,1.974195468926736,0.6847807978641088,13.225017003718513,1359.0194749286843,4.266985018044951,0.4927506545423507,0.4109376016582722,Web/CloudOps,2 -loop_seattle/5T/long,MoiraiAgent,89.19992493442076,89.19992493442076,5.442245962289807,0.8626336595383874,0.1713474284752358,0.1228051325082958,7.024216536755028,9.444571188488164,0.1669819596556889,0.0962200271007625,0.0776591104062574,Transport,1 -loop_seattle/5T/medium,MoiraiAgent,72.35018331432613,72.35018331432613,4.953836101361572,0.790429736224429,0.1595618444656121,0.1131743156975235,6.261203975998,8.505891094666456,0.1513372799370917,0.0881389230699556,0.0707967925811483,Transport,1 -loop_seattle/5T/short,MoiraiAgent,42.680793834892526,42.680793834892526,3.4930809898592217,0.5528067089401885,0.0967539992402924,0.0745168231066803,4.554367439560088,6.5330539439754,0.1120274518246318,0.0598986271454036,0.0479448018955368,Transport,1 -electricity/15T/long,MoiraiAgent,304465.72035074496,304465.72035074496,56.640382835551726,0.858409030333623,0.1260704083702485,0.1232076127408403,7.41599648397349,551.7841247723107,0.8705070904850035,0.08935714648637,0.0718638443955452,Energy,1 -electricity/15T/short,MoiraiAgent,130170.61075508365,130170.61075508365,48.42047416709254,0.9307153819128722,0.157745732438964,0.1501397690141391,11.133849978114135,360.79164451949777,0.7111914657905786,0.0954463012662933,0.0808629554539468,Energy,1 -electricity/15T/medium,MoiraiAgent,235968.520460288,235968.520460288,51.35811539593971,0.811049395204578,0.1237151851778871,0.1220334927255913,7.329091044427192,485.7659111756279,0.8370235686187709,0.0884952032187822,0.0717627472668905,Energy,1 diff --git a/results/MoiraiAgent/config.json b/results/MoiraiAgent/config.json deleted file mode 100644 index 1962fbf31a1d432ddc309d5c877a474d1e73aeab..0000000000000000000000000000000000000000 --- a/results/MoiraiAgent/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "MoiraiAgent", - "model_type": "agentic", - "model_dtype": "bf16", - "model_link": "https://huggingface.co/Salesforce/moirai-agent", - "code_link": "https://github.com/SalesforceAIResearch/uni2ts/tree/main/project/moirai-agent/gift_eval", - "org": "Salesforce AI Research", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/Moirai_base/config.json b/results/Moirai_base/config.json index 436cba8138127f1c5a08973f16a2a616962bf26f..f29995c47e4cec59b8afe69863dd78883db13420 100644 --- a/results/Moirai_base/config.json +++ b/results/Moirai_base/config.json @@ -1,10 +1,7 @@ { "model": "Moirai_base", - "model_type": "zero-shot", + "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/Salesforce/moirai-1.1-R-base", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai.ipynb", - "org": "Salesforce AI Research", - "testdata_leakage": "No", - "replication_code_available": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai.ipynb" } \ No newline at end of file diff --git a/results/Moirai_large/config.json b/results/Moirai_large/config.json index c17b8d2e9909b35baf0a7373e4a73b03a2c2579f..fa603bd566a01af6fde3d47879cef1265d05b6af 100644 --- a/results/Moirai_large/config.json +++ b/results/Moirai_large/config.json @@ -1,10 +1,7 @@ { "model": "Moirai_large", - "model_type": "zero-shot", + "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/Salesforce/moirai-1.1-R-large", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai.ipynb", - "org": "Salesforce AI Research", - "testdata_leakage": "No", - "replication_code_available": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai.ipynb" } \ No newline at end of file diff --git a/results/Moirai_small/config.json b/results/Moirai_small/config.json index 615e202e60d14d400d7908b7656e35f51509b5fa..a21cb00fc74c0a950e5796e844ae7d8fe6cb63a2 100644 --- a/results/Moirai_small/config.json +++ b/results/Moirai_small/config.json @@ -1,10 +1,7 @@ { "model": "Moirai_small", - "model_type": "zero-shot", + "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/Salesforce/moirai-1.1-R-large", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai.ipynb", - "org": "Salesforce AI Research", - "testdata_leakage": "No", - "replication_code_available": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai.ipynb" } \ No newline at end of file diff --git a/results/N-BEATS/config.json b/results/N-BEATS/config.json index 9ecbb6030f846102118f8a1f4e2bf1deece24ae3..f08e6a72d696eeb49059663f377f17282f1ee742 100644 --- a/results/N-BEATS/config.json +++ b/results/N-BEATS/config.json @@ -1,8 +1,5 @@ { "model": "N-BEATS", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "ServiceNow", - "testdata_leakage": "No", - "replication_code_available": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/PatchTST-FM-r1/all_results.csv b/results/PatchTST-FM-r1/all_results.csv deleted file mode 100644 index d225b0ee31439a6765b7cce466d9bb3c48dcc40e..0000000000000000000000000000000000000000 --- a/results/PatchTST-FM-r1/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/5T/long,PatchTST-FM-r1,3963596.2731543984,3963596.2731543984,352.5434826670514,0.8797694476386504,4.1081854286411685,0.8293920098845844,16.133208144852304,1990.8782667843852,5.261296865393238,0.9316671698199084,0.6834124649451816,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,PatchTST-FM-r1,2921895.543833521,2921895.543833521,266.30640919626904,0.9766616998331876,3.337360413806344,0.8158660437705388,20.803837593154928,1709.355300642181,5.193493808348751,0.8091124687563876,0.6216008248673692,Web/CloudOps,2 -bitbrains_fast_storage/5T/short,PatchTST-FM-r1,1697146.698417202,1697146.698417202,160.63920341217968,0.687624021348568,1.7001684981853635,0.7549356647945814,13.373434108661632,1302.745830320405,4.090299768255742,0.5043673763500158,0.3900113374237228,Web/CloudOps,2 -bitbrains_fast_storage/H/short,PatchTST-FM-r1,2518220.652597486,2518220.652597486,283.5538990137864,1.0395623242658685,3.523696400636635,0.5770457850160442,22.46941555384954,1586.8902459204562,4.523152559246631,0.8082206991351422,0.6349661997734519,Web/CloudOps,2 -bitbrains_rnd/5T/long,PatchTST-FM-r1,2195655.983634115,2195655.983634115,152.48983685401197,3.295938909839324,0.9695883816904296,0.695676242221646,115.5016216331991,1481.774606218542,5.676344924715512,0.5841542349722959,0.5610908192920235,Web/CloudOps,2 -bitbrains_rnd/5T/medium,PatchTST-FM-r1,2252246.626834685,2252246.626834685,144.54221157691424,4.373931127052579,0.3871518981710227,0.7209210436750704,159.12556401543253,1500.7486887666053,6.314413932837834,0.6081626867284261,0.595316550063181,Web/CloudOps,2 -bitbrains_rnd/5T/short,PatchTST-FM-r1,1631778.1516590363,1631778.1516590363,121.93695315818664,1.654129640210054,1.0066700704954157,0.6739013568166801,51.15528638640983,1277.410721600158,5.226795884019823,0.498930809096983,0.3990364964406948,Web/CloudOps,2 -bitbrains_rnd/H/short,PatchTST-FM-r1,1854241.5317335152,1854241.5317335152,155.89833127317675,5.786635360098222,1.623052644088761,0.56780917666553,193.90873746621847,1361.7053762593123,5.959489226580281,0.6822874036207192,0.6155677518047236,Web/CloudOps,2 -bizitobs_application/10S/long,PatchTST-FM-r1,4451116.373333333,4451116.373333333,1301.9711111111112,2.785832170430785,0.0401415210300021,0.0408913718329535,40.46877111177244,2109.766900236453,0.0813652763628114,0.0502118216282943,0.0437200696812821,Web/CloudOps,2 -bizitobs_application/10S/medium,PatchTST-FM-r1,3442616.7466666666,3442616.7466666666,1015.4584375,2.199273533026733,0.0317380332946777,0.0323016611735026,30.304967364082973,1855.4289926231795,0.0728572356864973,0.0398740641678682,0.0344653873934104,Web/CloudOps,2 -bizitobs_application/10S/short,PatchTST-FM-r1,396251.9111111111,396251.9111111111,335.6299305555556,0.8708288126948269,0.0129473718007405,0.0130046102735731,6.555705093572008,629.4854335972447,0.0242767370486586,0.0129439048703548,0.0101856385798053,Web/CloudOps,2 -bizitobs_l2c/5T/long,PatchTST-FM-r1,157.21184523809524,157.21184523809524,8.811754092261905,0.8840677895623555,0.6809548739967544,0.9995436507936508,4.642079122308026,12.53841478170567,0.7040132450290019,0.4947668186844734,0.3626443482743184,Web/CloudOps,7 -bizitobs_l2c/5T/medium,PatchTST-FM-r1,100.75716411564626,100.75716411564626,6.5881816007653065,0.6647171353147934,0.4385002691074258,0.7865620515784438,4.170532367329468,10.037786813618142,0.528135198444038,0.3466352356064204,0.2689970352129537,Web/CloudOps,7 -bizitobs_l2c/5T/short,PatchTST-FM-r1,20.210028366815475,20.210028366815475,2.596833147321429,0.265623910275975,0.1404599538346777,0.2040685017903646,2.9512146631857896,4.49555651358266,0.1543956073159361,0.0891857614663426,0.0723064029577012,Web/CloudOps,7 -bizitobs_l2c/H/long,PatchTST-FM-r1,83.54987599206349,83.54987599206349,5.243330698164683,0.5644076179043155,0.5760710067074558,0.7978568909660219,4.17785550972483,9.140562126700058,0.5583319159857263,0.3202777722282121,0.2538278972375415,Web/CloudOps,7 -bizitobs_l2c/H/medium,PatchTST-FM-r1,76.89467075892857,76.89467075892857,4.865476771763393,0.4960727680655011,0.4530371210146007,0.7660351707821801,3.580428882335067,8.768960643025407,0.5309738862215401,0.2946120087651027,0.2315709156645362,Web/CloudOps,7 -bizitobs_l2c/H/short,PatchTST-FM-r1,61.47789171006944,61.47789171006944,4.738043406653026,0.474215010550257,0.3937309137065348,0.6240475366985987,2.783165169019389,7.840783870893869,0.4226383312357205,0.2553926739574771,0.1951113630657894,Web/CloudOps,7 -bizitobs_service/10S/long,PatchTST-FM-r1,128241.6965079365,128241.6965079365,72.5991798941799,1.2238110706500405,0.0855837609023644,0.0777257267381779,13.928026600548078,358.10849823473404,0.2652982239332185,0.053783793403052,0.047989600673952,Web/CloudOps,2 -bizitobs_service/10S/medium,PatchTST-FM-r1,18247.91619047619,18247.91619047619,36.66415178571429,0.9934111057685164,0.0740190148732018,0.0610556950644841,9.588169038381151,135.08484811582753,0.1016680399812409,0.0275943046286886,0.0218422475219499,Web/CloudOps,2 -bizitobs_service/10S/short,PatchTST-FM-r1,2442.651216931217,2442.651216931217,18.5728835978836,0.6732427289396902,0.0585957586702215,0.0500974269644923,5.304329267074427,49.4231850140318,0.0366142754779052,0.0137593859294764,0.0108681388895428,Web/CloudOps,2 -car_parts/M/short,PatchTST-FM-r1,40.05825960137256,40.05825960137256,0.7061695409546466,2.008152544170945,0.8591699306386479,1.908908106597499,31.17455820027039,6.329159470369866,15.176675161985978,1.693322120027278,1.4839786785310778,Sales,1 -covid_deaths/D/short,PatchTST-FM-r1,287003.0756892231,287003.0756892231,103.20888157894736,31.882086235616217,0.0865231223321659,0.3588205437911184,551.7273442670315,535.7266800237068,0.2015697705047671,0.0388328439662581,0.0311557439650033,Healthcare,1 -electricity/15T/long,PatchTST-FM-r1,327372.61426426424,327372.61426426424,57.530309215465465,0.8614837323850127,0.1266968880795095,0.1298965532915728,7.14277252413473,572.1648488541255,0.9026601704201124,0.0907611133832402,0.0724784463493436,Energy,1 -electricity/15T/medium,PatchTST-FM-r1,268713.65016216214,268713.65016216214,52.29111204954955,0.8138964655774179,0.1214987090762035,0.1279756491721213,7.1466404202918055,518.3759737508695,0.8932140023438907,0.0901028516866377,0.0726879971462578,Energy,1 -electricity/15T/short,PatchTST-FM-r1,144612.09912162164,144612.09912162164,52.52096688485361,1.0135635292888876,0.1651946560334545,0.1675993817990964,10.808356687292893,380.2789753873091,0.7496048188620089,0.1035291783567063,0.0860886827995823,Energy,1 -electricity/D/short,PatchTST-FM-r1,1380235345.92,1380235345.92,4278.698522522523,1.3954907142785902,0.2443172257083966,0.0937556746199324,11.688438826630987,37151.51875657306,0.6105228337145602,0.070313226323179,0.0556802404124095,Energy,1 -electricity/H/long,PatchTST-FM-r1,6958948.721009009,6958948.721009009,254.90554354354356,1.1327494687445716,0.285434901871913,0.1365553209459459,9.54274180245952,2637.9819409937227,1.0404192972821875,0.1005346709792122,0.0792597652041922,Energy,1 -electricity/H/medium,PatchTST-FM-r1,5163774.81009009,5163774.81009009,235.77881052927927,1.0426958869592484,0.2308472162027709,0.1270900884404912,8.690971117446248,2272.394070158187,0.8869902906789597,0.0920322396690405,0.0728864767337134,Energy,1 -electricity/H/short,PatchTST-FM-r1,2055950.8381981985,2055950.8381981985,190.167060106982,1.0194697392006693,0.1994799301410361,0.1344485116219735,10.747430506719658,1433.8587232353814,0.6768837147804688,0.0897724329379163,0.0724728514547487,Energy,1 -electricity/W/short,PatchTST-FM-r1,40931973500.07928,40931973500.07928,27890.36756756757,1.592060495291272,0.2510549388354338,0.1034785073082726,12.018606458627303,202316.51810981543,0.4616195296569064,0.0636366149377407,0.0503978301945591,Energy,1 -ett1/15T/long,PatchTST-FM-r1,8.820806671626984,8.820806671626984,1.5571654885912698,0.9975315893139254,0.7794792608948488,0.3832854740203373,7.063422218259968,2.969984288111132,0.5660819418640551,0.2967972817613952,0.2293434859211032,Energy,7 -ett1/15T/medium,PatchTST-FM-r1,9.884695560515873,9.884695560515873,1.6121971106150794,1.003133508203864,0.7447542601493071,0.388727058531746,6.720399342043071,3.1439935687777534,0.5992482828095069,0.3072863633312439,0.2326261389563056,Energy,7 -ett1/15T/short,PatchTST-FM-r1,5.484738304501488,5.484738304501488,1.091576857793899,0.6922614354916726,0.5141553577743448,0.2288527352469308,4.550091010463932,2.3419518151536525,0.4427263771752645,0.2063534631807721,0.1599439473136791,Energy,7 -ett1/D/short,PatchTST-FM-r1,52220.23492063492,52220.23492063492,153.1155009920635,1.8001913783207997,1.470210193452381,0.489733402312748,10.444820194981355,228.51747180606327,0.6002087197006059,0.4021629422487111,0.3055318845617347,Energy,7 -ett1/H/long,PatchTST-FM-r1,158.34277447089949,158.34277447089949,7.678492373511904,1.476379937529955,2289177229767.006,0.5163721155237269,8.690766933202426,12.583432539291474,0.6025933304839346,0.3677063693076039,0.2779044900053851,Energy,7 -ett1/H/medium,PatchTST-FM-r1,136.15321800595237,136.15321800595237,6.8933128720238095,1.2391395052311946,2776759156259.317,0.4509731837681361,8.411404023419772,11.668471108330875,0.5588921060758472,0.3301733460294125,0.2513947555266528,Energy,7 -ett1/H/short,PatchTST-FM-r1,95.67706473214284,95.67706473214284,4.817394438244048,0.8136873272543726,0.4326001848493304,0.2589340391613188,5.780271307237646,9.781465367323182,0.4565410063252634,0.2248475072098057,0.1751606472517254,Energy,7 -ett1/W/short,PatchTST-FM-r1,1618740.5714285714,1618740.5714285714,983.8065708705356,1.6549116604989678,0.7051722662789481,0.5871509824480329,8.584535609935498,1272.29735967209,0.506277486287478,0.391480115797622,0.2935765810215489,Energy,7 -ett2/15T/long,PatchTST-FM-r1,11.442535962301587,11.442535962301587,2.158029978918651,0.8984678195818592,0.1357782238154739,0.1661544751364087,7.491029551011065,3.3826817707702843,0.1802692346624073,0.1150053238941272,0.0923703419369555,Energy,7 -ett2/15T/medium,PatchTST-FM-r1,11.430755208333334,11.430755208333334,2.1125906808035717,0.8772920978515794,0.1309846853334912,0.16078614250062,7.02294934282655,3.380939988868973,0.1801764119520326,0.1125837814464801,0.0895866785198107,Energy,7 -ett2/15T/short,PatchTST-FM-r1,7.991065615699405,7.991065615699405,1.77919921875,0.7658749480647952,0.1079764555964452,0.1286532447451636,5.10569469682148,2.826847292603441,0.1342004973644258,0.084464916336796,0.0660844741474704,Energy,7 -ett2/D/short,PatchTST-FM-r1,99528.62222222224,99528.62222222224,192.0461061507937,1.2928396779202158,0.3630573575458829,0.1375210716610863,12.33687510343608,315.4815719217562,0.18055842731133,0.1099130535156965,0.0903384473694837,Energy,7 -ett2/H/long,PatchTST-FM-r1,212.4878306878307,212.4878306878307,8.849791253306877,0.995040831417387,0.1763480615466066,0.1675084835637814,8.024874240453391,14.576962327173336,0.1996202798118545,0.1211910799116531,0.0973925723377336,Energy,7 -ett2/H/medium,PatchTST-FM-r1,235.9010044642857,235.9010044642857,9.475467354910714,1.0135360334555097,0.1687032991425204,0.1636167617071242,7.411102795971758,15.35906912753132,0.2069733335412375,0.1276880160524587,0.1005695463925868,Energy,7 -ett2/H/short,PatchTST-FM-r1,112.50398995535714,112.50398995535714,6.452224004836309,0.7348755480676931,0.1180113335980585,0.1087252843947637,5.3846243585810765,10.606789804429855,0.134523607087625,0.081832153070992,0.0652608323015423,Energy,7 -ett2/W/short,PatchTST-FM-r1,3796286.285714286,3796286.285714286,1341.316545758929,0.896233871436805,0.1455636024475097,0.1637557574680873,9.78554078659539,1948.406088502673,0.1634413999394827,0.1125158945532261,0.0942490967744963,Energy,7 -hierarchical_sales/D/short,PatchTST-FM-r1,29.04666701116164,29.04666701116164,2.3304671351798265,0.7482468357996437,0.6521103956086343,1.0820840546196775,6.799613286550546,5.389495988602426,1.653437684235598,0.714961508706515,0.5784059482788666,Sales,1 -hierarchical_sales/W/short,PatchTST-FM-r1,449.9978151483051,449.9978151483051,8.871621342028602,0.7148419039906404,0.5365570209452338,0.4554052191265559,6.752798568321068,21.21315193808561,0.9739061816018972,0.4073004752452977,0.3438194455876778,Sales,1 -hospital/M/short,PatchTST-FM-r1,3525.712081703607,3525.712081703607,18.84233417535854,0.7871073345853571,0.1975924588451692,0.1796070879721113,5.251376847291895,59.37770694211429,0.2155548645447793,0.0684020485135335,0.0537095499048737,Healthcare,1 -jena_weather/10T/long,PatchTST-FM-r1,1395.4095238095238,1395.4095238095238,9.642588458994709,0.64635107807728,0.8499680878467298,0.6306441049727183,5.568196948263015,37.35518068233005,0.2285827559428275,0.0590046522093821,0.0475573113702345,Nature,21 -jena_weather/10T/medium,PatchTST-FM-r1,1452.8141414141414,1452.8141414141414,9.336882778679652,0.587514197278201,0.7889349291164726,0.6405601072104978,5.386835581456005,38.11579910501866,0.233916007505432,0.0573002899956883,0.0465717677920455,Nature,21 -jena_weather/10T/short,PatchTST-FM-r1,783.411507936508,783.411507936508,5.326362537202381,0.2604325581669295,0.3530425746504508,0.5328485398065477,2.3377619256880986,27.989489240365,0.1738498080082676,0.0330833870001333,0.0274343014370998,Nature,21 -jena_weather/D/short,PatchTST-FM-r1,413.3138392857143,413.3138392857143,9.773030598958334,1.022871776521573,0.6221875805540134,0.4508122035435268,6.614118562949505,20.33012147739689,0.1224205593367543,0.0588496174835885,0.0474247722581613,Nature,21 -jena_weather/H/long,PatchTST-FM-r1,1293.9759259259258,1293.9759259259258,11.156355406746032,0.8317077235200512,2.449478274828768,0.640890712838955,6.095109131172405,35.971876875219145,0.2165905529546639,0.0671736199611674,0.0536240804508476,Nature,21 -jena_weather/H/medium,PatchTST-FM-r1,1330.7748015873017,1330.7748015873017,10.283680555555556,0.7337128345261501,2.480368045492966,0.6553655230809772,5.799961582485118,36.47978620533982,0.2232262815934479,0.0629276651620147,0.0516748556370981,Nature,21 -jena_weather/H/short,PatchTST-FM-r1,1154.8640350877192,1154.8640350877192,8.614584965016709,0.5219028885417614,1.752417190877734,0.6029011942290622,4.464025759535787,33.983290527665496,0.2083153831988647,0.0528068512560814,0.0432712982592907,Nature,21 -kdd_cup_2018/D/short,PatchTST-FM-r1,2981.430414256267,2981.430414256267,21.68049321763415,1.2302623916034487,0.5222843133893543,0.4838005820283928,9.35697453068686,54.602476264875264,1.223153409319229,0.4856660541589855,0.3862128416556334,Nature,1 -kdd_cup_2018/H/long,PatchTST-FM-r1,4197.730831199723,4197.730831199723,24.219867765011923,1.0297069345421592,0.96836268276246,0.6232839881123265,7.680827563940429,64.78989760139865,1.5202328068624802,0.5682959676949285,0.449173123685721,Nature,1 -kdd_cup_2018/H/medium,PatchTST-FM-r1,5391.546383609852,5391.546383609852,25.88762928216312,1.060904092775483,0.8903413105588154,0.5846818051673373,8.397398244563831,73.42715018036483,1.536963236039264,0.5418749655565148,0.4330946122680242,Nature,1 -kdd_cup_2018/H/short,PatchTST-FM-r1,5032.095442817463,5032.095442817463,23.826647401412067,0.9751485424073102,0.8176087371616299,0.5308170243529172,7.412632842619154,70.93726413400408,1.4848454159571625,0.4987348836118325,0.3980819380349292,Nature,1 -loop_seattle/5T/long,PatchTST-FM-r1,80.09930053892903,80.09930053892903,4.9528165491342735,0.7837556080896918,0.138813854489164,0.1129112301216173,6.173471946681516,8.94982125737319,0.1582346701705778,0.0875668094959991,0.0695593905373264,Transport,1 -loop_seattle/5T/medium,PatchTST-FM-r1,65.28651767285862,65.28651767285862,4.535457351731811,0.7216607980296514,0.1329069843430022,0.1041806390878462,5.403311552079315,8.08000728173302,0.1437599288359008,0.0806951037776023,0.0637276553609565,Transport,1 -loop_seattle/5T/short,PatchTST-FM-r1,42.45603160877838,42.45603160877838,3.442128026940628,0.5445497668427377,0.08762463194667,0.0740097630503746,3.934908631499723,6.515829310899602,0.1117320907740679,0.0590248981075587,0.0463043203093516,Transport,1 -loop_seattle/D/short,PatchTST-FM-r1,18.432232004643964,18.432232004643964,2.941358278508772,0.8901067596772662,0.0546490837545955,0.0542572052978263,7.615497996735771,4.293277536410145,0.076726611560882,0.0525660063162913,0.0431926587245215,Transport,1 -loop_seattle/H/long,PatchTST-FM-r1,62.1328173374613,62.1328173374613,4.138070551685587,0.8652544148117627,0.1165373149671052,0.0960810264689757,7.441994288448565,7.882437271394001,0.1408697071228603,0.0739528608473901,0.0600755057884975,Transport,1 -loop_seattle/H/medium,PatchTST-FM-r1,70.24130546955624,70.24130546955624,4.40032007868937,0.8986605913994742,0.1136999171584752,0.1011860346523477,8.10584950772152,8.381008618868986,0.1481333724591721,0.0777751560460787,0.0635027094069052,Transport,1 -loop_seattle/H/short,PatchTST-FM-r1,61.7024855215632,61.7024855215632,4.091126738091359,0.828937412272446,0.1088085940152116,0.0946005728559332,7.5842923412593395,7.855092967085953,0.1390431334669908,0.0724171035859602,0.0590219373751063,Transport,1 -m4_daily/D/short,PatchTST-FM-r1,387628.1284261043,387628.1284261043,192.83329425631143,3.5334215705645766,0.0426666076437377,0.0322980904910333,27.05273123660975,622.5978866219386,0.0961741677046019,0.0297874149259662,0.0235971120250444,Econ/Fin,1 -m4_hourly/H/short,PatchTST-FM-r1,765326.0161030596,765326.0161030596,180.656841284219,0.7843335370439745,0.0878497506109412,0.07911200531055,6.072899646240558,874.8291353761942,0.1194334868349889,0.024663646422671,0.0202622220296083,Econ/Fin,1 -m4_monthly/M/short,PatchTST-FM-r1,1918470.7778148148,1918470.7778148148,576.5575244502314,0.9653153274157986,0.1667570336306536,0.1361200570706968,8.101922861206269,1385.08872561104,0.2878689879464792,0.1198284470788653,0.0959990802466031,Econ/Fin,1 -m4_quarterly/Q/short,PatchTST-FM-r1,1902028.044,1902028.044,579.3026875,1.221790308597588,0.1173429940541585,0.105816264629364,9.790244113690177,1379.1403278854548,0.2308438050160155,0.0969650686986586,0.0775696831680223,Econ/Fin,1 -m4_weekly/W/short,PatchTST-FM-r1,276633.53417613026,276633.53417613026,258.53401542746946,2.112752896936325,0.0623103424392677,0.0614034989060491,15.3108742224459,525.9596316982229,0.0958216939206513,0.0471008910215792,0.0378293568553451,Econ/Fin,1 -m4_yearly/A/short,PatchTST-FM-r1,4325126.685818751,4325126.685818751,979.111774904965,3.575231155520207,0.1787096053834945,0.1703265379885249,42.54209557331665,2079.693892335781,0.3334885557321239,0.1570051116256554,0.1304166953694127,Econ/Fin,1 -m_dense/D/short,PatchTST-FM-r1,7901.029629629629,7901.029629629629,43.54366898148148,0.6755029487833056,0.121526764798288,0.098941492151331,8.133892530392894,88.88773610363597,0.1538778194139052,0.0753805319593148,0.0648268116345247,Transport,1 -m_dense/H/long,PatchTST-FM-r1,34384.05135802469,34384.05135802469,76.58469135802468,0.6712664328913986,0.2522273419600839,0.1853342918113426,6.820154754602706,185.4293702681015,0.3210056513794463,0.1325794220167594,0.1093831220925856,Transport,1 -m_dense/H/medium,PatchTST-FM-r1,36726.66666666666,36726.66666666666,78.02033854166666,0.6778659860750951,0.2542899068444708,0.189529062906901,7.449369088025636,191.64202740178536,0.3334329162814731,0.1357455322400402,0.1124177656189065,Transport,1 -m_dense/H/short,PatchTST-FM-r1,44221.24444444444,44221.24444444444,85.92638020833333,0.7676417394746983,0.284895783396381,0.2090252346462673,7.72623371468542,210.28847910535765,0.3729993614025479,0.1524119870080143,0.1256220528424411,Transport,1 -restaurant/D/short,PatchTST-FM-r1,143.88219155692536,143.88219155692536,7.1258807030447375,0.6892413165154886,0.5611537720650042,0.4063485956909031,4.8821726184758125,11.995090310494763,0.5520418484339751,0.3279495404538186,0.2585072165514963,Sales,1 -saugeen/D/short,PatchTST-FM-r1,1194.380520833333,1194.380520833333,12.849480794270834,2.8548305268853613,0.2823316955566406,0.3275675964355469,27.512098428315984,34.55981077542719,1.1194005054238605,0.4161977445133498,0.3383299869241916,Nature,1 -saugeen/M/short,PatchTST-FM-r1,402.68968563988096,402.68968563988096,12.711325509207589,0.7447688924089432,0.3636894907270159,0.3647028605143229,4.314330401393504,20.067129481813808,0.6024754278686558,0.3816321254057805,0.2905929412022678,Nature,1 -saugeen/W/short,PatchTST-FM-r1,918.7890625,918.7890625,14.161715698242189,1.1490605546673125,0.3720211029052734,0.3698776483535766,8.409806822915495,30.31153348974611,0.9149788431958076,0.4274831641767286,0.3424756516076759,Nature,1 -solar/10T/long,PatchTST-FM-r1,19.37535965695965,19.37535965695965,1.905439672910964,0.8333014524345763,2.974107765294077,1.4585635412719755,4.225786644060472,4.4017450695104605,0.9513666809047956,0.411830260193426,0.3118143391638325,Energy,1 -solar/10T/medium,PatchTST-FM-r1,17.947770819508957,17.947770819508957,1.852772262428113,0.8121506950944682,3.1657445386993444,1.4539580914067685,4.045358151154874,4.2364809476154806,0.9400919107210144,0.4111375072505182,0.3057631638658959,Energy,1 -solar/10T/short,PatchTST-FM-r1,17.20347665754258,17.20347665754258,1.837988459454836,0.8057920521254259,3.886132657638197,1.5104036577136557,3.739968080925874,4.147707397773206,1.2065676328434112,0.5346706438136236,0.3988353127826964,Energy,1 -solar/D/short,PatchTST-FM-r1,131709.2905109489,131709.2905109489,249.1885796836983,0.971375037839436,1.0949221126064477,0.4216084352664994,5.7565700468672185,362.91774620559534,0.5242925318947654,0.3599926229223684,0.2793463047838657,Energy,1 -solar/H/long,PatchTST-FM-r1,768.3583130575831,768.3583130575831,12.425672901459857,0.9442015292325132,4.637481253240181,1.4066143298864555,5.330198046557008,27.71927692162231,0.96107784781932,0.4308207246389802,0.3197422995365536,Energy,1 -solar/H/medium,PatchTST-FM-r1,844.8163625304136,844.8163625304136,12.003270415145986,0.9147287735275224,4.629227237321221,1.4018422815921532,5.177123574494133,29.06572487536503,1.0470152497445122,0.4323858161238152,0.327753810951106,Energy,1 -solar/H/short,PatchTST-FM-r1,826.1417595082597,826.1417595082597,11.752417833189268,0.9043902556729688,4.468420114475848,1.402995041778717,4.75528956276132,28.74268184265796,1.0594648549261745,0.4331980475180401,0.3276557075139752,Energy,1 -solar/W/short,PatchTST-FM-r1,1451483.094890511,1451483.094890511,963.816605839416,1.053115620781112,0.2145745493199703,0.1889469397329066,8.185293169843082,1204.775122124669,0.2459247232902412,0.196739065856201,0.1466862508824363,Energy,1 -sz_taxi/15T/long,PatchTST-FM-r1,15.44105791488604,15.44105791488604,2.672480413105413,0.5045377086171898,5746401936368.246,0.3964917131298967,3.857780427649524,3.929511154696732,0.3637085527729245,0.2473600265017121,0.1957735804382971,Transport,1 -sz_taxi/15T/medium,PatchTST-FM-r1,16.249744591346154,16.249744591346154,2.7377852480635685,0.5336850404624458,9351514889105.725,0.4049061310596955,3.749768880248773,4.031097194480202,0.3751608790193476,0.2547966150843886,0.2003979898179049,Transport,1 -sz_taxi/15T/short,PatchTST-FM-r1,16.707357665121336,16.707357665121336,2.731934488038004,0.5451107607894305,1219746130062.6614,0.3971493829300989,3.853095282488541,4.087463475692638,0.3821953163344857,0.2554475586314184,0.2007466596647954,Transport,1 -sz_taxi/H/short,PatchTST-FM-r1,7.104631994524572,7.104631994524572,1.8419471153846156,0.560211836972062,1.292352268838475,0.2964402875329694,3.910919047387869,2.665451555463834,0.2483032605938959,0.1715887402470281,0.1349215457159464,Transport,1 -temperature_rain/D/short,PatchTST-FM-r1,189.86368681957345,189.86368681957345,5.822427224689967,1.3460665476622526,16.810589748785073,1.5273758866404197,18.631840008157308,13.779103266162622,1.6221647444367997,0.6854536168645327,0.5539757347660184,Nature,1 -us_births/D/short,PatchTST-FM-r1,157440.86666666667,157440.86666666667,239.2118489583333,0.3523441113129549,0.0230890274047851,0.0227139393488566,3.3486796709768822,396.7881886682953,0.037195672937214,0.0224241697476499,0.018319605659466,Healthcare,1 -us_births/M/short,PatchTST-FM-r1,18822784.0,18822784.0,3174.5885416666665,0.3596739348155098,0.0098161771893501,0.0097802517314751,3.907856905823244,4338.523251061356,0.0134754236381187,0.0098602503663454,0.0089355789405207,Healthcare,1 -us_births/W/short,PatchTST-FM-r1,2579948.285714286,2579948.285714286,1242.8823939732142,1.1299754734535108,0.016774442579065,0.0168314299413136,7.115378921930659,1606.2217423862392,0.0218043026557427,0.016872006628067,0.0134476966636985,Healthcare,1 diff --git a/results/PatchTST-FM-r1/config.json b/results/PatchTST-FM-r1/config.json deleted file mode 100644 index a2bbae946ac74b9dc6ca2a86fca2fec5d2d21fc5..0000000000000000000000000000000000000000 --- a/results/PatchTST-FM-r1/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "PatchTST-FM-r1", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/ibm-research/patchtst-fm-r1", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/patchtst_fm.ipynb", - "org": "IBM TSFM & Rensselaer Polytechnic Institute", - "replication_code_available": "Yes", - "testdata_leakage": "No" -} \ No newline at end of file diff --git a/results/PatchTST/config.json b/results/PatchTST/config.json index 453de6b0bbc8baeb98f4d68eea430f047ab408c3..cee967e019ae586779525f1d408cacba18e8810e 100644 --- a/results/PatchTST/config.json +++ b/results/PatchTST/config.json @@ -1,8 +1,5 @@ { "model": "PatchTST", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "Princeton University", - "testdata_leakage": "No", - "replication_code_available": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/Samay/all_results.csv b/results/Samay/all_results.csv deleted file mode 100644 index 04826116ce615912bd25523104267a2709246a8a..0000000000000000000000000000000000000000 --- a/results/Samay/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,Samay,3426374.766587111,3426374.766587111,873.9192621753791,3.2812051451008624,0.1640653483397955,0.1453872914195184,27.91824536699693,1851.046937975132,0.296823953290195,0.1401370029757891,0.1125265291142284,Econ/Fin,1 -m4_quarterly/Q/short,Samay,1743054.9953987706,1743054.9953987706,552.2917557136882,1.1722414213321029,0.1117725308902263,0.1002146409640564,8.902215490103176,1320.248081005525,0.2209862811029966,0.0924439148482291,0.0736583340607427,Econ/Fin,1 -m4_monthly/M/short,Samay,1785273.218643579,1785273.218643579,542.8164132020478,0.9094723812315676,0.1540991551309116,0.1278176097959187,7.423193079347096,1336.1411671839091,0.2776960022400349,0.1128158847273476,0.0897767024948638,Econ/Fin,1 -m4_weekly/W/short,Samay,230534.24410321692,230534.24410321692,246.5164928890755,1.972384306713629,0.0598806139027559,0.0597725486591791,14.818910863946314,480.13981724412,0.0874740338123158,0.0449114846547057,0.0357992834570237,Econ/Fin,1 -m4_daily/D/short,Samay,347432.0132320521,347432.0132320521,174.8310844889236,3.2584015299972577,0.0382408111964769,0.029908374190613,24.42125810030524,589.4336376828626,0.0910512072774208,0.0270065708752476,0.0212990463272123,Econ/Fin,1 -m4_hourly/H/short,Samay,663108.6356556679,663108.6356556679,175.3532331637217,0.7040695939820947,0.0899250454062766,0.0800371127980879,5.6614214011449295,814.3148258847237,0.1111719478741608,0.0239395868491807,0.0195143057944333,Econ/Fin,1 -electricity/15T/short,Samay,124732.09064344651,124732.09064344651,47.74913488999746,0.9236110401104844,0.1537804371744333,0.1572336302668523,10.131180750283567,353.1743063183484,0.6961761763104043,0.0941229572909085,0.0785471758892922,Energy,1 -electricity/H/short,Samay,1581776.6862988477,1581776.6862988477,165.41931279772058,0.9009343379194178,0.1675378234012537,0.1221296217329731,10.03965826320032,1257.6870382964307,0.593718144151903,0.0780897348947613,0.0639810572983682,Energy,1 -electricity/D/short,Samay,1290386745.5346234,1290386745.5346234,4154.931503474101,1.3643983541162252,0.3402098714480294,0.0919545744238064,10.915372285553625,35921.95353171405,0.5903170216388917,0.0682793264035326,0.0536763733848107,Energy,1 -electricity/W/short,Samay,38535924425.94016,38535924425.94016,27009.694755175497,1.432356251682741,0.1603243152912733,0.0940178544449888,11.10043924582504,196305.6912724136,0.4479048063933098,0.0616272102028421,0.0479834378788082,Energy,1 -solar/10T/short,Samay,20.48523816518074,20.48523816518074,1.9412214836031896,0.8527204012009811,5.176681838533594,1.5062764897913676,3.7198013243883623,4.526062103548816,1.3166309757802663,0.5647011193589375,0.4303541085281823,Energy,1 -solar/H/short,Samay,678.9231726245581,678.9231726245581,11.308572450086368,0.8722136313120288,4.112616260079241,1.4060983288488862,5.239667658818893,26.05615421785337,0.9604385104363032,0.4168377416065493,0.3250354257450761,Energy,1 -solar/D/short,Samay,129209.41062751536,129209.41062751536,248.8069609042716,0.9701223262280764,1.0934284723112282,0.421123854551895,5.406705728810335,359.4571054069113,0.5192930846500652,0.3594413137671229,0.2752552884419362,Energy,1 -solar/W/short,Samay,2356831.538104413,2356831.538104413,1231.9647878449548,1.351401126556834,0.28366033049878,0.2366267895592953,6.16088716212218,1535.1975567022027,0.3133722031560806,0.2514748138389432,0.1829210550537137,Energy,1 -hospital/M/short,Samay,2801.318707818038,2801.318707818038,17.72830233937365,0.7517305763387505,0.1903020964628169,0.1713378787593286,4.968950466994184,52.92748537213947,0.1921390624129258,0.0643578542549307,0.0502581073664659,Healthcare,1 -covid_deaths/D/short,Samay,542543.3357689827,542543.3357689827,117.47276919390424,34.68868095799192,0.0897092245990628,0.3665486760281471,592.1431757847579,736.5754107822108,0.2771400828949842,0.0441997010974456,0.0360043118384795,Healthcare,1 -us_births/D/short,Samay,127621.3528085578,127621.3528085578,217.91029770665764,0.3207889943010152,0.0209475429946552,0.0207854585255463,2.763812657012963,357.2413089335524,0.0334884738665014,0.0204273221699238,0.0164228636248603,Healthcare,1 -us_births/M/short,Samay,36293387.31079231,36293387.31079231,4919.239523682411,0.5551576368957291,0.0151606231099853,0.0153177199795044,4.00679526314659,6024.399331949394,0.018711743251199,0.0152791244216062,0.0117302841698487,Healthcare,1 -us_births/W/short,Samay,1940109.9400202872,1940109.9400202872,1074.8143112549255,0.9772119752576774,0.0145460180343081,0.0145710291389119,5.786003523449176,1392.8782933265516,0.0189081862540268,0.0145904988849856,0.0113293849311792,Healthcare,1 -saugeen/D/short,Samay,1254.387167565359,1254.387167565359,13.17592238413568,2.927543996187861,0.2776930602592728,0.3268717563543886,30.4085070391111,35.41732863395204,1.1471757131783318,0.4267712653888142,0.3487704338295977,Nature,1 -saugeen/M/short,Samay,376.4748290343606,376.4748290343606,12.112350648749558,0.7097505379597879,0.351237134520372,0.3519545157221101,4.25686485391205,19.402959285489437,0.5825350460830707,0.3636491031870788,0.2863103604660737,Nature,1 -saugeen/W/short,Samay,979.3745948727104,979.3745948727104,15.176717707883338,1.2312135761294052,0.3992010776142541,0.4029233352276058,9.496580551449572,31.294961173848904,0.9446644255854778,0.4581218438376255,0.3629328721285453,Nature,1 -temperature_rain/D/short,Samay,182.71624070599995,182.71624070599995,5.733543299927763,1.3336723305911375,20.409838753112368,1.5107169576508357,18.906807226996374,13.517257144332202,1.5913385444051795,0.674989633752497,0.5458441714941981,Nature,1 -kdd_cup_2018/H/short,Samay,4242.784476110059,4242.784476110059,22.039384602754463,0.927054058152192,0.8622704218085089,0.4877278076341472,7.474845842173669,65.13666000118565,1.3634282618118376,0.4613242349206902,0.3696423112224992,Nature,1 -kdd_cup_2018/D/short,Samay,2848.8720204667684,2848.8720204667684,20.92700347757109,1.185581292668937,0.5267365092255006,0.4595602589106712,9.274159289235172,53.374825718373714,1.1956527900209923,0.4687871062429253,0.3732955003904065,Nature,1 -car_parts/M/short,Samay,1.3384211992868926,1.3384211992868926,0.4599854585923099,0.8361147533214801,0.8000438136776126,1.8805058984317289,13.739495391289154,1.1569015512509664,2.7741312444300807,1.1029979441937932,0.9443808343305322,Sales,1 -restaurant/D/short,Samay,142.04641498810204,142.04641498810204,7.182992401738404,0.688724378241453,0.6819697535730777,0.3951660580588244,4.689146011776683,11.918322658331668,0.5485088232125664,0.3305779531541775,0.2588785589781934,Sales,1 -hierarchical_sales/D/short,Samay,28.571552586447154,28.571552586447154,2.282561673551283,0.7382821566643768,0.6154139825966685,1.0523597959800297,7.101239628005842,5.345236438778659,1.639859344494613,0.7002646435998622,0.5670185740295962,Sales,1 -hierarchical_sales/W/short,Samay,463.4281414303178,463.4281414303178,8.903475643207349,0.7170959405621643,0.5273862507862404,0.4559179202019592,6.681204722710766,21.52738120232737,0.988332600824202,0.4087629217935116,0.3436421992655509,Sales,1 -loop_seattle/5T/short,Samay,38.55813319252253,38.55813319252253,3.3885630709928374,0.5378221149403747,0.0931979578899835,0.0720455744464023,4.066120330760099,6.209519562133815,0.1064795502612416,0.0581063749330038,0.0460251500885219,Transport,1 -loop_seattle/H/short,Samay,55.81825259720216,55.81825259720216,3.975140063364976,0.808238127676004,0.110387514109605,0.0914557373875245,7.089053846955458,7.471161395472738,0.1322471562109404,0.0703640225519357,0.0570087749140559,Transport,1 -loop_seattle/D/short,Samay,17.802813659299687,17.802813659299687,2.871785610037512,0.8681149143269017,0.0535884265074303,0.0529871357001426,6.8577148671525645,4.219338059376102,0.0754052120439682,0.0513226482027172,0.041552691698511,Transport,1 -sz_taxi/15T/short,Samay,16.648693018679445,16.648693018679445,2.721941928001121,0.5427728594082852,1098291225732.1908,0.3915186551325712,3.853526334751557,4.080280997514686,0.3815237345409672,0.2545132186256715,0.2000825753096237,Transport,1 -sz_taxi/H/short,Samay,7.153471543198912,7.153471543198912,1.8425849098617983,0.5610129588199412,1.1790575947330804,0.295030945706845,3.9040593149792095,2.6745974544216766,0.249155257520135,0.1716481547383359,0.1351278028546425,Transport,1 -m_dense/H/short,Samay,44498.98357786117,44498.98357786117,86.96075528364749,0.7766761881736786,0.295119833567124,0.2058282322236983,8.572013529508698,210.94782193201516,0.3741688712793366,0.1542467106418734,0.1275370931795377,Transport,1 -m_dense/D/short,Samay,7778.25060765272,7778.25060765272,42.99086642699632,0.6453971032499232,0.1181132031771508,0.0934388367991274,6.673478417702638,88.19439102149705,0.1526775365175504,0.0744235489672917,0.0610795283329058,Transport,1 -ett1/15T/short,Samay,4.617635439225449,4.617635439225449,1.0492422391707004,0.6655749117590196,0.4783169978084347,0.2329960047023165,4.503513005999038,2.1488684090063424,0.4062255762863912,0.1983504580758548,0.1542293727081914,Energy,7 -ett1/H/short,Samay,96.65407976682026,96.65407976682026,4.846328431391759,0.8011600086460764,0.4674644980439704,0.2571674083577633,5.483578162004828,9.831280677857809,0.4588660906707909,0.2261979750438782,0.1743434993666022,Energy,7 -ett1/D/short,Samay,40205.55812657437,40205.55812657437,135.47567493352528,1.6655863146445755,1.4091291141679874,0.4745552248409031,10.942893222324535,200.5132367864385,0.5266546675115957,0.3558313539869522,0.2725896391145538,Energy,7 -ett1/W/short,Samay,1103997.7462532148,1103997.7462532148,850.1096364352487,1.531306496972603,0.6616878007018754,0.5479124520481282,7.980507779776629,1050.712970441126,0.4181037690919071,0.3382789145409565,0.2503168577017842,Energy,7 -ett2/15T/short,Samay,7.338334596524514,7.338334596524514,1.68297016156363,0.7329797246606085,0.1057609675102194,0.1251658852119955,5.07077860970141,2.708936063572656,0.1286028318583387,0.0798965806615331,0.0631519605522973,Energy,7 -ett2/H/short,Samay,106.94046873398244,106.94046873398244,6.376223021428723,0.730319750099228,0.1176397258894232,0.1087843368735444,5.116376454851897,10.341202480078534,0.13115522084379,0.0808682491359932,0.0640519660201485,Energy,7 -ett2/D/short,Samay,103608.15062495152,103608.15062495152,191.7919909251705,1.2679436100685777,0.4325559846288829,0.1314846938029824,12.5067680937732,321.8821999193983,0.184221675589254,0.1097676166674683,0.0915267755372188,Energy,7 -ett2/W/short,Samay,2714423.582366889,2714423.582366889,1135.998742299009,0.7444675804984574,0.1151113142001125,0.1204247231805154,8.24967309095999,1647.550782940207,0.1382042521957397,0.0952928785567111,0.0804042518361372,Energy,7 -jena_weather/10T/short,Samay,771.8527805034779,771.8527805034779,5.352463624134633,0.283560623791611,0.3233294602887206,0.542206867797869,2.4155647767413297,27.78223857977391,0.1725625216543106,0.0332455074630332,0.0273280768883533,Nature,21 -jena_weather/H/short,Samay,1070.0898864852602,1070.0898864852602,8.262659174915733,0.5132576045593908,1.2826676037766755,0.6024770958431436,4.460045114827071,32.71222839375606,0.2005238541449212,0.050649568818621,0.0409357148597698,Nature,21 -jena_weather/D/short,Samay,371.8915792703849,371.8915792703849,9.417101371176738,0.9926143817505416,0.5930614046294015,0.4449270124341239,6.608734004491506,19.28449064067768,0.1161241526953458,0.0567063417929948,0.0446166269086009,Nature,21 -bitbrains_fast_storage/5T/short,Samay,1671099.2367567704,1671099.2367567704,154.64386869139634,0.656938041137114,1.8626124600197635,0.7314290297906211,12.865643873224144,1292.7100358381883,4.058789837761036,0.4855435057481507,0.379503567013003,Web/CloudOps,2 -bitbrains_fast_storage/H/short,Samay,2532877.4228740376,2532877.4228740376,288.1919389270312,0.9741020306043314,3.0577217673402357,0.577428049916003,21.396691989249398,1591.5016251559523,4.536296321477517,0.8214405890449902,0.6212449436939622,Web/CloudOps,2 -bitbrains_rnd/5T/short,Samay,1619617.605454138,1619617.605454138,118.16096654286709,1.6220266978780402,0.8711213973062959,0.6488222926423284,51.919131584416135,1272.6419785053995,5.207283595439851,0.4834805649131573,0.3902978000948883,Web/CloudOps,2 -bitbrains_rnd/H/short,Samay,1871226.212671417,1871226.212671417,152.28459850412406,5.827441802638028,1.486478396341202,0.5496139812268374,196.3866991987,1367.9277073995604,5.986721472426602,0.6664719713278003,0.6109961597190564,Web/CloudOps,2 -bizitobs_application/10S/short,Samay,301496.40477924206,301496.40477924206,301.7726631507327,1.0776098847235676,0.0147452760880211,0.0149237047071999,8.104625795649934,549.0868827237107,0.0211760863036585,0.0116381653979162,0.0091123804565192,Web/CloudOps,2 -bizitobs_service/10S/short,Samay,2080.61485713415,2080.61485713415,18.273402689187776,0.745475591605829,0.0556712420510271,0.0451606164205747,6.069098023179578,45.613757323138266,0.0337921306535733,0.0135375212961278,0.010457487065329,Web/CloudOps,2 -bizitobs_l2c/5T/short,Samay,17.726114750336706,17.726114750336706,2.512252064555331,0.2608243000673791,0.1477020960114234,0.2019330585772505,2.425666377542203,4.210239274713102,0.1445966584561286,0.0862809047257688,0.0689421552739553,Web/CloudOps,7 -bizitobs_l2c/H/short,Samay,50.84110648158205,50.84110648158205,4.3348575833674605,0.4309464278355327,0.3916003484877057,0.5963605389460007,2.6606805507883067,7.1302949785813245,0.3843411603465736,0.2336599255056459,0.1813924887731308,Web/CloudOps,7 -electricity/15T/medium,Samay,242724.41598436437,242724.41598436437,51.75031990794648,0.8155643784824596,0.1277580054276688,0.1299582800589168,6.919511366403489,492.6706973063898,0.848921130305646,0.0891710026801363,0.0717736999148718,Energy,1 -electricity/15T/long,Samay,312028.4345661408,312028.4345661408,57.05694247118867,0.8629403698089536,0.1324021086466365,0.1320949251304158,7.0686358908431215,558.5950541905476,0.8812521145492465,0.090014315066416,0.07202585658628,Energy,1 -electricity/H/medium,Samay,5518507.552688236,5518507.552688236,236.48919519284803,1.0523750538643428,0.2177197079802592,0.1281265850071292,8.80217802345847,2349.1503895426185,0.9169508357669856,0.0923095286479996,0.0736633658054768,Energy,1 -electricity/H/long,Samay,8570053.034365457,8570053.034365457,274.908202110904,1.1813708184531584,0.2900865310407699,0.1405655663357254,9.798014212975415,2927.4652917439444,1.154591576474511,0.1084237259297845,0.0862017244941693,Energy,1 -solar/10T/medium,Samay,20.280064176612505,20.280064176612505,1.835408389065509,0.8022498268575012,3.701286591829096,1.4458960486795889,4.140940340955147,4.503339225132002,0.9993088391751443,0.4072844027501866,0.305644721336971,Energy,1 -solar/10T/long,Samay,19.625406209486407,19.625406209486407,1.8462488182502836,0.8075674637014677,3.4990656163826657,1.4472694431393611,4.567795741845066,4.430057133885116,0.9574859210369302,0.3990371222719222,0.3025310328576442,Energy,1 -solar/H/medium,Samay,711.7691250982714,711.7691250982714,11.631769177744946,0.8877630807476572,3.7350756363567954,1.4082832013545736,4.832540595421729,26.67900157611359,0.9610399058660324,0.4190034744644982,0.3210003685119622,Energy,1 -solar/H/long,Samay,717.2776929529667,717.2776929529667,12.392479071916206,0.9429562336311031,4.252781822123085,1.412426426912635,4.467862280615921,26.78204049270643,0.928582152763952,0.4296698340746666,0.3215983303293471,Energy,1 -kdd_cup_2018/H/medium,Samay,5159.102497362385,5159.102497362385,25.322082336317084,1.0424006921905429,0.980812633780532,0.5565108480728174,9.203473831758911,71.82689257765774,1.5034669332669075,0.5300370391626409,0.4217154135041678,Nature,1 -kdd_cup_2018/H/long,Samay,4276.128540967201,4276.128540967201,24.847666030311554,1.041253092102275,0.9938741530586452,0.6265561569685083,7.83780705803773,65.39211375209706,1.534363198291351,0.5830266393418359,0.455110853538054,Nature,1 -loop_seattle/5T/medium,Samay,75.99349541787396,75.99349541787396,5.220173423008357,0.825651590194965,0.1700386969981668,0.1191239850133104,6.072267220412091,8.717424815728206,0.1551009024843787,0.0928776130736185,0.0734761590513988,Transport,1 -loop_seattle/5T/long,Samay,89.47903437173392,89.47903437173392,5.643748518630759,0.890119499545923,0.1807370718317134,0.1268946806800902,6.54976711818862,9.459335831427802,0.1672429937366579,0.0997826290315952,0.0789892448602225,Transport,1 -loop_seattle/H/medium,Samay,63.13053610207088,63.13053610207088,4.312356658773779,0.885137903674674,0.1154802689653155,0.0983901654588574,7.338634153623139,7.945472679587469,0.1404353243545932,0.0762204126210687,0.0615128016574829,Transport,1 -loop_seattle/H/long,Samay,57.48836224834892,57.48836224834892,4.094478337581876,0.856750811967711,0.123460307413264,0.0951820321541312,6.931753706955241,7.582108034600201,0.1355024241808857,0.0731738110254613,0.058758569772178,Transport,1 -sz_taxi/15T/medium,Samay,16.371997773374105,16.371997773374105,2.746410037190861,0.5354706790602541,9767299396210.418,0.4041480658764955,4.022414202925397,4.046232540694381,0.3765694756162614,0.2555992956733813,0.2024888818212803,Transport,1 -sz_taxi/15T/long,Samay,15.473236676056551,15.473236676056551,2.6785565727254124,0.5054051328792011,6972770314482.103,0.3969702493156323,4.19567820253263,3.933603522987104,0.3640873350920753,0.2479224250125716,0.1987950917692964,Transport,1 -m_dense/H/medium,Samay,35772.67363384761,35772.67363384761,78.27863427740432,0.6792082971851874,0.2739592987098184,0.1898365782169876,7.480905859183906,189.13665333257757,0.3290738819214339,0.1361949341880248,0.1129356576351953,Transport,1 -m_dense/H/long,Samay,34520.37936583727,34520.37936583727,78.73297882292394,0.6816469103226492,0.2791016192541885,0.1916749513762405,7.025613708908508,185.79660751972105,0.3216413933495371,0.1362984251931344,0.1127276335336491,Transport,1 -ett1/15T/medium,Samay,8.84183992308233,8.84183992308233,1.5507852845696608,0.9802147271022836,0.7509508851211425,0.3792230205844334,6.690905142601753,2.9735231499153207,0.5667564524229571,0.2955812085664968,0.2289892094377814,Energy,7 -ett1/15T/long,Samay,8.428724861390833,8.428724861390833,1.5414562084838848,0.9884787053475376,0.779238910137851,0.3825071641265952,7.127902193690479,2.903226629354111,0.553357866098824,0.2938030774404926,0.2293777847706224,Energy,7 -ett1/H/medium,Samay,145.3223827836894,145.3223827836894,6.84615114740939,1.271158978713131,3058205983025.619,0.4372157612186362,9.000975259356249,12.054973363043544,0.5774046478762207,0.3279144112168558,0.2564615875038838,Energy,7 -ett1/H/long,Samay,148.63899130726324,148.63899130726324,7.056263798375412,1.3633763189751118,3440840520352.156,0.4780592360383492,9.71356094001997,12.19175915556337,0.5838369404428497,0.3379091904978477,0.2651407303161724,Energy,7 -ett2/15T/medium,Samay,10.872887580989731,10.872887580989731,2.061635723325673,0.8529263815546998,0.1276945133541172,0.1549106539262036,6.389259084005328,3.297406189869506,0.1757247445962091,0.1098682995273208,0.0869053592864997,Energy,7 -ett2/15T/long,Samay,10.92074992393766,10.92074992393766,2.10358589571754,0.8746198944642622,0.1326103275770149,0.1599778514379432,7.046330059883586,3.304655795077251,0.1761110891804669,0.1121039001494023,0.0901168380555699,Energy,7 -ett2/H/medium,Samay,247.91287675795655,247.91287675795655,9.684680737265282,1.0287662833596944,0.1697484080647428,0.1664024834140067,7.433703426126807,15.745249339339043,0.2121773602385477,0.1305073009197782,0.1051295429671233,Energy,7 -ett2/H/long,Samay,233.4196369858939,233.4196369858939,9.399864697447429,1.0242180506017922,0.1750291581750207,0.1717490113645985,7.961804323746026,15.278077005496923,0.2092215057137872,0.1287239123613683,0.10452193366625,Energy,7 -jena_weather/10T/medium,Samay,1416.981383610722,1416.981383610722,9.455225556512994,0.5933178855888651,0.6840018156383681,0.6443894272346313,5.558385764237267,37.642813173442846,0.2310133009292006,0.058026557600143,0.0472854466800201,Nature,21 -jena_weather/10T/long,Samay,1372.4216944454504,1372.4216944454504,9.54195176024074,0.61376268673841,0.7097757585128349,0.6334715478903288,6.15857048953259,37.04621025753445,0.2266921129338911,0.0583888389933841,0.0479500295668371,Nature,21 -jena_weather/H/medium,Samay,1128.9691654022777,1128.9691654022777,9.661280347702313,0.7803319002710077,1.9712391538041567,0.6818652436340481,6.204549752998575,33.60013638963803,0.2056051936563626,0.0591190879055578,0.0475930460114697,Nature,21 -jena_weather/H/long,Samay,1169.0015079462416,1169.0015079462416,10.80242101180582,0.9425618498653976,2.0494149692371635,0.6515749492461353,7.12920186154339,34.1906640465821,0.2058656782748495,0.0650425427706251,0.0533290992759895,Nature,21 -bitbrains_fast_storage/5T/medium,Samay,2986718.6067451304,2986718.6067451304,292.540575610461,0.9650566072233672,4.661843863834121,0.805957157283106,22.099081032723667,1728.212546750292,5.250786458588755,0.8888189684142559,0.6463556716447275,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Samay,4066588.803205387,4066588.803205387,395.13651273913774,0.8797344577743398,5.729865734037965,0.8199530901748231,16.527791761376395,2016.5784892251,5.329214622248238,1.0442277812269842,0.7370077396582204,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Samay,2387367.216424705,2387367.216424705,174.59472712342816,4.378173538814573,1.5219176042844762,0.7361326874403736,161.3208765305932,1545.11074568288,6.501068161456204,0.7346089753317993,0.6703578403955308,Web/CloudOps,2 -bitbrains_rnd/5T/long,Samay,2652694.36501589,2652694.36501589,212.24268689709072,3.303269718340693,3.092644520864326,0.7164351186893089,120.52138125995548,1628.709416997363,6.239216926222792,0.8130536673611415,0.678537706849988,Web/CloudOps,2 -bizitobs_application/10S/medium,Samay,2811537.8553653643,2811537.8553653643,928.0878432453084,2.01561800029345,0.0284358224459332,0.0290747666126019,10.562206327359224,1676.7641024799416,0.0658415912927634,0.0364432780785101,0.028854031805611,Web/CloudOps,2 -bizitobs_application/10S/long,Samay,6356817.264642498,6356817.264642498,1525.0386471573634,3.2224657093011544,0.0467563330895607,0.0478595460172065,44.95230345388813,2521.272945288252,0.097235419683789,0.0588146448671747,0.0524764306459226,Web/CloudOps,2 -bizitobs_service/10S/medium,Samay,7405.5345421918255,7405.5345421918255,29.767332385447965,0.9567322623373726,0.064474414956262,0.0564896572077176,9.09589282951809,86.05541553087653,0.0647673336338478,0.0224035958237422,0.020673693040407,Web/CloudOps,2 -bizitobs_service/10S/long,Samay,130837.78767480358,130837.78767480358,76.61798244704465,1.3282142542461493,0.0803387698932944,0.0769145493546716,18.06056592698256,361.7150642077319,0.2679700833050314,0.0567610508121025,0.0520805772291165,Web/CloudOps,2 -bizitobs_l2c/5T/medium,Samay,123.47484424907466,123.47484424907466,7.614681611241665,0.7354054720992385,0.4946781528094024,0.7359956535300068,4.669203832419689,11.111923517063762,0.5846505849095727,0.4006442314938641,0.3020996691747571,Web/CloudOps,7 -bizitobs_l2c/5T/long,Samay,169.7492849163432,169.7492849163432,8.647765457128433,0.8375486229964803,0.6465360394658859,0.8235496604980413,5.274151164060892,13.028786778374386,0.7315468995344983,0.4855591019851789,0.3666668469769763,Web/CloudOps,7 -bizitobs_l2c/H/medium,Samay,71.46531443998963,71.46531443998963,4.914156808567125,0.4987315095061558,0.4867604858756992,0.7570853073544465,3.4793265410950784,8.453716013682364,0.5118853450857237,0.2975596589342985,0.2336920910672835,Web/CloudOps,7 -bizitobs_l2c/H/long,Samay,82.00141465631263,82.00141465631263,5.324816915612648,0.5707786329117613,0.6177549829404847,0.7734023994538566,4.700206562817383,9.055463249128264,0.5531338309331555,0.3252551855736402,0.2618950443672843,Web/CloudOps,7 \ No newline at end of file diff --git a/results/Samay/config.json b/results/Samay/config.json deleted file mode 100644 index e47cc2b12e17f17d7181a6d53debddbbc140e3bb..0000000000000000000000000000000000000000 --- a/results/Samay/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "model": "Samay", - "model_type": "agentic", - "model_dtype": "float32", - "org": "Kairosity", - "testdata_leakage": "No", - "replication_code_available": "No" -} \ No newline at end of file diff --git a/results/Synapse/all_results.csv b/results/Synapse/all_results.csv deleted file mode 100644 index 5e2f621448f2e0bd4be54a817fbd08d4a6c02f55..0000000000000000000000000000000000000000 --- a/results/Synapse/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -ett1/H/medium,Synapse,146.94783899266662,146.94783899266662,6.812165460290234,1.2520202351775702,3706092206008.374,0.4295186471891941,8.51083495152364,12.122204378439864,0.5806248541431446,0.3262865773666353,0.2538212572586146,Energy,7 -ett1/H/short,Synapse,102.23762957893258,102.23762957893258,4.89924163905195,0.8078586493757001,0.4836807399835478,0.2555377594077143,5.6700665288837335,10.111262511622009,0.4719339883056911,0.2286676509222781,0.1773400784346221,Energy,7 -ett1/H/long,Synapse,168.90610221307776,168.90610221307776,7.301347488134321,1.381653835345041,2873537270902.665,0.4703581107103424,8.936339938458019,12.996388044879152,0.6223688752469988,0.3496457175859827,0.2694626772379737,Energy,7 -ett2/15T/medium,Synapse,11.322445765179747,11.322445765179747,2.1163561504382,0.8767372941606136,0.132961101892253,0.1617390893867512,6.273410928922816,3.3648842127448826,0.1793207705793173,0.1127844501393037,0.0881407423454904,Energy,7 -ett2/15T/short,Synapse,7.580972410327561,7.580972410327561,1.7054272210391332,0.7425053643420214,0.1077816135557725,0.126755601162929,5.19530112899079,2.7533565715917656,0.1307116313980036,0.0809626971648336,0.0636393045062353,Energy,7 -ett2/15T/long,Synapse,11.995197532628923,11.995197532628923,2.194376328919091,0.9095490203128408,0.1384998491210363,0.1657568349873146,6.888294563981599,3.4634083693132296,0.1845713012245015,0.1169422866773154,0.0929551983787349,Energy,7 -car_parts/M/short,Synapse,1.3464592325830005,1.3464592325830005,0.4477781070115988,0.8227317658336394,0.8232880177923485,1.8921934778228056,15.442694559572052,1.160370299767708,2.782448947547479,1.0737259673387398,0.9214493614470212,Sales,1 -ett1/W/short,Synapse,990107.056342276,990107.056342276,797.488573022938,1.4482045554314027,0.609402791962008,0.5287779530156235,7.911713233618646,995.04123348848,0.3959506561994088,0.3173397374628678,0.2362862020408518,Energy,7 -m4_weekly/W/short,Synapse,227953.00465362697,227953.00465362697,245.7936042530203,1.908159359946333,0.0597115686800384,0.0597871971787989,14.332530220876109,477.44424245520753,0.0869829418600125,0.0447797855480667,0.0358958452396063,Econ/Fin,1 -bitbrains_rnd/5T/medium,Synapse,2257668.000651224,2257668.000651224,141.90787365561948,4.371371748200497,0.2958777069899697,0.6500461968972642,161.02741094045035,1502.5538262076418,6.322009065249867,0.5970786856571357,0.569362580595316,Web/CloudOps,2 -bitbrains_rnd/5T/short,Synapse,1654426.9278309008,1654426.9278309008,119.5441676486456,1.6456569748656964,1.009674219078477,0.5946036995410038,53.278841370448674,1286.2452829188144,5.262944358395888,0.4891402215896661,0.4037378643040601,Web/CloudOps,2 -bitbrains_rnd/5T/long,Synapse,2262192.590860991,2262192.590860991,158.47370054857174,3.30490457199308,0.9883426459014656,0.6438223533222612,116.28104033420357,1504.0587059224088,5.761710293864808,0.6070770696397663,0.5730204701512923,Web/CloudOps,2 -hospital/M/short,Synapse,2412.513498068645,2412.513498068645,16.837225410080656,0.7396826011034865,0.1876236248569014,0.1693925218559581,4.965526484565613,49.11734416749999,0.1783073650713089,0.0611230380809081,0.0486025714333033,Healthcare,1 -us_births/W/short,Synapse,1639888.7317731977,1639888.7317731977,1015.4762124599192,0.9230990334870104,0.0137477701666977,0.0137787251173128,6.483098156305313,1280.581403805786,0.0173837669899894,0.0137849900122066,0.0107312364793986,Healthcare,1 -bizitobs_l2c/5T/medium,Synapse,80.09261905249899,80.09261905249899,5.551777773973511,0.5523995155251742,0.3612898610537071,0.6554230745588171,6.28211496066632,8.949447974735593,0.4708725708030745,0.2921051533388585,0.2388022969428518,Web/CloudOps,7 -bizitobs_l2c/5T/short,Synapse,19.70327910016791,19.70327910016791,2.5729161809340657,0.2644536996257037,0.1449592054132906,0.1954602621238108,2.638085056390621,4.438837584342089,0.1524476497049153,0.0883643560320265,0.0704493936664133,Web/CloudOps,7 -bizitobs_l2c/5T/long,Synapse,95.475542381737,95.475542381737,6.429354015788381,0.6414238321334064,0.5182730165308294,0.7936130782080516,6.142408213491541,9.771158702105756,0.5486359532147373,0.3609986160849975,0.2962100542435223,Web/CloudOps,7 -jena_weather/D/short,Synapse,370.6618087868761,370.6618087868761,9.414158625464053,1.0089797978208577,0.5668208276245095,0.4416897165009706,6.522194114578399,19.25257927621325,0.1159319941245632,0.0566886216541096,0.0446547732622854,Nature,21 -hierarchical_sales/D/short,Synapse,29.014296120389357,29.014296120389357,2.283002907306539,0.7377682104892537,0.604404845577547,1.0518901084250387,7.244624206436351,5.38649200504274,1.6525160953465976,0.7004000092296057,0.5673425877971991,Sales,1 -saugeen/D/short,Synapse,1168.9911856147469,1168.9911856147469,12.671955529131528,2.815609085509478,0.2739006878602503,0.3171334954363322,28.32539814037392,34.190513093762526,1.1074388655238756,0.4104476588773562,0.3387309062806614,Nature,1 -solar/H/medium,Synapse,752.7114494259124,752.7114494259124,10.758042884631145,0.8231819667549871,4.341390633903047,1.3625981468139656,6.950302529751128,27.435587280499615,0.9882938888176932,0.3875298140993908,0.3200316996228562,Energy,1 -solar/H/short,Synapse,733.6985812109981,733.6985812109981,10.846179977776778,0.8369671029695357,4.406789577884148,1.3659838847697938,6.527985172498565,27.086871011820435,0.9984311145368244,0.3997938174163019,0.3249162588013766,Energy,1 -solar/H/long,Synapse,705.0765018267107,705.0765018267107,11.312951613184756,0.8611303967590513,4.863522780887982,1.3648148863288976,5.521483810770984,26.553276668364504,0.9206504940637948,0.3922406496975598,0.3109348211916878,Energy,1 -m_dense/D/short,Synapse,8262.044407141879,8262.044407141879,43.28525385410651,0.6499379365928549,0.1204000202204381,0.0939547717037757,7.654769334833467,90.89578872061058,0.1573540555237795,0.0749331771492241,0.0627383901090543,Transport,1 -bizitobs_service/10S/medium,Synapse,61157.67921730199,61157.67921730199,50.68659711218989,1.0906921614162928,0.079108787118012,0.0700027636374499,9.1519932559558,247.3007869322336,0.1861243999153801,0.038147927421857,0.0313431974445886,Web/CloudOps,2 -bizitobs_service/10S/short,Synapse,3596.461740794029,3596.461740794029,20.18299734036958,0.7745537471512611,0.0590785564187312,0.0463923737913154,5.601673011405553,59.970507258101705,0.044428068985729,0.0149522096639525,0.0117533548569934,Web/CloudOps,2 -bizitobs_service/10S/long,Synapse,128319.18145286982,128319.18145286982,75.05730455760909,1.2999571095698363,0.0863187081792614,0.0794245038713278,19.06389106333583,358.216668306864,0.2653783597807627,0.0556048507379906,0.0518816336203644,Web/CloudOps,2 -m4_yearly/A/short,Synapse,3315479.964806075,3315479.964806075,842.3316185292182,3.125027075275981,0.1580982484742505,0.1401107196386843,29.014482624916266,1820.845947576586,0.2919810879407846,0.1350717795277617,0.1088648365883799,Econ/Fin,1 -electricity/15T/medium,Synapse,257151.14898771344,257151.14898771344,53.43909903452861,0.8198643175185893,0.1251875509251994,0.1223619126506959,7.213582586386679,507.1007286404876,0.8737856196208658,0.0920809488620763,0.0740026993143613,Energy,1 -electricity/15T/short,Synapse,133146.52027772734,133146.52027772734,51.53456798365712,0.9878637710823396,0.1644986972910436,0.1584469200463396,10.07648649537564,364.89247769408365,0.7192749990119791,0.1015847916892498,0.0839473763571108,Energy,1 -electricity/15T/long,Synapse,328360.07404889254,328360.07404889254,58.75493365428791,0.8703973097099399,0.1314473322021058,0.1251520148534265,7.298508939929695,573.0271145843733,0.9040205002840872,0.0926931085186665,0.0739257626257306,Energy,1 -kdd_cup_2018/H/medium,Synapse,5073.431211377557,5073.431211377557,25.114526163936592,1.0404898556759636,0.9906961340013491,0.557447957710553,8.945774861114296,71.22802265525526,1.4909315141332262,0.5256925171370577,0.4188053019267645,Nature,1 -kdd_cup_2018/H/short,Synapse,4262.630761920218,4262.630761920218,22.089677260855485,0.9289124710710708,0.8483788707124738,0.4904689836895579,7.542432932622407,65.28882570486482,1.366613369497348,0.4623769526661479,0.3710148668516113,Nature,1 -kdd_cup_2018/H/long,Synapse,4110.639748448135,4110.639748448135,24.01573632483864,1.015996323019437,1.0112369022303025,0.6068232760125196,7.906184102583698,64.11427102017252,1.5043798771312542,0.5635062192348722,0.4463653277836563,Nature,1 -ett2/H/medium,Synapse,230.8713104875281,230.8713104875281,9.499860041496866,1.020314746171514,0.1717335457454356,0.1663774926895727,6.969969618349351,15.1944499896353,0.2047549847954824,0.1280167231905569,0.1014165651372813,Energy,7 -ett2/H/short,Synapse,104.37565115208616,104.37565115208616,6.283492998273024,0.7220791245233226,0.1177810640573338,0.1078438469891324,4.997455192574861,10.216440238756656,0.1295728884849507,0.0796921744300521,0.0629563820182046,Energy,7 -ett2/H/long,Synapse,222.4462084014408,222.4462084014408,9.168925572733787,1.0142504289715344,0.178592235823418,0.1715216637096052,7.497373471157779,14.914630682703503,0.2042443880520802,0.1255613787922924,0.102282169215304,Energy,7 -m4_daily/D/short,Synapse,354382.6304079325,354382.6304079325,169.16872181651064,3.1640393898299277,0.0377642892016585,0.0294081726014986,24.95019064771856,595.3004538952852,0.0919574687254221,0.0261318924653429,0.0207084043510823,Econ/Fin,1 -us_births/M/short,Synapse,52512880.96291367,52512880.96291367,6027.396184455261,0.6810469795824474,0.0185156925138092,0.0187573585030276,3.8754974708715024,7246.5771894677055,0.0225078193439475,0.01872105145465,0.0139450172563461,Healthcare,1 -us_births/D/short,Synapse,173152.55419109843,173152.55419109843,247.5000894770577,0.3643589008871609,0.0238560705280271,0.023434893750666,3.798166661205532,416.1160345277486,0.0390075016501148,0.0232011250410877,0.0190869522209751,Healthcare,1 -electricity/H/medium,Synapse,5732405.686947512,5732405.686947512,236.83959484288192,1.0508267607828758,0.2166060097950284,0.1219270797984435,9.093218072390965,2394.244283056245,0.9345524442583422,0.0924462987440163,0.0742486021081844,Energy,1 -electricity/H/short,Synapse,1505989.6831213266,1505989.6831213266,163.71440130324052,0.8911392230448414,0.1780196219441764,0.1149852230568949,10.105657730092489,1227.1877130746243,0.5793202388060784,0.0772848888955752,0.0632452038016243,Energy,1 -electricity/H/long,Synapse,8929666.45333153,8929666.45333153,274.6828691836707,1.188124215965291,0.2947410231821484,0.1349969522921131,10.17743375404128,2988.254750407256,1.178566789713393,0.1083348423620652,0.0873607253934506,Energy,1 -ett2/W/short,Synapse,3279576.55735223,3279576.55735223,1258.314460212263,0.8514538935432421,0.1320826833864451,0.1424574151281107,8.331669273886538,1810.9601203097293,0.1519117903832141,0.1055532920753879,0.0876552923341272,Energy,7 -solar/W/short,Synapse,1366653.9855074722,1366653.9855074722,951.081748425174,1.0406283460235448,0.2141624711483608,0.1881918534816372,6.836096291945386,1169.0397707124732,0.2386302446391337,0.1941395630708056,0.1413689343806533,Energy,1 -covid_deaths/D/short,Synapse,406213.67051453085,406213.67051453085,106.02786239694495,34.76822942621669,0.0890346953153257,0.1292819723847498,667.3733837191428,637.3489393687973,0.2398056401341448,0.039893499217768,0.0322807399352041,Healthcare,1 -hierarchical_sales/W/short,Synapse,464.97197564947663,464.97197564947663,8.857559102426299,0.7153647093904841,0.5210280538187463,0.4534882498940069,6.61238502851093,21.5632088439888,0.9899774653774812,0.4066548709468029,0.3415550802474633,Sales,1 -kdd_cup_2018/D/short,Synapse,2894.194524450775,2894.194524450775,21.124843578818577,1.1949101118806624,0.5187398191496736,0.4646995705426894,9.155668231112465,53.79771858035223,1.2051259832236578,0.4732189126262951,0.374212379990483,Nature,1 -electricity/W/short,Synapse,52003441959.636,52003441959.636,30117.597252898933,1.4866042132857489,0.237392731417054,0.0962835495602032,11.602888420777548,228042.6318906971,0.5203180316595193,0.068718418091463,0.0530581119990483,Energy,1 -bizitobs_application/10S/medium,Synapse,4128064.1094782,4128064.1094782,1114.1234589312237,2.402684326071172,0.0346363716260885,0.0354001056528419,30.21734840325128,2031.7637927372853,0.0797813843026518,0.0437483491709634,0.0385169302647742,Web/CloudOps,2 -bizitobs_application/10S/short,Synapse,412997.39431499,412997.39431499,345.70164093109656,1.1289557001904804,0.0158220565857121,0.0159746206226616,8.46140609911413,642.6487332244498,0.0247843929000676,0.0133323304817625,0.0108397328010548,Web/CloudOps,2 -bizitobs_application/10S/long,Synapse,6348768.4045307785,6348768.4045307785,1521.784607133485,3.28225897984727,0.0477706295709492,0.0488086658330317,59.51608750538327,2519.676249943785,0.0971738415265266,0.0586891495502234,0.0540797200910186,Web/CloudOps,2 -m4_quarterly/Q/short,Synapse,1704216.4412085095,1704216.4412085095,541.3088207979406,1.1344178775177756,0.1095378682242531,0.0981620663666799,8.639374569077994,1305.4564110718172,0.2185104148730246,0.0906055644629858,0.0722425192203886,Econ/Fin,1 -m4_hourly/H/short,Synapse,550902.0901453486,550902.0901453486,169.54285748284195,0.6761003933068471,0.0862793371402033,0.0790502901426883,5.513544829163199,742.2277885833624,0.101330476125734,0.0231463423179611,0.0195314602798149,Econ/Fin,1 -bizitobs_l2c/H/medium,Synapse,70.43793434980273,70.43793434980273,4.787732579546283,0.4791288487875105,0.4850201019597101,0.757807664991636,3.910773349886712,8.3927310423844,0.5081926124427799,0.2899044798397099,0.2336389808309401,Web/CloudOps,7 -bizitobs_l2c/H/short,Synapse,58.98437467945725,58.98437467945725,4.566292845750443,0.4532059615119996,0.4004464843323466,0.5976727478706291,2.840177075816273,7.680128558784498,0.4139785984676602,0.2461348788640394,0.191615783910041,Web/CloudOps,7 -bizitobs_l2c/H/long,Synapse,94.11873929016753,94.11873929016753,5.546876742722112,0.5892841360408446,0.661434381125864,0.7817607540207485,5.446899286956702,9.701481293604989,0.5925944776126896,0.338819240717615,0.2806884737460488,Web/CloudOps,7 -restaurant/D/short,Synapse,142.28578260876435,142.28578260876435,7.1809971587463775,0.6879314750289621,0.6784336164760226,0.3953640268783044,4.640575575148456,11.928360432547482,0.5489707848392674,0.3304861274487529,0.258505632945248,Sales,1 -sz_taxi/15T/medium,Synapse,16.429632383109688,16.429632383109688,2.7571556599345888,0.5376984534533001,10251564322715.84,0.4050376108094555,3.8446236271787546,4.053348292845026,0.3772317151264809,0.2565993552302848,0.2025289108635015,Transport,1 -sz_taxi/15T/short,Synapse,16.623572000455418,16.623572000455418,2.7218355856046643,0.5429454585826672,1069151538645.5548,0.3919598365694669,3.862407484931298,4.077201491275041,0.3812357769027509,0.2545032680626843,0.1999092160154406,Transport,1 -sz_taxi/15T/long,Synapse,15.640912782662342,15.640912782662342,2.700579366740374,0.5097701125222233,7044606482071.678,0.3990091661293664,3.6805718221117663,3.954859388481762,0.3660547401388221,0.2499608155970147,0.1977300197065558,Transport,1 -loop_seattle/5T/medium,Synapse,73.40544759435019,73.40544759435019,4.994110712180381,0.7950069677115564,0.1751483929738529,0.1145499917032593,5.818477893690894,8.567697916847337,0.1524369471297423,0.0888554893019444,0.070023114995968,Transport,1 -loop_seattle/5T/short,Synapse,37.30412537653298,37.30412537653298,3.344604292908245,0.53037125512481,0.0917061685818246,0.0710274360512014,4.002077667981049,6.1077103219236735,0.1047337509239739,0.0573525813258246,0.0452808286074293,Transport,1 -loop_seattle/5T/long,Synapse,86.24588183813162,86.24588183813162,5.404616616109952,0.856915549602864,0.1863705621083247,0.1218926334236065,6.2873314337051225,9.286866093474783,0.1641936918023557,0.0955547270783804,0.0755394566911762,Transport,1 -m4_monthly/M/short,Synapse,1810053.294227752,1810053.294227752,546.833279510535,0.9055893443901412,0.1572518680126872,0.1291039015166657,7.384518303910223,1345.3822112053333,0.2796166111091683,0.1136507285327203,0.0904017878907207,Econ/Fin,1 -temperature_rain/D/short,Synapse,185.8350878324541,185.8350878324541,5.824136279142628,1.3456407126378602,20.53097061631961,1.5101373129655495,20.171157900163408,13.63213438286368,1.6048626213296848,0.6856548177573539,0.5519994332832616,Nature,1 -electricity/D/short,Synapse,1428292788.9686604,1428292788.9686604,4262.271657029427,1.384841407356544,0.4134501314783911,0.0931872616481992,11.324116216473143,37792.76106569432,0.6210605744251769,0.0700432783693523,0.0549055693358672,Energy,1 -loop_seattle/H/medium,Synapse,64.21670498132806,64.21670498132806,4.385264661497428,0.9013433773364754,0.1147794979785593,0.0999422857958225,6.745637720315491,8.013532615602688,0.1416382759693417,0.0775090532625305,0.0618063590854122,Transport,1 -loop_seattle/H/short,Synapse,55.2341980659034,55.2341980659034,3.962806585104059,0.80563462505355,0.1087950649119793,0.0912135234818682,6.801692833112259,7.431971344529216,0.131553450469674,0.0701457063875975,0.0568416984316428,Transport,1 -loop_seattle/H/long,Synapse,57.57632657883847,57.57632657883847,4.134720832429267,0.8677600225586954,0.1222967955290665,0.0958862862913527,7.225835957045198,7.587906600561084,0.1356060497145731,0.0738929968796432,0.0593480880431031,Transport,1 -solar/10T/medium,Synapse,18.547857536814483,18.547857536814483,1.820801993801284,0.793256090250791,3.4951458345583584,1.416252148818831,5.032151458561582,4.3067223658850455,0.955678759789687,0.4040431779495504,0.3123540128853747,Energy,1 -solar/10T/short,Synapse,21.983464895982248,21.983464895982248,2.033531699894617,0.8924449634250201,5.067972035141637,1.5029474856716107,4.235106062410286,4.688652780488469,1.3639285860945345,0.5915541512815382,0.4422164606151873,Energy,1 -solar/10T/long,Synapse,19.33910462268322,19.33910462268322,1.8980100308443029,0.8270466806722322,3.229264771274896,1.4169051275616706,5.4379015759713925,4.397624884262324,0.950476168868698,0.4102244620834378,0.3174284895290182,Energy,1 -jena_weather/10T/medium,Synapse,1484.4180981127488,1484.4180981127488,9.753715081303463,0.6057369517402115,0.7046283030918842,0.6485726795223886,5.754323651037319,38.52814682946416,0.2364465784411346,0.059858382711006,0.0484903390122338,Nature,21 -jena_weather/10T/short,Synapse,788.1377583262587,788.1377583262587,5.478248191973733,0.2746318749104626,0.3130254507758209,0.5390755897922194,2.240588613376969,28.07379130659517,0.1743734294971399,0.0340267872778036,0.0273720439294918,Nature,21 -jena_weather/10T/long,Synapse,1415.2158444997017,1415.2158444997017,9.845282922273812,0.6513908336326328,0.7607495019405257,0.6453685028044016,6.121386780021486,37.61935465288715,0.2301992817669648,0.060244974386504,0.0487150286312064,Nature,21 -ett1/D/short,Synapse,44956.64487000874,44956.64487000874,141.27273539668013,1.679238404715881,1.3339957720974018,0.4729700562260733,10.464018692128548,212.0298207092784,0.5569033571939261,0.3710575255838885,0.2824028793236845,Energy,7 -bitbrains_fast_storage/H/short,Synapse,2576507.8225229643,2576507.8225229643,280.69334911564584,1.032876739715154,3.788816974835621,0.53211739023351,19.4178926162198,1605.1504049536804,4.575199942659383,0.800067202933442,0.583156367754806,Web/CloudOps,2 -sz_taxi/H/short,Synapse,7.133412769065301,7.133412769065301,1.8385662561861813,0.5603673530165401,1.16640567443035,0.2953611585879397,3.9345546400698543,2.67084495414191,0.2488056889628166,0.1712737923497913,0.1347913277445439,Transport,1 -bitbrains_rnd/H/short,Synapse,1910684.5004340415,1910684.5004340415,157.36717732585188,5.8224675269572925,1.598068783876308,0.52789189116111,203.5107402367307,1382.275117490741,6.049512482270871,0.6887157928883513,0.5880745007883234,Web/CloudOps,2 -jena_weather/H/medium,Synapse,1379.3681124051764,1379.3681124051764,10.403582706568782,0.8045796246679728,2.046999702863032,0.676479694133611,6.052725443148616,37.13984534708211,0.2272653005453004,0.0636613676890822,0.0513485836542469,Nature,21 -jena_weather/H/short,Synapse,1100.9354426460386,1100.9354426460386,8.182852107362145,0.5169484762506563,1.1390769591942052,0.5932482594370792,4.61552611540064,33.18034723516375,0.2033933925067541,0.0501603566322417,0.0408238981672407,Nature,21 -jena_weather/H/long,Synapse,1242.781955319888,1242.781955319888,11.2974455556546,1.000252513353406,1.98102717047092,0.652446244801361,7.567937043324608,35.25311270398527,0.2122627963065589,0.068023138974995,0.0550500046467677,Nature,21 -ett1/15T/medium,Synapse,9.373120500718164,9.373120500718164,1.5654177961955036,0.9863928747378472,0.7339281877638908,0.3829447700109294,7.022949457949263,3.0615552421470635,0.5835354562434096,0.2983701797501704,0.2320640101544294,Energy,7 -ett1/15T/short,Synapse,4.8318869752897085,4.8318869752897085,1.0666263271968577,0.6773525695659657,0.4874768160238661,0.2375770124749354,4.812097813729572,2.1981553574053194,0.4155428611107469,0.2016367743281862,0.1565547703839609,Energy,7 -ett1/15T/long,Synapse,8.810082696721697,8.810082696721697,1.5725842632317446,1.00716221096466,0.7880472747350004,0.3850280243156201,7.261755881008521,2.9681783465151983,0.5657377276102589,0.2997361154530694,0.23198564523351,Energy,7 -loop_seattle/D/short,Synapse,17.905539386700084,17.905539386700084,2.864706950291469,0.8668597823056257,0.0531577165788378,0.0529073611604941,7.258227125307386,4.231493753593415,0.075622452730073,0.0511961445647791,0.0414263895765929,Transport,1 -ett2/D/short,Synapse,104370.83973541432,104370.83973541432,195.05111505640727,1.2783255431109632,0.4261417455135873,0.1334460801986309,11.070363228798492,323.06476090006214,0.1848984864393069,0.1116328993968648,0.0914132873843379,Energy,7 -saugeen/W/short,Synapse,1018.0628460028788,1018.0628460028788,15.273826122692292,1.2391523587254116,0.3967476114685623,0.4041926501399649,10.709811885993972,31.907097110249293,0.9631422578322588,0.4610531420867403,0.3714148127444749,Nature,1 -solar/D/short,Synapse,135526.43957088602,135526.43957088602,252.56845680306196,0.9855218506528988,1.125512575537161,0.4235735976075132,5.326414455820764,368.1391578885436,0.5318356933409599,0.3648753941589133,0.2811440913709339,Energy,1 -m_dense/H/medium,Synapse,35443.918268838555,35443.918268838555,77.5733069035829,0.6732899546211619,0.2522531354736324,0.1858969170757366,7.295827481678306,188.26555252843931,0.3275582765741267,0.1349677536662211,0.1116340529796323,Transport,1 -m_dense/H/short,Synapse,44694.34331993142,44694.34331993142,87.1006878161677,0.7761561419483677,0.2848384605214743,0.2057436954456219,8.727122414833012,211.41036710608924,0.3749893111592742,0.1544949160856121,0.1287031037819354,Transport,1 -m_dense/H/long,Synapse,34529.99131339011,34529.99131339011,78.89504999314346,0.6878083215993914,0.2546061346853125,0.1883003671089787,7.34714703588209,185.822472573664,0.3216861695814625,0.1365789943472599,0.1130239210587437,Transport,1 -saugeen/M/short,Synapse,397.81982398265706,397.81982398265706,12.511034062211928,0.7330210933360634,0.3547603128162583,0.3592763197466192,4.54602108454984,19.94542112823535,0.5988213779725894,0.3756187752982575,0.2907421344741516,Nature,1 -bitbrains_fast_storage/5T/medium,Synapse,3175837.270290936,3175837.270290936,268.43408883142973,0.9715503951117148,3.1614179686106625,0.735463007137945,21.450480403197083,1782.08789634264,5.414475534788863,0.8155769475029686,0.6040038588958575,Web/CloudOps,2 -bitbrains_fast_storage/5T/short,Synapse,1598857.9145629297,1598857.9145629297,148.79235292090698,0.6706583537531366,1.7665442694710327,0.6422919029835097,13.454023761573085,1264.459534569189,3.9700902669134464,0.467171195259883,0.3638916331232154,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Synapse,5079313.224209629,5079313.224209629,383.4828909311811,0.8917345468769259,3.798354128246866,0.7485911553381718,16.198055522206186,2253.733175025302,5.955943910295123,1.0134307886372995,0.739296800111958,Web/CloudOps,2 \ No newline at end of file diff --git a/results/Synapse/config.json b/results/Synapse/config.json deleted file mode 100644 index f50fec76dbc2560116091817814e2781709e6ada..0000000000000000000000000000000000000000 --- a/results/Synapse/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "model": "Synapse", - "model_type": "agentic", - "model_dtype": "float32", - "model_link": "https://arxiv.org/abs/2511.05460", - "org": "Google Cloud AI Research", - "testdata_leakage": "No", - "replication_code_available": "No" -} diff --git a/results/TSOrchestra-test/all_results.csv b/results/TSOrchestra-test/all_results.csv deleted file mode 100644 index 4a633cc77cd1542c1358a86deffd7da2c536b6d2..0000000000000000000000000000000000000000 --- a/results/TSOrchestra-test/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/5T/long,"TSOrchestra-test",3935933.7645889753,3935933.7645889753,349.58813997497964,0.9269540787531104,4.862012611677347,0.8328672722129261,26.82744338636312,1983.9187898170064,5.24290504557977,0.9238570814275756,0.6493157422172859,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,"TSOrchestra-test",3217856.558815273,3217856.558815273,287.70779258514955,0.9987420682463012,4.609855566296933,0.8105082483269215,30.629692943250262,1793.8384985319256,5.450176678906839,0.8741357166604711,0.6105192205604982,Web/CloudOps,2 -bitbrains_fast_storage/5T/short,"TSOrchestra-test",1972437.9243349463,1972437.9243349463,162.07881755000224,0.7029727859199087,1.6414042448417012,0.7272514825965226,21.18755093466908,1404.435090822978,4.409578840431848,0.5088874017893563,0.3920441656326121,Web/CloudOps,2 -bitbrains_fast_storage/H/short,"TSOrchestra-test",2498448.981346684,2498448.981346684,266.6039956961493,0.9865470511084616,3.0480100896031463,0.5766203127614407,30.552464429757336,1580.6482788231808,4.505361084049783,0.7599079967087182,0.584607682123271,Web/CloudOps,2 -bitbrains_rnd/5T/long,"TSOrchestra-test",2741014.3092923723,2741014.3092923723,193.610297739826,3.383457250497965,2.009947758563582,0.7066837753441967,124.91210443346228,1655.6008907017333,6.342233802518832,0.7416774065160113,0.631509489967593,Web/CloudOps,2 -bitbrains_rnd/5T/medium,"TSOrchestra-test",2263536.404681218,2263536.404681218,146.41216215363497,4.452550887955358,0.3957768536657749,0.7199647306432045,168.41223096881703,1504.505368777798,6.330221020104104,0.6160306009520887,0.5808039507248792,Web/CloudOps,2 -bitbrains_rnd/5T/short,"TSOrchestra-test",1752930.746871569,1752930.746871569,123.50603177337376,1.6659307680385191,0.9296500554170668,0.6516288286382356,59.392445795324,1323.982910339695,5.417355918055654,0.5053510335502674,0.403352349057457,Web/CloudOps,2 -bitbrains_rnd/H/short,"TSOrchestra-test",1756632.355034649,1756632.355034649,144.89788264714036,5.809152255556768,1.1851856357234645,0.564185657269599,215.09030876541024,1325.3800794619815,5.800511745138296,0.634144033975212,0.5889087226784738,Web/CloudOps,2 -bizitobs_application/10S/long,"TSOrchestra-test",5884915.995134343,5884915.995134343,1465.0117940495852,3.213011035819848,0.0459765167601622,0.0469714424036286,100.99723605807336,2425.8845799283904,0.0935566717100521,0.0564996490769967,0.0503051296002101,Web/CloudOps,2 -bizitobs_application/10S/medium,"TSOrchestra-test",3828266.337022155,3828266.337022155,1100.421361694994,2.356715896820709,0.0340752889343855,0.0346987677846694,61.218211075929815,1956.5955987434284,0.0768297505577268,0.0432103081401784,0.0369991787862529,Web/CloudOps,2 -bizitobs_application/10S/short,"TSOrchestra-test",664049.1079234522,664049.1079234522,414.5743819470433,1.2483539298962758,0.0175773060136128,0.0178031365298497,25.050352158876624,814.8920836058307,0.0314271304479298,0.015988476810534,0.0127843258467027,Web/CloudOps,2 -bizitobs_l2c/5T/long,"TSOrchestra-test",91.82933679300142,91.82933679300142,6.427153720920439,0.6173795630481984,0.5829287999115433,0.8135781718082119,15.723615639386017,9.582762482343044,0.5380577871279156,0.3608750728176093,0.2847021405440567,Web/CloudOps,7 -bizitobs_l2c/5T/medium,"TSOrchestra-test",73.52162372929024,73.52162372929024,5.944841553673787,0.6077701713344603,0.4636258758036122,0.7222894780971079,15.196299658694844,8.574475128501467,0.4511434848766517,0.3127860884042988,0.2484161002608965,Web/CloudOps,7 -bizitobs_l2c/5T/short,"TSOrchestra-test",20.957101863161707,20.957101863161707,2.6911774727408004,0.2773151837196029,0.1543018104450295,0.2091604607124603,6.909908275871829,4.5778927317229385,0.1572233663187331,0.0924259274782537,0.0745043675525424,Web/CloudOps,7 -bizitobs_l2c/H/long,"TSOrchestra-test",87.50371503667041,87.50371503667041,5.623410359120878,0.5915964014150494,0.6853376151569022,0.8458569712774032,14.615826382270823,9.35434204188998,0.5713902102123579,0.3434941348968711,0.2717095131496422,Web/CloudOps,7 -bizitobs_l2c/H/medium,"TSOrchestra-test",74.57246236988911,74.57246236988911,4.87528932305638,0.4872094958753631,0.4700800864948974,0.7646663653826199,11.821017844833518,8.63553486298846,0.5228947525781809,0.2952061736503026,0.2297831314253154,Web/CloudOps,7 -bizitobs_l2c/H/short,"TSOrchestra-test",57.93384375939396,57.93384375939396,4.785254307164727,0.4750951303999911,0.4757265858016408,0.6327419175680208,11.137742878278091,7.611428496635435,0.4102754891739257,0.2579374624042636,0.1994469349308772,Web/CloudOps,7 -bizitobs_service/10S/long,"TSOrchestra-test",129609.8550861882,129609.8550861882,75.11599541260756,1.4999397997721398,0.0871011745056588,0.0824391880280784,34.53666891595601,360.0136873595061,0.2667096489442151,0.0556483307996729,0.0506102816981973,Web/CloudOps,2 -bizitobs_service/10S/medium,"TSOrchestra-test",56159.60582751026,56159.60582751026,52.13330916775173,1.252231409318365,0.0766055228893532,0.0697650514089244,28.608471818620583,236.98018024195665,0.1783568681140486,0.0392367570068012,0.0304204698870069,Web/CloudOps,2 -bizitobs_service/10S/short,"TSOrchestra-test",3881.729375382179,3881.729375382179,22.136281506066148,0.8410756046412172,0.0579897674974819,0.0479519212981181,18.69069223154232,62.30352618738509,0.0461564435409787,0.0163992648581233,0.0128854832452221,Web/CloudOps,2 -car_parts/M/short,"TSOrchestra-test",1.3571386982905318,1.3571386982905318,0.4520421170816914,0.8354162290912995,0.8491219679468779,1.9052332164806256,26.498828735096527,1.1649629600508902,2.7934616757894397,1.0839506260827942,0.933131133376858,Sales,1 -covid_deaths/D/short,"TSOrchestra-test",341661.47496713034,341661.47496713034,89.43748471422953,33.01452582259661,0.085330203802731,0.3608294291627087,1282.6248087411086,584.5181562339449,0.2199277905059623,0.0336512872899344,0.0272617478091108,Healthcare,1 -electricity/D/short,"TSOrchestra-test",1230488522.604079,1230488522.604079,4170.16461421997,1.3902266944274326,0.3984162729579725,0.094695451636671,36.629052415856606,35078.319837245326,0.5764532917223313,0.0685296539299661,0.0538406171882295,Energy,1 -electricity/H/long,"TSOrchestra-test",10542849.241315251,10542849.241315251,294.90034990515613,1.2219609718219435,0.2966046256564434,0.145322628525944,26.33709604853039,3246.9753989390265,1.2806060975366071,0.1163086071972157,0.0925612193602453,Energy,1 -electricity/H/medium,"TSOrchestra-test",6717206.2563067535,6717206.2563067535,249.51903304495917,1.0794100690495547,0.2262837409275522,0.1308458976630657,23.33750877033781,2591.7573683326827,1.011648286996088,0.097395499068357,0.0778650105653855,Energy,1 -electricity/W/short,"TSOrchestra-test",31080592419.92485,31080592419.92485,25637.68543401974,1.4219617623728595,0.1193387766666435,0.0927880248780248,27.60636874978625,176296.8871532474,0.4022513183203626,0.0584967376965239,0.0452820611324487,Energy,1 -ett1/15T/long,"TSOrchestra-test",8.243749552140653,8.243749552140653,1.5706371320387684,1.019936228932009,0.7666978238724542,0.393974889065195,23.282035069472816,2.871193053791516,0.5472522349236316,0.2993649903224766,0.2315341704375025,Energy,7 -ett1/15T/medium,"TSOrchestra-test",9.62130002943135,9.62130002943135,1.616982551856934,1.0275573699156375,0.7271256870647294,0.3961183550865695,24.6452636555272,3.101822049929904,0.5912103496203284,0.3081984731635112,0.2398625076575753,Energy,7 -ett1/15T/short,"TSOrchestra-test",4.658571776411579,4.658571776411579,1.093816154159389,0.692003401866187,0.4649557597359685,0.2415243677992025,17.45309586298432,2.158372483240921,0.4080222419253055,0.2067767834049118,0.1588046263888257,Energy,7 -ett1/D/short,"TSOrchestra-test",40315.522051434666,40315.522051434666,136.2583918139499,1.6806150286411232,1.376117619059447,0.4720882395027222,45.98375255620722,200.78725569974472,0.5273743872766946,0.3578871858363717,0.2725179724620256,Energy,7 -ett1/H/long,"TSOrchestra-test",136.38238429532774,136.38238429532774,6.854423516370071,1.3136035358866145,3912244942427.572,0.4639939552636891,27.01066694148744,11.678286873310132,0.5592478649494957,0.3282434965483124,0.256173450764951,Energy,7 -ett1/H/medium,"TSOrchestra-test",132.05527359751693,132.05527359751693,6.589209046620069,1.2299896795059144,4206597318493.5903,0.433320842160278,28.42749728740139,11.491530515885032,0.5504170711338863,0.3156074936681489,0.2473610485932525,Energy,7 -ett1/H/short,"TSOrchestra-test",100.08987683942247,100.08987683942247,4.946661781605223,0.8186681618457503,0.4834980991410276,0.2601233488004902,20.23439453202309,10.004492832693844,0.466950610577265,0.2308809429790778,0.1774015397583596,Energy,7 -ett1/W/short,"TSOrchestra-test",989510.3656643648,989510.3656643648,812.4721059322254,1.494997089594253,0.7054403469054227,0.5293702695860415,29.63079542303231,994.741356164689,0.3958313278548756,0.3233020428306749,0.2467594360180067,Energy,7 -ett2/15T/long,"TSOrchestra-test",11.38472768326637,11.38472768326637,2.1665369423910272,0.9005037546896144,0.1375508085188752,0.1654055922686201,22.30923616420953,3.374126210334517,0.1798132933600978,0.1154586753762907,0.0916396359038175,Energy,7 -ett2/15T/medium,"TSOrchestra-test",11.680949215525512,11.680949215525512,2.145229392605787,0.889812784512305,0.1365361800032471,0.1640630924885128,22.39977677062312,3.4177403668982103,0.1821375707107257,0.1143231574787731,0.0895711169001382,Energy,7 -ett2/15T/short,"TSOrchestra-test",7.853729322785832,7.853729322785832,1.7319636586573233,0.7427096967836446,0.1062751939885833,0.12598484365775,18.404858474329288,2.8024505923897807,0.1330422992150986,0.0822224762607753,0.0649307567329398,Energy,7 -ett2/D/short,"TSOrchestra-test",106039.05185816904,106039.05185816904,193.00609061549403,1.2668341065190438,0.4344814991951588,0.1311453496619999,32.81435040036002,325.636379813695,0.1863702917934554,0.1104624779531802,0.0906612914101508,Energy,7 -ett2/H/long,"TSOrchestra-test",231.57209114114323,231.57209114114323,9.454091518556336,1.041781590815038,0.1789606874661358,0.1769398075184539,23.227294463611894,15.217492932186406,0.2083918534587461,0.129466506940411,0.1038574614019068,Energy,7 -ett2/H/medium,"TSOrchestra-test",235.8876371451095,235.8876371451095,9.493494757896926,1.0268141732646603,0.172210494271238,0.1679117098861441,24.048873198636727,15.358633960906468,0.2069674693911224,0.127930946900687,0.1035922956956065,Energy,7 -ett2/H/short,"TSOrchestra-test",105.45599173791662,105.45599173791662,6.390917671787323,0.738693513701533,0.1212411967967107,0.1107537717126933,19.584927308662287,10.26917677995255,0.1302417345616434,0.0810546181889846,0.0644437352637196,Energy,7 -ett2/W/short,"TSOrchestra-test",2995530.0831473866,2995530.0831473866,1182.5991714157824,0.779092255795969,0.1191979958307905,0.1223002050555992,16.211684881804597,1730.7599727135437,0.1451842275430943,0.0992019401314874,0.0838411613763393,Energy,7 -hierarchical_sales/D/short,"TSOrchestra-test",28.4842233160427,28.4842233160427,2.27820184536824,0.7376950524573519,0.6116411310292618,1.0525248787733132,22.419521369438403,5.337061299633226,1.637351302338999,0.6989270965954891,0.565355362894863,Sales,1 -hierarchical_sales/W/short,"TSOrchestra-test",460.9457036024806,460.9457036024806,8.885153464363665,0.7169887615970751,0.5251711196614735,0.4541357911563696,20.13071392018048,21.469646098678027,0.9856819539753212,0.4079217416007538,0.3429796702761664,Sales,1 -hospital/M/short,"TSOrchestra-test",2373.4745643489155,2373.4745643489155,17.023377762327495,0.7458241606357002,0.189495797554006,0.170684745246498,18.590377542384097,48.71831857062511,0.1768588094952555,0.061798814346778,0.049108556667034,Healthcare,1 -jena_weather/10T/long,"TSOrchestra-test",1414.038540847993,1414.038540847993,9.557618991493175,0.633012747166124,0.6617059580775166,0.6388649964314539,16.42150816183156,37.60370381821441,0.2301035116259308,0.0584847094679009,0.0482795008650062,Nature,21 -jena_weather/10T/medium,"TSOrchestra-test",1467.4276163978743,1467.4276163978743,9.595084251816562,0.5996706442160938,0.7198642768035857,0.648576785130473,14.782994315926958,38.30701784788101,0.2350895136614313,0.0588848680222908,0.048503394488175,Nature,21 -jena_weather/10T/short,"TSOrchestra-test",800.5215229688597,800.5215229688597,5.542351885082043,0.2950321584416642,0.338764170383925,0.5515697860457086,6.855517361233827,28.29348905612137,0.1757380346416513,0.0344249528803179,0.028285874272936,Nature,21 -jena_weather/D/short,"TSOrchestra-test",366.93209755426886,366.93209755426886,9.516880473985957,1.0522399271954166,0.6196601375932497,0.4644041943836069,25.908594679068464,19.15547173927776,0.1153472480373017,0.057307175073288,0.0446467735129981,Nature,21 -jena_weather/H/long,"TSOrchestra-test",1149.852669418736,1149.852669418736,10.966845953812722,0.986955345172656,2.2202380906971317,0.652203920740879,28.89533113111513,33.90947757513725,0.2041726241830413,0.066032563092121,0.0556120978495563,Nature,21 -jena_weather/H/medium,"TSOrchestra-test",1273.604755862531,1273.604755862531,10.144651732280815,0.8207946346005405,2.014562790647134,0.683665593626558,21.53640461772101,35.687599469038695,0.2183787504572522,0.0620769231352046,0.0495441452055206,Nature,21 -jena_weather/H/short,"TSOrchestra-test",1099.3809380776306,1099.3809380776306,8.115456017093539,0.5223417034546075,1.0781263933868823,0.6039790013943063,14.09786143635967,33.15691388048096,0.2032497556828221,0.0497472249279499,0.0403982573536892,Nature,21 -kdd_cup_2018/D/short,"TSOrchestra-test",2759.2638939349863,2759.2638939349863,20.57910820251712,1.1673717958082008,0.5392989166170776,0.4504731666720218,36.16275939410855,52.528695909331184,1.1766985835522044,0.4609938825526378,0.3634636466224152,Nature,1 -kdd_cup_2018/H/long,"TSOrchestra-test",3330.0505673663147,3330.0505673663147,17.381728495064245,0.738931517147235,0.6383453741950432,0.4406006791305596,18.97185321529629,57.70659032871648,1.3540297662639584,0.4078455794627592,0.3286952493268724,Nature,1 -kdd_cup_2018/H/medium,"TSOrchestra-test",4365.935427859983,4365.935427859983,19.35674283586879,0.7958567399779825,0.7012709315728645,0.426179745246127,19.614676915940997,66.07522552258133,1.3830740625693372,0.4051716621529448,0.3292789169351806,Nature,1 -kdd_cup_2018/H/short,"TSOrchestra-test",3730.788196117708,3730.788196117708,16.06121059342498,0.6575140143600239,0.5481967557864017,0.3606052391858282,16.347406807524145,61.08017842244493,1.278518758261331,0.3361902265913107,0.2720765810906044,Nature,1 -loop_seattle/5T/long,"TSOrchestra-test",97.99790625894012,97.99790625894012,5.952054784144496,0.9361972757576484,0.2059163973364845,0.1340509766699815,21.125389541962548,9.899389186153869,0.1750232359936708,0.105233552246805,0.0827925997498006,Transport,1 -loop_seattle/5T/medium,"TSOrchestra-test",89.49789364680997,89.49789364680997,5.664116810426221,0.8918757271521369,0.1974732176457712,0.1290703868530394,20.9293094401421,9.46033263933198,0.1683187587982904,0.1007762779139227,0.0790344109865617,Transport,1 -loop_seattle/5T/short,"TSOrchestra-test",42.92612809658386,42.92612809658386,3.5651197225389164,0.5628349487096386,0.1030390999397972,0.0756535524560667,14.86407256881877,6.551803423225079,0.1123489641938595,0.0611339324733869,0.0485993845601641,Transport,1 -loop_seattle/D/short,"TSOrchestra-test",17.530503625863297,17.530503625863297,2.9046756926056467,0.8744281573114658,0.0540667389572972,0.0536054626782722,22.6225832164226,4.186944425934419,0.0748262901940435,0.0519104349578245,0.0416299848994604,Transport,1 -loop_seattle/H/long,"TSOrchestra-test",59.08106273202914,59.08106273202914,4.226861725092698,0.8833852289631207,0.1274596495010389,0.097887165859567,19.156786980084394,7.686420671029471,0.1373666201283598,0.0755396736369552,0.0597525848895843,Transport,1 -loop_seattle/H/medium,"TSOrchestra-test",66.40178046472927,66.40178046472927,4.559420433347472,0.93393472025802,0.1211754643722498,0.1046869654728148,18.851500507584745,8.148728763723165,0.1440278689034534,0.0805872458748838,0.0635718381799733,Transport,1 -loop_seattle/H/short,"TSOrchestra-test",59.67731116386764,59.67731116386764,4.156474241426581,0.8442688039515677,0.114818951892185,0.0956195001107355,20.73211595397908,7.725109136048995,0.1367422895295372,0.0735738219530071,0.0588567012147225,Transport,1 -m4_daily/D/short,"TSOrchestra-test",341419.66447293625,341419.66447293625,163.23850965974492,3.0544616206385644,0.0362288075067664,0.0284196726593063,72.13090235719206,584.3112736144462,0.0902599431206534,0.0252158383079749,0.0200191578669637,Econ/Fin,1 -m4_hourly/H/short,"TSOrchestra-test",641692.1397090574,641692.1397090574,176.22853989723487,0.7044492460415077,0.089566523667005,0.0827726426622496,12.75995912086896,801.0568891839439,0.1093619531212013,0.024059086912657,0.019437390732106,Econ/Fin,1 -m4_monthly/M/short,"TSOrchestra-test",1805253.1834707716,1805253.1834707716,553.1512365861266,0.9246498736817312,0.1599700046918361,0.1309469674389335,24.214065341792832,1343.5971060815707,0.2792456030329347,0.1149638161095658,0.0916051730657556,Econ/Fin,1 -m4_quarterly/Q/short,"TSOrchestra-test",1749923.9488398822,1749923.9488398822,551.4261801567609,1.164283693257738,0.1118402811837156,0.100168651615758,26.53019068763368,1322.846910583338,0.2214212776754138,0.0922990320173491,0.0735520572727876,Econ/Fin,1 -m4_weekly/W/short,"TSOrchestra-test",226531.90535031445,226531.90535031445,242.1903553044085,1.8754249439292183,0.0599904930172875,0.0592667849429472,39.99205063284441,475.9536798369296,0.0867113759883913,0.0441233251243227,0.0353332559292307,Econ/Fin,1 -m4_yearly/A/short,"TSOrchestra-test",3393220.25587344,3393220.25587344,869.2719107131835,3.251692440959419,0.1694425371849296,0.1447013269406119,68.45215372679151,1842.0695578271304,0.2953843917077551,0.1393917800137427,0.1127099181619983,Econ/Fin,1 -m_dense/D/short,"TSOrchestra-test",8459.138268785375,8459.138268785375,44.42356322818332,0.67511425545139,0.1238765756079324,0.0971225064439763,15.4165918205579,91.97357375238484,0.1592198608390708,0.0769037590537641,0.0653445212396726,Transport,1 -m_dense/H/long,"TSOrchestra-test",35604.71778174727,35604.71778174727,82.61866302423915,0.7107365685947067,0.2654010896389083,0.1970646196219469,14.992896901440664,188.69212432358503,0.3266539609722422,0.1430251189541852,0.120083552253017,Transport,1 -m_dense/H/medium,"TSOrchestra-test",35131.699704776795,35131.699704776795,80.668470977089,0.7019422396840811,0.2666796113477175,0.1951980872603445,14.9411773403438,187.43452111277895,0.3261123869005416,0.1403529481216893,0.1163188954216024,Transport,1 -m_dense/H/short,"TSOrchestra-test",43103.062087056256,43103.062087056256,88.52637195974556,0.7787776034677959,0.2912347058323554,0.2120909966830781,19.144305301479204,207.61276956646057,0.3682533421048231,0.1570237341694683,0.128942593000014,Transport,1 -restaurant/D/short,"TSOrchestra-test",137.42741059064306,137.42741059064306,7.020433411467456,0.6730968324205545,0.6454114461708541,0.3886357990457914,18.529050397248582,11.722943768125951,0.5395170339969592,0.3230966117765124,0.2531340237625726,Sales,1 -saugeen/D/short,"TSOrchestra-test",1050.320747839035,1050.320747839035,12.879756155241266,2.861161185196317,0.36629479902698,0.3458412403042566,76.47755042403,32.408652360735935,1.0497239718253513,0.4171783706687537,0.3387762405778192,Nature,1 -saugeen/M/short,"TSOrchestra-test",348.2943444549976,348.2943444549976,11.678133898022034,0.6842701086016695,0.3259618263649427,0.3365956167539972,20.7502319488856,18.6626456981586,0.5603086112714382,0.3506126136921869,0.2713357354698533,Nature,1 -saugeen/W/short,"TSOrchestra-test",924.9944024175212,924.9944024175212,14.222742743791722,1.153730949496004,0.3593660414554995,0.3760323877275426,32.11340328103771,30.413720627662794,0.918063446925016,0.4293253162921753,0.346267406142088,Nature,1 -solar/10T/long,"TSOrchestra-test",19.099645840150583,19.099645840150583,1.8209082124404703,0.7950534034317218,3.080609587161629,1.455984132663277,12.771292977760911,4.370314158061247,0.9445735061039228,0.3935601863645742,0.3077411716670427,Energy,1 -solar/10T/medium,"TSOrchestra-test",20.871416005775963,20.871416005775963,1.88365729542199,0.8237387751136988,3.298937344373401,1.458104451664151,12.08122416681055,4.568524488910612,1.0137737952699262,0.4179910625417683,0.3161087705778983,Energy,1 -solar/10T/short,"TSOrchestra-test",26.24325131775436,26.24325131775436,2.2342662142964027,0.9851943846829878,4.658959506384312,1.5350574242791053,21.28437318105071,5.122816736694214,1.4902268339377496,0.6499477997828199,0.4916986854738969,Energy,1 -solar/D/short,"TSOrchestra-test",119227.47752059253,119227.47752059253,251.9462696410496,0.982337273668483,1.0181197210617945,0.4326281193173633,27.677108550760195,345.293320990419,0.4988312568531653,0.3639765576235937,0.2715448867667153,Energy,1 -solar/H/long,"TSOrchestra-test",566.4859775721334,566.4859775721334,9.62192978445607,0.7329388708800524,4.323821629920818,1.3771451559142622,6.268021912153806,23.800965895781065,0.8252229633442273,0.3336098814051239,0.2532065481542812,Energy,1 -solar/H/medium,"TSOrchestra-test",589.1484997733236,589.1484997733236,9.45801900027286,0.7260339945860715,3.827431781763633,1.3807847930722088,4.335409182633233,24.27238141949248,0.8743477933272837,0.3406999049337208,0.2604156398735464,Energy,1 -solar/H/short,"TSOrchestra-test",566.910647534711,566.910647534711,9.268722056456992,0.7143731464339246,3.6350369170476338,1.379413747645156,16.066816934328212,23.80988550024361,0.877640322696474,0.3416481871166332,0.2653870519859833,Energy,1 -solar/W/short,"TSOrchestra-test",953565.8694580324,953565.8694580324,736.751263503925,0.7953183181692459,0.1510129593094157,0.1451373471139387,18.56209025015053,976.5069735839228,0.1993294871877068,0.1503893525717984,0.1190884689883655,Energy,1 -sz_taxi/15T/long,"TSOrchestra-test",15.465088285040917,15.465088285040917,2.684776207650952,0.5065839297050271,6714525898278.196,0.3963712690634701,13.123436539017252,3.932567645322953,0.3639914561007082,0.2484981033421369,0.1960624504105242,Transport,1 -sz_taxi/15T/medium,"TSOrchestra-test",16.29573928427503,16.29573928427503,2.752058424545861,0.5364999836218196,10118253562112.38,0.4041915645206366,15.912307448354577,4.036798147576248,0.3756914478624134,0.2561249723968414,0.2017452550322822,Transport,1 -sz_taxi/15T/short,"TSOrchestra-test",16.655308314145863,16.655308314145863,2.725003821997501,0.5435464969946024,1083729224513.2222,0.392602362747154,17.1659401209527,4.081091559147609,0.3815995308405965,0.254799522368537,0.2002327513367735,Transport,1 -sz_taxi/H/short,"TSOrchestra-test",7.144328559341722,7.144328559341722,1.8398692047442256,0.560828326808545,1.187600856742935,0.2941186632699729,17.046522493255075,2.6728876817669915,0.2489959876967143,0.1713951742129332,0.1347733263741267,Transport,1 -temperature_rain/D/short,"TSOrchestra-test",182.28311460355567,182.28311460355567,5.788020299834798,1.3407939179828552,20.01165346730017,1.5229168981955554,44.21261729394454,13.501226411091537,1.5894513096674106,0.6814030196839135,0.5496268645399822,Nature,1 -us_births/D/short,"TSOrchestra-test",122046.34669101665,122046.34669101665,214.16335928297883,0.3152231367580344,0.020652934573644,0.0204011324283101,7.500468688845998,349.35132272687423,0.0327488516831706,0.0200760770973555,0.015865867586829,Healthcare,1 -us_births/M/short,"TSOrchestra-test",51693736.38319152,51693736.38319152,6114.71563915383,0.6911864780572511,0.0188061067664586,0.0190413634240933,14.268876843587904,7189.835629775657,0.0223315804464031,0.0189922650856072,0.0138570043642932,Healthcare,1 -us_births/W/short,"TSOrchestra-test",2047159.4150952871,2047159.4150952871,1114.3205312669563,1.01342062000649,0.0151113020863059,0.0151636780246314,20.276625077197963,1430.789787178846,0.019422831065682,0.0151267919478892,0.0117630325186122,Healthcare,1 -electricity/15T/long,"TSOrchestra-test",337020.59082676214,337020.59082676214,60.41434821922858,0.902348998043153,0.14773511177359833,0.138944831092834,23.37094318280884,580.5347455809705,0.9158647272580135,0.09531104033902663,0.0759710850923222,Energy,1 -electricity/15T/medium,"TSOrchestra-test",278809.17104931944,275698.84170026396,56.77634808868695,0.8830145824975804,0.1490597210525614,0.13666913433666478,7.348450202429224,528.0238356829353,0.9098382357829771,0.09783136458678733,0.07882560778436892,Energy,1 -electricity/15T/short,"TSOrchestra-test",148674.4875842627,148674.4875842627,56.10791376586797,1.0677924492260855,0.17636132552140887,0.16946602763004223,30.880625096852786,385.5833082282773,0.7600607120958128,0.1105997588097062,0.09163342138798426,Energy,1 -electricity/H/short,"TSOrchestra-test",1708259.4721944854,1708259.4721944854,181.22039162414148,1.0125950798453054,0.21242744102681949,0.13343124038907841,27.39378087424175,1307.004006189149,0.6169992357073165,0.08554896740729222,0.06964545031616531,Energy,1 diff --git a/results/TSOrchestra-test/config.json b/results/TSOrchestra-test/config.json deleted file mode 100644 index 793f3e893b0d8498ae9e27655ed90143cc28969f..0000000000000000000000000000000000000000 --- a/results/TSOrchestra-test/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "model": "TSOrchestra-test", - "model_type": "fine-tuned", - "model_dtype": "float32", - "model_link": "https://huggingface.co/Melady/TEMPO", - "org": "Melady Lab @ USC", - "testdata_leakage": "Yes", - "replication_code_available": "No" -} \ No newline at end of file diff --git a/results/TSOrchestra/all_results.csv b/results/TSOrchestra/all_results.csv deleted file mode 100644 index 438ae957f04f33d24a6aeb7801d2dfe225a661e7..0000000000000000000000000000000000000000 --- a/results/TSOrchestra/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/5T/long,TSOrchestra,5264177.712564898,5264177.712564898,388.0607584403318,0.901473426367395,3.4958704288678635,0.8225785367482992,19.78252062537251,2294.379592082552,6.063360328637647,1.0255287381162714,0.8569765170641817,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,TSOrchestra,2629666.35881228,2629666.35881228,267.13693240499754,0.9755113575078024,3.497072296304505,0.8046394155192599,22.31743248298805,1621.6246047751863,4.926943452114991,0.8116357855296321,0.6244223901352561,Web/CloudOps,2 -bitbrains_fast_storage/5T/short,TSOrchestra,1598900.2355052335,1598900.2355052335,146.95083760491636,0.6656270729567958,1.701972644929568,0.727685243571386,13.181080607240563,1264.4762692534937,3.970142760173626,0.4613892867780263,0.365964340187127,Web/CloudOps,2 -bitbrains_fast_storage/H/short,TSOrchestra,2912910.151516303,2912910.151516303,287.49859311070884,1.0675768306007718,3.659132051164256,0.5831312763838772,22.55778526125413,1706.7249782892095,4.864720434315338,0.8194643533866295,0.6087710720492973,Web/CloudOps,2 -bitbrains_rnd/5T/long,TSOrchestra,2195425.021583008,2195425.021583008,172.2106230296033,3.333250711252761,1.8806075964333708,0.7172136397691566,120.6746930735072,1481.6966698967128,5.676046120403083,0.6597000982714744,0.6188268065990985,Web/CloudOps,2 -bitbrains_rnd/5T/medium,TSOrchestra,2319427.3742827107,2319427.3742827107,149.38444791950795,4.398683090057088,0.6034995579301339,0.7158910108844455,163.85520814343516,1522.966635971619,6.4078972973992325,0.6285365532556089,0.5979848836750823,Web/CloudOps,2 -bitbrains_rnd/5T/short,TSOrchestra,1473726.8591539084,1473726.8591539084,108.79970436662305,1.641868127256456,0.9423694118471172,0.6508360180064867,53.01885007776693,1213.971523205511,4.967221013854713,0.4451769810909985,0.3750148337304279,Web/CloudOps,2 -bitbrains_rnd/H/short,TSOrchestra,1845706.057453438,1845706.057453438,172.64039404718713,5.82570641067154,2.346208482457765,0.5546866192080948,196.37279957948388,1358.5676492002294,5.945756719893324,0.7555588296507487,0.6334180543872927,Web/CloudOps,2 -bizitobs_application/10S/long,TSOrchestra,6300652.343285634,6300652.343285634,1510.8551271272145,3.273066065427611,0.0475715841818773,0.0486614762046312,48.19020274993594,2510.110026131451,0.0968049105113743,0.0582676431927616,0.051692036906652,Web/CloudOps,2 -bizitobs_application/10S/medium,TSOrchestra,3141437.0235548923,3141437.0235548923,972.3970452281378,2.112017627778224,0.0300273240536565,0.0307198523157431,11.472651662318707,1772.4099479394974,0.069597322141417,0.0381831700306023,0.0305904860945084,Web/CloudOps,2 -bizitobs_application/10S/short,TSOrchestra,287095.4478290099,287095.4478290099,298.5058670468253,1.0443476467186994,0.0143907985096621,0.0145381525319758,6.394522497266609,535.8128850905042,0.0206641612726306,0.0115121781299457,0.0089604197896585,Web/CloudOps,2 -bizitobs_l2c/5T/long,TSOrchestra,83.0697872504921,83.0697872504921,5.642939220666352,0.5696564400947636,0.3978218093830707,0.7564623852529254,13.804302889347444,9.114262847344929,0.5117522330309839,0.3168426010311262,0.2892075038115793,Web/CloudOps,7 -bizitobs_l2c/5T/medium,TSOrchestra,61.70962552958317,61.70962552958317,4.927370302463452,0.48960438687586,0.3476752288925723,0.6210855772084146,7.861138640933592,7.855547436657942,0.41331731599612,0.2592521380279029,0.2234929985028826,Web/CloudOps,7 -bizitobs_l2c/5T/short,TSOrchestra,14.642827870443844,14.642827870443844,2.349119750534425,0.2440084874488821,0.1430302911092752,0.1886742734370725,2.0339245158186103,3.826594813988521,0.1314207547992615,0.0806782807525158,0.0652488445674679,Web/CloudOps,7 -bizitobs_l2c/H/long,TSOrchestra,101.02002269874252,101.02002269874252,5.79270528052348,0.6140380201230761,0.7098728268221247,0.8065474676895348,13.48080129651983,10.0508717382495,0.6139362543744982,0.3538351573113746,0.3186120537518076,Web/CloudOps,7 -bizitobs_l2c/H/medium,TSOrchestra,73.83455529516395,73.83455529516395,5.058936382243523,0.5084998332319391,0.5583875041249089,0.7863359079229258,8.072527822902423,8.592703608013252,0.5203012550324917,0.3063262820279485,0.2657579988717222,Web/CloudOps,7 -bizitobs_l2c/H/short,TSOrchestra,49.411486134753,49.411486134753,4.253030272676025,0.4223228430760049,0.3861958512891212,0.586142903277352,2.661657162434647,7.029330418663857,0.3788989119923153,0.229249223896013,0.1791046004349885,Web/CloudOps,7 -bizitobs_service/10S/long,TSOrchestra,129831.95913686778,129831.95913686778,75.44123178895673,1.2994572040936647,0.0828167071728293,0.077985466375148,18.741404874106536,360.322021443136,0.266938072690561,0.055889276304818,0.0511866544270041,Web/CloudOps,2 -bizitobs_service/10S/medium,TSOrchestra,39670.67651103585,39670.67651103585,44.52470586583992,1.0328695904209553,0.0713432420459128,0.0634714228586425,8.594384075020741,199.17498967248835,0.1499037908080025,0.0335103428642107,0.0256455515962885,Web/CloudOps,2 -bizitobs_service/10S/short,TSOrchestra,2000.266305347596,2000.266305347596,18.217186367952937,0.7267567470449694,0.0586501257235197,0.047384793134761,5.983825966646507,44.7243368351907,0.0331332195026727,0.0134958744469421,0.010455704230382,Web/CloudOps,2 -car_parts/M/short,TSOrchestra,1.3700726447974214,1.3700726447974214,0.4408185728037948,0.8195973504475315,0.8654278847865639,1.920954398310368,14.72832418262796,1.1705010229800832,2.8067413825967145,1.0570377182205044,0.9327451811489212,Sales,1 -covid_deaths/D/short,TSOrchestra,311992.3614518785,311992.3614518785,98.59279415444828,36.69020130413997,0.0950309243678922,0.3752581449153261,757.462743110833,558.5627641114994,0.2101619582759646,0.0370960186083255,0.0309701459662416,Healthcare,1 -electricity/15T/long,TSOrchestra,310425.9014339793,310425.9014339793,56.07605102932675,0.8395787020321144,0.1221391154146066,0.1270251062617314,7.107499451353006,557.1587757847661,0.8789862963787067,0.088466847426724,0.0707787446331732,Energy,1 -electricity/15T/medium,TSOrchestra,245566.43095452964,245566.43095452964,50.36244413874705,0.7903216412228886,0.1196486336044607,0.1249713373077373,7.083785740611692,495.54659816664025,0.8538767067306245,0.0867795644308659,0.0701145340638231,Energy,1 -electricity/15T/short,TSOrchestra,116434.440520035,116434.440520035,44.29597434719481,0.8624967938156302,0.1467345147876314,0.1499845461587394,10.16103748067126,341.2249119276537,0.6726215644733806,0.0873160971658479,0.0734790224803248,Energy,1 -electricity/D/short,TSOrchestra,1224080519.5167174,1224080519.5167174,4072.5329783997927,1.3610285160018476,0.3025935440578299,0.0918415183145351,10.85786686999034,34986.862098746686,0.5749503632197721,0.0669252449261329,0.0527147867229691,Energy,1 -electricity/H/long,TSOrchestra,8737538.278338678,8737538.278338678,260.2225376284691,1.0726853287241551,0.2751443207303686,0.1352372190753684,15.477120804705814,2955.932725611102,1.1658190092631169,0.1026316933323431,0.0867843767630209,Energy,1 -electricity/H/medium,TSOrchestra,5309387.3294271855,5309387.3294271855,223.158714682565,0.981482804326806,0.2272428230764321,0.1248897037373496,12.03473862540513,2304.2107823346337,0.8994093624032592,0.0871061965450988,0.0720375131385696,Energy,1 -electricity/H/short,TSOrchestra,1274750.2930848591,1274750.2930848591,155.29329891674394,0.8711263945444471,0.1614354892467023,0.118956384681309,10.417302208157947,1129.04840156871,0.5329914777456352,0.0733095275265689,0.0605888341827548,Energy,1 -electricity/W/short,TSOrchestra,48364210749.69932,48364210749.69932,28130.67533618961,1.4009793891654654,0.1731995060136251,0.0924915131305552,10.70808266738772,219918.6457526949,0.5017817762168063,0.0641849179639128,0.0489666350224332,Energy,1 -ett1/15T/long,TSOrchestra,8.502633824360304,8.502633824360304,1.5161477340920249,0.976190896208976,0.7991965710447076,0.3726871487766647,7.01577678909811,2.915927609588466,0.5557786854894926,0.2889792571978366,0.226670370262314,Energy,7 -ett1/15T/medium,TSOrchestra,8.860969280152908,8.860969280152908,1.5245030898220766,0.9638912242370012,0.774010885155166,0.3685173150450723,6.703608262125902,2.9767380267925674,0.5673692111344462,0.2905717962612809,0.225127574410428,Energy,7 -ett1/15T/short,TSOrchestra,4.5548065395560124,4.5548065395560124,1.0433138558969484,0.6627107203847414,0.481375563789342,0.2320841333667981,4.493944557858422,2.134199273628405,0.4034524990949027,0.1972297468672331,0.1528031943911897,Energy,7 -ett1/D/short,TSOrchestra,37569.48047298358,37569.48047298358,132.11503651198552,1.6687915589731777,1.140941643460078,0.48506038639743,10.852955982873452,193.82848209946744,0.5090969375906879,0.3470045256992627,0.2662914729773892,Energy,7 -ett1/H/long,TSOrchestra,140.7263311186074,140.7263311186074,6.963460643578585,1.3710534517725228,3460590802211.142,0.4735984831347956,14.704805596657438,11.862812951345369,0.5680844191708649,0.3334650484123063,0.2715791310055458,Energy,7 -ett1/H/medium,TSOrchestra,130.7154885645762,130.7154885645762,6.587860508131094,1.227015679073317,3327017308791.1543,0.4360795951628884,7.9559802390765935,11.433087446730047,0.5476177866601692,0.3155429018712262,0.2441497526406543,Energy,7 -ett1/H/short,TSOrchestra,91.83944910140734,91.83944910140734,4.744725942498384,0.7911264709577285,0.446807239111667,0.2505594145897483,5.486726935723474,9.583290097946913,0.447291355734865,0.2214557712142258,0.1704711432819546,Energy,7 -ett1/W/short,TSOrchestra,988239.5510698168,988239.5510698168,803.9342057586836,1.4413842667935195,0.6480276113750326,0.5068042283259001,8.008444048002799,994.1023846012124,0.3955770658189554,0.3199046085711649,0.2382186069678824,Energy,7 -ett2/15T/long,TSOrchestra,10.906570528816651,10.906570528816651,2.084904515936629,0.8651548992265667,0.1318447875485323,0.1592655139626665,6.938325731175051,3.3025097318276977,0.175996721585242,0.1111083356051278,0.0898259725801528,Energy,7 -ett2/15T/medium,TSOrchestra,11.085153001526876,11.085153001526876,2.0581154721893364,0.8479812392777987,0.1274211848919623,0.154523168562836,6.437763615212594,3.3294373400811845,0.1774317425715761,0.1096806989721488,0.0874697726212914,Energy,7 -ett2/15T/short,TSOrchestra,7.166351966570083,7.166351966570083,1.6654006014047462,0.7318496047955346,0.107077319205867,0.1263417749101006,5.353688242263978,2.6770042896062165,0.1270869169522746,0.0790624911378557,0.0633022745111527,Energy,7 -ett2/D/short,TSOrchestra,103787.35028606544,103787.35028606544,192.05004761541545,1.2717858005834717,0.4316229733444652,0.1317618389735265,12.512040230608996,322.1604418392572,0.1843809207811547,0.1099153093198916,0.0917119274639686,Energy,7 -ett2/H/long,TSOrchestra,222.8343717645239,222.8343717645239,9.062079041029108,0.9938115170679755,0.1718489760886219,0.1683400547663302,7.617442290544013,14.92763784945642,0.2044225111897064,0.1240981977757609,0.1013221653223104,Energy,7 -ett2/H/medium,TSOrchestra,222.28786093118632,222.28786093118632,9.266008274044712,1.0165991915547234,0.170351161744908,0.1684797752755465,7.214586118297748,14.90932127667743,0.2009126920289612,0.1248654202396949,0.1009739051872204,Energy,7 -ett2/H/short,TSOrchestra,104.38881570829506,104.38881570829506,6.253949926677166,0.7201984297442577,0.117663488550283,0.1076956236348013,4.968535065650653,10.21708450137783,0.1295810595275857,0.0793174860814754,0.0626647022353241,Energy,7 -ett2/W/short,TSOrchestra,3076388.739422191,3076388.739422191,1213.5099072356493,0.8074475839845398,0.1234139578241071,0.1303552044806999,8.901796228590152,1753.963722379169,0.1471306664048768,0.1017948769763117,0.0858445831893209,Energy,7 -hierarchical_sales/D/short,TSOrchestra,28.65776778607616,28.65776778607616,2.265505375458588,0.7349741957235518,0.5880808038151779,1.045506768438966,7.392867962591191,5.353295040073558,1.6423316341282623,0.6950319602321303,0.5649973387605687,Sales,1 -hierarchical_sales/W/short,TSOrchestra,454.05375215562304,454.05375215562304,8.772305450994908,0.7118111813530944,0.5232868112779785,0.4509312343472947,6.567601707612059,21.30853707216014,0.9782853597635984,0.4027408341089252,0.339992227803521,Sales,1 -hospital/M/short,TSOrchestra,2706.16266386736,2706.16266386736,17.678560827944356,0.7541515116233255,0.1895319242814566,0.1716049021878072,5.026695844999406,52.02079068860218,0.1888475501643325,0.0641772810177586,0.0502628852966397,Healthcare,1 -jena_weather/10T/long,TSOrchestra,1367.360743898416,1367.360743898416,9.45679984689938,0.6153420083676869,0.7024373442857476,0.6377755775446029,6.489222050013878,36.97784125524929,0.2262737512855751,0.0578677798345248,0.0486184534884789,Nature,21 -jena_weather/10T/medium,TSOrchestra,1402.0967004107497,1402.0967004107497,9.394945191622392,0.5926309300085866,0.6735300246304835,0.6495793938205904,5.629035784012749,37.444581723004326,0.229796757906691,0.057656618031322,0.0473077385012322,Nature,21 -jena_weather/10T/short,TSOrchestra,739.1265559557013,739.1265559557013,5.161388450222621,0.2793659524053724,0.3073376639876402,0.5408502639199575,2.4553468493107378,27.18688205653052,0.168864611471903,0.0320586911544367,0.0263968800056602,Nature,21 -jena_weather/D/short,TSOrchestra,362.93796922783406,362.93796922783406,9.35951137643174,0.974992120156175,0.6574550435042238,0.4332717556650878,6.230684154227491,19.050930928115665,0.1147177414900727,0.056359555898148,0.0441735332577431,Nature,21 -jena_weather/H/long,TSOrchestra,1170.0082255172315,1170.0082255172315,10.839875044363598,0.9462749258591938,2.1461700209261743,0.6544612236331708,6.936597416467992,34.20538299036033,0.2059543026238902,0.0652680575429071,0.0533239308989769,Nature,21 -jena_weather/H/medium,TSOrchestra,1117.7973908341417,1117.7973908341417,9.723950432775448,0.7873854779750674,2.3337507789491787,0.6946030390635268,7.106695110211534,33.43347709757604,0.204585375890694,0.0595025772708538,0.0480816857441824,Nature,21 -jena_weather/H/short,TSOrchestra,1084.9853786030358,1084.9853786030358,8.118496665118306,0.5083193095039987,1.206190023424702,0.5995148878949694,4.770251172818076,32.93911623894964,0.2019146620296428,0.0497658619142848,0.0408853515209117,Nature,21 -kdd_cup_2018/D/short,TSOrchestra,2953.077362197069,2953.077362197069,21.33902991058219,1.202172615033467,0.5135663260563103,0.4691432782438243,9.127078098569628,54.34222448701441,1.2173235499948267,0.4780169359905948,0.377505783346053,Nature,1 -kdd_cup_2018/H/long,TSOrchestra,3275.874733963697,3275.874733963697,18.46223006263726,0.774968644478529,0.7656532408243212,0.4689899116524367,11.045545046759432,57.23525778716906,1.3429704004717906,0.4331985118861584,0.3745503545715841,Nature,1 -kdd_cup_2018/H/medium,TSOrchestra,4450.612928924193,4450.612928924193,20.745775276851752,0.8403193290410146,0.8261441694976048,0.4637410615853204,11.697989131162313,66.71291425896632,1.39642210613223,0.4342466451546699,0.3763040845198839,Nature,1 -kdd_cup_2018/H/short,TSOrchestra,4077.067684649612,4077.067684649612,19.26172007828463,0.7971889638336594,0.753076205784193,0.4363784779510665,10.0519207872656,63.85191997622007,1.336536318644231,0.4031826835242207,0.3422814404886714,Nature,1 -loop_seattle/5T/long,TSOrchestra,81.07792364278055,81.07792364278055,5.353183931156703,0.8458632953061199,0.1758420895701546,0.121759381943452,6.457416198122687,9.004328050597698,0.1591983622289088,0.0946453871695278,0.0754862299023081,Transport,1 -loop_seattle/5T/medium,TSOrchestra,69.20785084230609,69.20785084230609,4.983048998001908,0.789722063291649,0.1695798905736172,0.1148399485001279,6.007284404063621,8.319125605633449,0.1480143409997241,0.088658682242895,0.0706484759419623,Transport,1 -loop_seattle/5T/short,TSOrchestra,35.65603957574113,35.65603957574113,3.306925597541728,0.5249706771670014,0.0886532083452657,0.0703297138890953,3.825523897514137,5.971267836543688,0.1023940606278516,0.0567064732977171,0.0448415636615264,Transport,1 -loop_seattle/D/short,TSOrchestra,17.78847961778851,17.78847961778851,2.868630194344143,0.8666274564465737,0.0532445132105997,0.0529421592515609,7.46955266060555,4.217639104734841,0.0753748494531603,0.0512662567057342,0.0418462594222781,Transport,1 -loop_seattle/H/long,TSOrchestra,60.397729392177006,60.397729392177006,4.1308797888078255,0.8637564806537221,0.120570596747324,0.0960153093584247,7.021677704982859,7.771597608740239,0.1388888566262963,0.073824353705981,0.0593585520464996,Transport,1 -loop_seattle/H/medium,TSOrchestra,68.77586023545186,68.77586023545186,4.416765022250802,0.904308069768835,0.1141777829202182,0.1009677872438146,7.78115746562433,8.293121260143966,0.1465799859398595,0.0780658252246449,0.0633245862968145,Transport,1 -loop_seattle/H/short,TSOrchestra,54.396354279882914,54.396354279882914,3.925675857916711,0.7992886529897059,0.1079996064118218,0.0902608355895633,7.281595697164838,7.375388415526528,0.1305518760669905,0.0694884552795897,0.0565810136355457,Transport,1 -m4_daily/D/short,TSOrchestra,355769.9125231597,355769.9125231597,177.8378660012413,3.2995769295718285,0.0375017495135556,0.0300147756021593,25.265304478579505,596.4645106988007,0.092137282851309,0.0274710355226881,0.0217718154112364,Econ/Fin,1 -m4_hourly/H/short,TSOrchestra,726819.9045574181,726819.9045574181,178.32164641081673,0.7074862280789587,0.0883624922179717,0.0809819844857741,5.900785848904817,852.537333233811,0.1163901638019081,0.0243448407783554,0.0199296247181108,Econ/Fin,1 -m4_monthly/M/short,TSOrchestra,1749042.3080492043,1749042.3080492043,535.1451673220623,0.8961719320900801,0.1524581577563677,0.1261345748885239,7.354080723349466,1322.5136324625182,0.2748637339024646,0.1112215369728296,0.0885745615599206,Econ/Fin,1 -m4_quarterly/Q/short,TSOrchestra,1707804.6020043525,1707804.6020043525,542.4916205866759,1.1396343937326596,0.1116825226735565,0.0983383682341368,8.702626299105429,1306.8299820574796,0.2187403260868371,0.0908035441608706,0.0724809874402638,Econ/Fin,1 -m4_weekly/W/short,TSOrchestra,241002.26398020607,241002.26398020607,253.16184762994305,1.9938194071484856,0.0616214474290862,0.0617122597179351,15.44891938542444,490.9198142061553,0.0894379821975302,0.0461221653031759,0.0365298513281534,Econ/Fin,1 -m4_yearly/A/short,TSOrchestra,3151355.892402279,3151355.892402279,813.2221025838802,2.986572287394955,0.1598681398135171,0.1342118129960987,28.66207364037356,1775.205873244644,0.2846624907953738,0.1304039338650136,0.1062224423074585,Econ/Fin,1 -m_dense/D/short,TSOrchestra,7548.7553040676385,7548.7553040676385,42.69386288590373,0.6429433015570036,0.1179432079702561,0.0933417835748855,6.687918340805185,86.88357326944858,0.1504083170934098,0.0739093919050818,0.0607345720647658,Transport,1 -m_dense/H/long,TSOrchestra,34985.15343176641,34985.15343176641,77.89172533611324,0.6790073261674846,0.2522616732973491,0.185693611915911,7.532192214566241,187.04318600731327,0.3237994049893685,0.1348420910475864,0.1111252189576625,Transport,1 -m_dense/H/medium,TSOrchestra,36329.79374944451,36329.79374944451,77.57854876331007,0.6742254783039038,0.2519448018188089,0.1850931518699299,8.126874894745093,190.6037611104369,0.3316264641055353,0.1349768738399078,0.1122078029594581,Transport,1 -m_dense/H/short,TSOrchestra,39662.86872111388,39662.86872111388,83.56477975406979,0.7532844146222547,0.280121403604371,0.2008755310818401,8.294578081854437,199.15538838081653,0.3532520326456076,0.1482230962752655,0.122603551063729,Transport,1 -restaurant/D/short,TSOrchestra,140.48485044511008,140.48485044511008,7.13922832449558,0.684479889695842,0.6768524620460686,0.3932912037246885,4.577511586636783,11.852630528499152,0.5454855191905386,0.3285638289191145,0.2572937243254204,Sales,1 -saugeen/D/short,TSOrchestra,1031.5953004531996,1031.5953004531996,11.882222943054549,2.6395981842409126,0.2664095796671774,0.3054031075792974,33.40782447889774,32.118457317455324,1.0403244852300488,0.3848680322483577,0.327738521328172,Nature,1 -saugeen/M/short,TSOrchestra,389.6522757522033,389.6522757522033,12.207400111764892,0.7153084955641655,0.3513048780828361,0.3559474479143182,4.770257872657849,19.739611844010597,0.5926423658380924,0.3665027732125175,0.2962968771743757,Nature,1 -saugeen/W/short,TSOrchestra,961.1118058881008,961.1118058881008,13.70013012942934,1.1115333220824897,0.3274210235727913,0.3609286873665441,14.21381846527961,31.00180326832781,0.9358152104391,0.4135498199620214,0.3473861588667375,Nature,1 -solar/10T/long,TSOrchestra,19.072231600715256,19.072231600715256,1.8093557002054788,0.7916271357300975,3.4959613872903046,1.44177916095407,4.106746328483359,4.367176616615735,0.943895360544706,0.3910632889224734,0.2905986800765083,Energy,1 -solar/10T/medium,TSOrchestra,19.535292751274547,19.535292751274547,1.8072338506785437,0.7900439667520253,3.620970226575423,1.4419782381757824,3.816507424704341,4.419874743844507,0.98078775847163,0.4010323686004028,0.2923231318893934,Energy,1 -solar/10T/short,TSOrchestra,14.463628095042484,14.463628095042484,1.482361704635389,0.6394082291592021,3.716553025687052,1.4859320327416938,3.982135579119115,3.803107689119845,1.1063236149139792,0.4312188593486321,0.3389465800320743,Energy,1 -solar/D/short,TSOrchestra,133473.64041520184,133473.64041520184,260.58363875435936,1.0164320421404245,1.1052705892765504,0.4358911023848787,4.540491904320637,365.3404445379704,0.5277925058033948,0.3764546020724946,0.2896335207708216,Energy,1 -solar/H/long,TSOrchestra,426.4887757339958,426.4887757339958,9.871971538197608,0.7500968224650443,4.282089187441105,1.3846369417451954,11.092614813677418,20.651604676973548,0.7160287706306995,0.3422792443861936,0.2929021733813005,Energy,1 -solar/H/medium,TSOrchestra,497.5666838768608,497.5666838768608,9.947122012658488,0.7585741798236169,3.6942675550517095,1.38777713077112,10.48299425502282,22.30620281170376,0.8035214882097256,0.3583185515923577,0.3032320000904033,Energy,1 -solar/H/short,TSOrchestra,553.2711290515517,553.2711290515517,9.95519717385518,0.7660174647846357,3.865679342716398,1.3897744900019104,8.411912566209931,23.521716116209543,0.8670182790858169,0.3669518788081871,0.3044867747827744,Energy,1 -solar/W/short,TSOrchestra,2571144.1325249397,2571144.1325249397,1291.993283860178,1.4181205261558905,0.2983187174601605,0.2466952103205555,5.537384610858478,1603.478759611408,0.3273101037841726,0.2637281306621184,0.2040665171403055,Energy,1 -sz_taxi/15T/long,TSOrchestra,15.587735303782,15.587735303782,2.6836383556071346,0.5067955601925199,7216114656101.197,0.3968168822719398,4.432063735938127,3.948130608754224,0.3654319362733322,0.2483927857838366,0.1998557901179174,Transport,1 -sz_taxi/15T/medium,TSOrchestra,16.26002945798758,16.26002945798758,2.7432885595225813,0.5350636625429297,10332014578591.74,0.4030856644124362,3.904442534581739,4.032372683419475,0.3752795845549752,0.2553087900741924,0.2020753575205816,Transport,1 -sz_taxi/15T/short,TSOrchestra,16.541380207629896,16.541380207629896,2.712826763495046,0.5412170713721057,1082673018134.4116,0.3921787787532371,3.815450876333277,4.067109564251976,0.3802921385647101,0.2536609046662987,0.1993444355970598,Transport,1 -sz_taxi/H/short,TSOrchestra,7.136339174596171,7.136339174596171,1.8362879284534213,0.5593445069328163,1.1252976721344528,0.2946877097387816,3.905490143798744,2.671392740612314,0.2488567186528636,0.1710615520621799,0.1346363397583088,Transport,1 -temperature_rain/D/short,TSOrchestra,175.18378758790365,175.18378758790365,5.675767891769761,1.3320718770543332,26.80574575856789,1.4788610109750635,20.770954382906584,13.235701250326846,1.5581919826394912,0.6681879453920052,0.5518110006380146,Nature,1 -us_births/D/short,TSOrchestra,136185.5533306498,136185.5533306498,221.81999999492183,0.3265413928981386,0.0212334882443231,0.0212017632881967,2.5705498539357,369.0332685960032,0.0345938743986226,0.0207938250340444,0.0165232988682387,Healthcare,1 -us_births/M/short,TSOrchestra,34401604.522149734,34401604.522149734,4815.480266309441,0.5434104346508024,0.0148486078210598,0.0149988336728121,4.027955583018929,5865.288102229057,0.0182175448564943,0.0149568488756274,0.0114364322610615,Healthcare,1 -us_births/W/short,TSOrchestra,1613257.4669588804,1613257.4669588804,981.7236995497444,0.8930040082350467,0.0133385143121964,0.0133650082345211,5.278024559486367,1270.1407272262709,0.0172420358291771,0.0133268029590342,0.0103840207472239,Healthcare,1 \ No newline at end of file diff --git a/results/TSOrchestra/config.json b/results/TSOrchestra/config.json deleted file mode 100644 index fe8fc64ecd9e2ebbda764b20ec4f24940e2e3bc5..0000000000000000000000000000000000000000 --- a/results/TSOrchestra/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "TSOrchestra", - "model_type": "agentic", - "model_dtype": "float32", - "model_link": "https://github.com/DC-research/TSorchestra", - "code_link": "https://github.com/DC-research/TSorchestra/blob/main/cli/eval.sh", - "org": "Melady Lab @ USC", - "testdata_leakage": "No", - "replication_code_available": "No" -} \ No newline at end of file diff --git a/results/TTM-R1-Pretrained/all_results.csv b/results/TTM-R1-Pretrained/all_results.csv deleted file mode 100644 index c25ff4929171059b08eaa7e4565f49dadaf906cb..0000000000000000000000000000000000000000 --- a/results/TTM-R1-Pretrained/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,TTM-R1-Pretrained,4839935.5,4839935.5,1196.179443359375,5.112272262573242,0.232824757695198,0.2032455503940582,204.4908905029297,2199.9853515625,0.3527778387069702,0.1918129175901413,0.1918129175901413,Econ/Fin,1.0 -m4_quarterly/Q/short,TTM-R1-Pretrained,2172303.25,2172303.25,762.6624145507812,1.9847232103347776,0.1766680777072906,0.1416244357824325,79.38892364501953,1473.87353515625,0.2467004805803299,0.1276562511920929,0.1276562511920929,Econ/Fin,1.0 -m4_monthly/M/short,TTM-R1-Pretrained,2287464.25,2287464.25,733.611572265625,1.4358307123184204,0.2279524058103561,0.1735519915819168,57.4332275390625,1512.4365234375,0.3143362402915954,0.152469664812088,0.152469664812088,Econ/Fin,1.0 -m4_weekly/W/short,TTM-R1-Pretrained,422039.78125,422039.78125,342.1238708496094,2.8267741203308105,0.0932142734527587,0.0928001403808593,113.07096099853516,649.6458740234375,0.1183554157614708,0.0623296685516834,0.0623296722769737,Econ/Fin,1.0 -m4_daily/D/short,TTM-R1-Pretrained,411048.875,411048.875,216.05897521972656,4.402057647705078,0.0544536747038364,0.040205355733633,176.08229064941406,641.1309204101562,0.0990370139479637,0.0333751402795314,0.0333751402795314,Econ/Fin,1.0 -m4_hourly/H/short,TTM-R1-Pretrained,1733123.125,1733123.125,274.3367919921875,2.284323215484619,0.1546705365180969,0.1287814080715179,91.3729248046875,1316.4813232421875,0.1797287613153457,0.0374530255794525,0.0374530255794525,Econ/Fin,1.0 -electricity/15T/short,TTM-R1-Pretrained,257502.15625,257502.15625,88.42657470703125,1.5933691263198853,0.2426293641328811,0.2388309836387634,63.73476409912109,507.44671630859375,1.0002775192260742,0.1743062138557434,0.1743062138557434,Energy,1.0 -electricity/H/short,TTM-R1-Pretrained,1847637.375,1847637.375,209.80812072753903,1.1588753461837769,0.2957945764064789,0.1582918018102646,46.35501480102539,1359.2783203125,0.6416764259338379,0.099044419825077,0.099044419825077,Energy,1.0 -electricity/D/short,TTM-R1-Pretrained,2313620736.0,2313620736.0,5347.56494140625,1.626549243927002,0.6676267981529236,0.1234149485826492,65.06197357177734,48100.11328125,0.7904446125030518,0.0878782495856285,0.0878782570362091,Energy,1.0 -electricity/W/short,TTM-R1-Pretrained,401613684736.0,401613684736.0,71897.9140625,2.820096492767334,0.3193942010402679,0.1503859013319015,112.8038558959961,633729.9375,1.44596266746521,0.1640473157167434,0.1640473008155822,Energy,1.0 -solar/10T/short,TTM-R1-Pretrained,24.345251083374023,24.345251083374023,2.662130355834961,1.1646230220794678,5.660563945770264,1.5489561557769775,46.584922790527344,4.934090614318848,1.4353264570236206,0.7744134068489075,0.7744134068489075,Energy,1.0 -solar/H/short,TTM-R1-Pretrained,620.20068359375,620.20068359375,12.117817878723145,0.9356493949890136,4.952815055847168,1.4163239002227783,37.42597579956055,24.903827667236328,0.9179633855819702,0.446666806936264,0.446666806936264,Energy,1.0 -solar/D/short,TTM-R1-Pretrained,118481.9609375,118481.9609375,253.32144165039065,0.9883174300193788,1.0210517644882202,0.4342885315418243,39.53269577026367,344.21209716796875,0.4972692430019378,0.3659631907939911,0.3659631907939911,Energy,1.0 -solar/W/short,TTM-R1-Pretrained,7674999.5,7674999.5,2448.83447265625,2.70544981956482,0.5475783348083496,0.4141778945922851,108.2179946899414,2770.37890625,0.5655035972595215,0.4998683333396911,0.4998683035373688,Energy,1.0 -hospital/M/short,TTM-R1-Pretrained,12836.142578125,12836.142578125,32.64528274536133,0.9913527369499208,0.2491465359926223,0.203258529305458,39.654109954833984,113.29669952392578,0.4112933278083801,0.1185099557042121,0.1185099706053733,Healthcare,1.0 -covid_deaths/D/short,TTM-R1-Pretrained,1131897.75,1131897.75,222.81866455078125,44.56435394287109,0.1296754330396652,0.4075009226799011,1782.57421875,1063.9068603515625,0.4003001153469085,0.0838366076350212,0.0838366150856018,Healthcare,1.0 -us_births/D/short,TTM-R1-Pretrained,1401044.5,1401044.5,1088.5601806640625,1.6032034158706665,0.1065103560686111,0.1045429110527038,64.12813568115234,1183.6572265625,0.1109582632780075,0.1020436882972717,0.1020436808466911,Healthcare,1.0 -us_births/M/short,TTM-R1-Pretrained,368808928.0,368808928.0,14531.2119140625,1.643504619598389,0.0464245378971099,0.0449309945106506,65.74018096923828,19204.3984375,0.0596487298607826,0.0451338477432727,0.0451338477432727,Healthcare,1.0 -us_births/W/short,TTM-R1-Pretrained,6060797.0,6060797.0,1816.890869140625,1.6510907411575315,0.0242670644074678,0.0244892444461584,66.04362487792969,2461.86865234375,0.0334196239709854,0.0246641151607036,0.0246641170233488,Healthcare,1.0 -saugeen/D/short,TTM-R1-Pretrained,1217.6488037109375,1217.6488037109375,18.195556640625,4.038330078125,0.736946702003479,0.5322720408439636,161.533203125,34.89482498168945,1.1302517652511597,0.589358389377594,0.589358389377594,Nature,1.0 -saugeen/M/short,TTM-R1-Pretrained,403.4919128417969,403.4919128417969,12.90295696258545,0.7559629678726196,0.430253803730011,0.3774499595165252,30.2385196685791,20.087108612060547,0.6030752658843994,0.3873854875564575,0.3873854577541351,Nature,1.0 -saugeen/W/short,TTM-R1-Pretrained,1417.9427490234375,1417.9427490234375,22.10640335083008,1.7919658422470093,0.9105833172798156,0.621161162853241,71.67863464355469,37.65558242797852,1.1366649866104126,0.6673001646995544,0.6673002243041992,Nature,1.0 -temperature_rain/D/short,TTM-R1-Pretrained,164.8345947265625,164.8345947265625,6.6282057762146,1.636677384376526,95.407470703125,1.397904634475708,65.46709442138672,12.838791847229004,1.511465311050415,0.7803150415420532,0.7803150415420532,Nature,1.0 -kdd_cup_2018/H/short,TTM-R1-Pretrained,4423.197265625,4423.197265625,24.123720169067383,0.9970476627349854,1.096936583518982,0.5092742443084717,39.88190841674805,66.50711822509766,1.392114520072937,0.504953145980835,0.504953145980835,Nature,1.0 -kdd_cup_2018/D/short,TTM-R1-Pretrained,2660.199462890625,2660.199462890625,20.4514102935791,1.1857279539108276,0.5912871360778809,0.4502138793468475,47.42911911010742,51.57712173461914,1.15538227558136,0.4581333100795746,0.4581332802772522,Nature,1.0 -car_parts/M/short,TTM-R1-Pretrained,1.949265718460083,1.949265718460083,0.7429021596908569,1.3475180864334106,0.5989811420440674,1.742508053779602,53.900726318359375,1.3961610794067385,3.347851037979126,1.7814031839370728,1.7814030647277832,Sales,1.0 -restaurant/D/short,TTM-R1-Pretrained,189.1906890869141,189.1906890869141,8.840666770935059,0.8296910524368286,0.9069666266441344,0.4570354223251343,33.18764114379883,13.754660606384276,0.633021354675293,0.4068679809570312,0.4068679809570312,Sales,1.0 -hierarchical_sales/D/short,TTM-R1-Pretrained,28.03666305541992,28.03666305541992,2.56404709815979,0.8271127343177795,0.8048070073127747,0.9882603287696838,33.084510803222656,5.294965744018555,1.624436855316162,0.7866212129592896,0.7866212129592896,Sales,1.0 -hierarchical_sales/W/short,TTM-R1-Pretrained,564.8724365234375,564.8724365234375,12.2916898727417,0.9056759476661682,1.0096296072006226,0.5597119331359863,36.22703552246094,23.76704406738281,1.0911566019058228,0.564317524433136,0.564317524433136,Sales,1.0 -loop_seattle/5T/short,TTM-R1-Pretrained,82.84699249267578,82.84699249267578,5.735431671142578,0.8927099704742432,0.1390590518712997,0.117562897503376,35.708396911621094,9.102032661437988,0.1560797691345214,0.0983499959111213,0.0983499959111213,Transport,1.0 -loop_seattle/H/short,TTM-R1-Pretrained,78.17028045654297,78.17028045654297,5.784073352813721,1.1600669622421265,0.1653453558683395,0.1292116194963455,46.402679443359375,8.841395378112793,0.1565016955137252,0.102383978664875,0.102383978664875,Transport,1.0 -loop_seattle/D/short,TTM-R1-Pretrained,39.735931396484375,39.735931396484375,5.312212944030762,1.637539029121399,0.100871056318283,0.0958108827471733,65.50155639648438,6.303644180297852,0.1126545518636703,0.0949363484978675,0.0949363484978675,Transport,1.0 -sz_taxi/15T/short,TTM-R1-Pretrained,18.08315086364746,18.08315086364746,2.9162304401397705,0.5838834047317505,1241597870080.0,0.4137653708457947,23.355337142944336,4.252429008483887,0.397620290517807,0.2726800143718719,0.2726800143718719,Transport,1.0 -sz_taxi/H/short,TTM-R1-Pretrained,7.639774799346924,7.639774799346924,1.977967381477356,0.6016649603843689,1.2737908363342283,0.3176651895046234,24.06659889221192,2.76401424407959,0.2574849724769592,0.1842598617076873,0.1842598617076873,Transport,1.0 -m_dense/H/short,TTM-R1-Pretrained,61866.4765625,61866.4765625,125.69860076904295,1.0748567581176758,0.4138478338718414,0.2934266030788421,42.99427032470703,248.729721069336,0.4411845505237579,0.2229579836130142,0.2229579985141754,Transport,1.0 -m_dense/D/short,TTM-R1-Pretrained,18425.615234375,18425.615234375,86.44058990478516,1.195383071899414,0.2065937370061874,0.1755319833755493,47.81532287597656,135.74098205566406,0.2349877208471298,0.1496414542198181,0.1496414393186569,Transport,1.0 -ett1/15T/short,TTM-R1-Pretrained,7.1254191398620605,7.1254191398620605,1.507292866706848,0.929407835006714,0.4586927890777588,0.3414961099624634,37.17631530761719,2.6693480014801025,0.5046178698539734,0.2849410772323608,0.2849410772323608,Energy,7.0 -ett1/H/short,TTM-R1-Pretrained,106.6740493774414,106.6740493774414,5.530933856964111,0.9026903510093688,0.49075448513031,0.2867715656757355,36.10761260986328,10.328312873840332,0.4820646345615387,0.2581512928009033,0.2581512928009033,Energy,7.0 -ett1/D/short,TTM-R1-Pretrained,56256.45703125,56256.45703125,158.85206604003906,1.9342973232269287,1.07516610622406,0.5437031984329224,77.37189483642578,237.18443298339844,0.6229727864265442,0.4172302186489105,0.4172302186489105,Energy,7.0 -ett1/W/short,TTM-R1-Pretrained,1469617.875,1469617.875,879.71875,1.447729229927063,0.7957108616828918,0.4806247055530548,57.9091682434082,1212.2779541015625,0.4823943376541137,0.3500610888004303,0.3500610888004303,Energy,7.0 -ett2/15T/short,TTM-R1-Pretrained,9.67288589477539,9.67288589477539,2.0208580493927,0.8757458925247192,0.122892290353775,0.1461191624403,35.02983856201172,3.110126256942749,0.1476487666368484,0.0959373190999031,0.0959373190999031,Energy,7.0 -ett2/H/short,TTM-R1-Pretrained,120.1141357421875,120.1141357421875,6.845362663269043,0.7833001017570496,0.1272993832826614,0.1162665784358978,31.332002639770508,10.959659576416016,0.1389989703893661,0.0868182405829429,0.0868182480335235,Energy,7.0 -ett2/D/short,TTM-R1-Pretrained,116203.3046875,116203.3046875,213.6118621826172,1.5472772121429443,0.4215363562107086,0.1568193733692169,61.891090393066406,340.88604736328125,0.1950980871915817,0.1222556978464126,0.1222556978464126,Energy,7.0 -ett2/W/short,TTM-R1-Pretrained,8909286.0,8909286.0,2076.41357421875,1.2180850505828855,0.2469210624694824,0.2033569812774658,48.72340393066406,2984.8427734375,0.2503825426101684,0.1741792559623718,0.1741792559623718,Energy,7.0 -jena_weather/10T/short,TTM-R1-Pretrained,782.1746826171875,782.1746826171875,7.494993209838867,0.3400033116340637,0.4026773571968078,0.5611201524734497,13.60013198852539,27.96738624572754,0.1737125217914581,0.0465533025562763,0.046553298830986,Nature,21.0 -jena_weather/H/short,TTM-R1-Pretrained,1094.249755859375,1094.249755859375,9.976097106933594,0.5863804817199707,1.7231212854385376,0.6161960363388062,23.455219268798828,33.079444885253906,0.2027748823165893,0.0611528344452381,0.0611528381705284,Nature,21.0 -jena_weather/D/short,TTM-R1-Pretrained,718.0636596679688,718.0636596679688,15.289955139160156,2.276055097579956,1.530358076095581,0.4862256646156311,91.04220581054688,26.796709060668945,0.1613599956035614,0.0920705199241638,0.0920705199241638,Nature,21.0 -bitbrains_fast_storage/5T/short,TTM-R1-Pretrained,1628531.125,1628531.125,187.12472534179688,1.171704649925232,2.403828859329224,0.7738839983940125,46.86818313598633,1276.13916015625,4.00676155090332,0.5875253677368164,0.5875253677368164,Web/CloudOps,2.0 -bitbrains_fast_storage/H/short,TTM-R1-Pretrained,2192248.25,2192248.25,295.1697998046875,1.3008264303207395,4.625766277313232,0.6319354772567749,52.03305816650391,1480.624267578125,4.220260143280029,0.8413297533988953,0.8413297533988953,Web/CloudOps,2.0 -bitbrains_rnd/5T/short,TTM-R1-Pretrained,1510926.75,1510926.75,147.6888885498047,2.705741167068481,1.7197303771972656,0.7114022970199585,108.22964477539062,1229.1976318359375,5.029521465301514,0.604300320148468,0.604300320148468,Web/CloudOps,2.0 -bitbrains_rnd/H/short,TTM-R1-Pretrained,1933281.5,1933281.5,244.4737243652344,5.975783824920654,5.3333740234375,0.6560726165771484,239.0313415527344,1390.4249267578125,6.085179805755615,1.0699366331100464,1.0699366331100464,Web/CloudOps,2.0 -bizitobs_application/10S/short,TTM-R1-Pretrained,6637856.5,6637856.5,1252.2286376953125,4.147694110870361,0.059621375054121,0.0600129775702953,165.9077606201172,2576.40380859375,0.0993615984916687,0.0482934527099132,0.0482934489846229,Web/CloudOps,2.0 -bizitobs_service/10S/short,TTM-R1-Pretrained,27536.26171875,27536.26171875,59.36934661865234,2.4597558975219727,0.4024703204631805,0.1219735294580459,98.3902359008789,165.94053649902344,0.1229340583086013,0.0439827106893062,0.0439827106893062,Web/CloudOps,2.0 -bizitobs_l2c/5T/short,TTM-R1-Pretrained,23.40999794006348,23.40999794006348,2.9665751457214355,0.308409571647644,0.1606444716453552,0.2220607250928878,12.336382865905762,4.838387966156006,0.1661698222160339,0.1018841937184333,0.1018842011690139,Web/CloudOps,7.0 -bizitobs_l2c/H/short,TTM-R1-Pretrained,205.36241149902344,205.36241149902344,9.765654563903809,0.9571609497070312,1.011512279510498,0.8193023204803467,38.28643798828125,14.33047103881836,0.7724491357803345,0.5263938307762146,0.5263938307762146,Web/CloudOps,7.0 -electricity/15T/medium,TTM-R1-Pretrained,651701.3125,651701.3125,105.96629333496094,1.6732470989227295,0.2470926195383072,0.2312248647212982,66.92987823486328,807.2802124023438,1.3910250663757324,0.1825906038284301,0.1825905889272689,Energy,1.0 -electricity/H/medium,TTM-R1-Pretrained,6365218.5,6365218.5,276.15478515625,1.2359539270401,0.3267567455768585,0.1584555804729461,49.43815612792969,2522.9384765625,0.9847859740257264,0.1077923104166984,0.107792317867279,Energy,1.0 -solar/10T/medium,TTM-R1-Pretrained,26.01571273803711,26.01571273803711,2.7237861156463623,1.1924405097961426,3.2334301471710205,1.5169990062713623,47.6976203918457,5.100560188293457,1.1318345069885254,0.6044189929962158,0.6044189929962158,Energy,1.0 -solar/H/medium,TTM-R1-Pretrained,660.3209228515625,660.3209228515625,12.42536163330078,0.9503023624420166,5.636295795440674,1.412003517150879,38.01209259033203,25.69671058654785,0.925655484199524,0.4475905001163482,0.447590559720993,Energy,1.0 -kdd_cup_2018/H/medium,TTM-R1-Pretrained,4856.63623046875,4856.63623046875,25.49256134033203,1.081132888793945,1.1571370363235474,0.5635550618171692,43.24531936645508,69.68956756591797,1.458728909492493,0.5336055159568787,0.5336055159568787,Nature,1.0 -loop_seattle/5T/medium,TTM-R1-Pretrained,283.7026062011719,283.7026062011719,12.758612632751465,1.9841841459274288,0.320991039276123,0.2810423076152801,79.36736297607422,16.843473434448242,0.2996800243854522,0.2270019352436065,0.2270019352436065,Transport,1.0 -loop_seattle/H/medium,TTM-R1-Pretrained,90.72705841064452,90.72705841064452,6.5600266456604,1.3280991315841677,0.1687787175178527,0.143565759062767,53.12396240234375,9.52507495880127,0.1683546304702758,0.1159477308392524,0.1159477308392524,Transport,1.0 -sz_taxi/15T/medium,TTM-R1-Pretrained,18.11094856262207,18.11094856262207,2.979530096054077,0.5815892219543457,12430919860224.0,0.4278164207935333,23.263568878173828,4.255696296691895,0.3960635662078857,0.2772950232028961,0.2772950232028961,Transport,1.0 -m_dense/H/medium,TTM-R1-Pretrained,56997.91015625,56997.91015625,121.361083984375,1.0112087726593018,0.4383679628372192,0.2912840247154236,40.4483528137207,238.7423553466797,0.415381520986557,0.2111529558897018,0.2111529558897018,Transport,1.0 -ett1/15T/medium,TTM-R1-Pretrained,15.956341743469238,15.956341743469238,2.393866777420044,1.4170336723327637,0.7636879682540894,0.5661241412162781,56.68134689331055,3.994539022445679,0.7613630890846252,0.4562734067440033,0.4562733471393585,Energy,7.0 -ett1/H/medium,TTM-R1-Pretrained,147.88861083984375,147.88861083984375,7.473272800445557,1.3740049600601196,4030999822336.0,0.4816672801971435,54.96020126342773,12.160945892333984,0.5824804902076721,0.3579520583152771,0.3579520285129547,Energy,7.0 -ett2/15T/medium,TTM-R1-Pretrained,15.00575351715088,15.00575351715088,2.521174669265747,1.052714705467224,0.1634148955345153,0.1984668672084808,42.10858917236328,3.873726129531861,0.2064378708600998,0.1343579590320587,0.1343579739332199,Energy,7.0 -ett2/H/medium,TTM-R1-Pretrained,305.8819885253906,305.8819885253906,11.393686294555664,1.2107949256896973,0.1966899782419204,0.2058309763669967,48.43179702758789,17.48948097229004,0.2356820106506347,0.15353724360466,0.15353724360466,Energy,7.0 -jena_weather/10T/medium,TTM-R1-Pretrained,1537.4500732421875,1537.4500732421875,11.29836082458496,0.680471658706665,0.9417414665222168,0.6616309285163879,27.218868255615234,39.210330963134766,0.2406331300735473,0.0693378448486328,0.0693378448486328,Nature,21.0 -jena_weather/H/medium,TTM-R1-Pretrained,1495.2928466796875,1495.2928466796875,12.530884742736816,0.8401451706886292,3.385291337966919,0.6819665431976318,33.60580825805664,38.66901779174805,0.2366225719451904,0.0766787081956863,0.0766787081956863,Nature,21.0 -bitbrains_fast_storage/5T/medium,TTM-R1-Pretrained,2465359.25,2465359.25,268.6639404296875,2.242617607116699,3.788220882415772,0.819374680519104,89.70470428466797,1570.146240234375,4.770537853240967,0.8162752389907837,0.8162752389907837,Web/CloudOps,2.0 -bitbrains_rnd/5T/medium,TTM-R1-Pretrained,2117112.5,2117112.5,198.3486785888672,7.39147138595581,2.527876615524292,0.7744432091712952,295.6588439941406,1455.030029296875,6.122053146362305,0.8345540761947632,0.8345540761947632,Web/CloudOps,2.0 -bizitobs_application/10S/medium,TTM-R1-Pretrained,52785060.0,52785060.0,4526.90380859375,13.303730010986328,0.1894544214010238,0.1915669292211532,532.149169921875,7265.33251953125,0.2852882444858551,0.1777581870555877,0.1777581870555877,Web/CloudOps,2.0 -bizitobs_service/10S/medium,TTM-R1-Pretrained,191248.296875,191248.296875,196.9491271972656,7.098899841308594,1.7195463180541992,0.2512445151805877,283.9559936523437,437.3194274902344,0.3291369080543518,0.1482285559177398,0.1482285559177398,Web/CloudOps,2.0 -bizitobs_l2c/5T/medium,TTM-R1-Pretrained,200.64273071289065,200.64273071289065,10.290497779846191,1.0041199922561646,0.6574810743331909,1.018429160118103,40.164798736572266,14.164841651916504,0.7452789545059204,0.5414315462112427,0.5414315462112427,Web/CloudOps,7.0 -bizitobs_l2c/H/medium,TTM-R1-Pretrained,281.8178405761719,281.8178405761719,13.066701889038086,1.2794169187545776,1.4472025632858276,1.070118427276611,51.17667770385742,16.787431716918945,1.016504406929016,0.7912086248397827,0.7912086844444275,Web/CloudOps,7.0 -electricity/15T/long,TTM-R1-Pretrained,1342790.5,1342790.5,123.45643615722656,1.7905689477920532,0.2591364085674286,0.2366401106119156,71.62275695800781,1158.788330078125,1.8281307220458984,0.1947676688432693,0.1947676688432693,Energy,1.0 -electricity/H/long,TTM-R1-Pretrained,9800309.0,9800309.0,320.5233764648437,1.3680378198623655,0.3523074686527252,0.171502023935318,54.72151565551758,3130.544677734375,1.2346858978271484,0.1264143288135528,0.1264143288135528,Energy,1.0 -solar/10T/long,TTM-R1-Pretrained,27.684722900390625,27.684722900390625,2.788307905197144,1.218901753425598,2.6426501274108887,1.5232243537902832,48.75607299804688,5.261627197265625,1.137216567993164,0.602648138999939,0.602648138999939,Energy,1.0 -solar/H/long,TTM-R1-Pretrained,689.2293701171875,689.2293701171875,13.81916046142578,1.0523871183395386,6.962733268737793,1.4221395254135132,42.095481872558594,26.253177642822266,0.9102455377578736,0.4791354835033417,0.4791354835033417,Energy,1.0 -kdd_cup_2018/H/long,TTM-R1-Pretrained,3769.7626953125,3769.7626953125,23.29082298278809,1.0037591457366943,1.1482057571411133,0.5922478437423706,40.15036392211914,61.39839553833008,1.4406543970108032,0.5464968085289001,0.5464968085289001,Nature,1.0 -loop_seattle/5T/long,TTM-R1-Pretrained,265.4754333496094,265.4754333496094,12.66295337677002,1.972639441490173,0.3322114944458008,0.2708892822265625,78.90557861328125,16.29341697692871,0.288070946931839,0.2238836139440536,0.2238836139440536,Transport,1.0 -loop_seattle/H/long,TTM-R1-Pretrained,78.83251953125,78.83251953125,5.85994291305542,1.2037372589111328,0.184073269367218,0.1325975954532623,48.14948654174805,8.878767967224121,0.1586754769086837,0.1047250255942344,0.1047250181436538,Transport,1.0 -sz_taxi/15T/long,TTM-R1-Pretrained,16.81537437438965,16.81537437438965,2.87433123588562,0.5435872673988342,8260150624256.0,0.4148517251014709,21.74349021911621,4.10065507888794,0.3795493543148041,0.2660429775714874,0.2660429775714874,Transport,1.0 -m_dense/H/long,TTM-R1-Pretrained,60970.38671875,60970.38671875,126.48969268798828,1.0458335876464844,0.45860356092453,0.2954391837120056,41.833343505859375,246.9218292236328,0.4274581670761108,0.2189723551273346,0.2189723551273346,Transport,1.0 -ett1/15T/long,TTM-R1-Pretrained,16.58780860900879,16.58780860900879,2.484389305114746,1.488469123840332,0.8089041709899902,0.5957595705986023,59.53876495361328,4.072813510894775,0.7762822508811951,0.4735270738601684,0.4735270738601684,Energy,7.0 -ett1/H/long,TTM-R1-Pretrained,149.5264892578125,149.5264892578125,7.462743759155273,1.3938250541687012,2593034338304.0,0.4885442852973938,55.75300216674805,12.228102684020996,0.5855773687362671,0.3573746383190155,0.3573746681213379,Energy,7.0 -ett2/15T/long,TTM-R1-Pretrained,15.252047538757324,15.252047538757324,2.600973129272461,1.0827611684799194,0.167321115732193,0.2018365561962127,43.31044769287109,3.905386924743652,0.2081251442432403,0.1386105716228485,0.1386105567216873,Energy,7.0 -ett2/H/long,TTM-R1-Pretrained,288.520751953125,288.520751953125,11.162341117858888,1.2191311120986938,0.208345890045166,0.214778259396553,48.76524353027344,16.985898971557617,0.2326088100671768,0.1528596729040146,0.1528596729040146,Energy,7.0 -jena_weather/10T/long,TTM-R1-Pretrained,1562.930419921875,1562.930419921875,11.357297897338867,0.6737896203994751,0.8498238921165466,0.6583142876625061,26.95158576965332,39.53391647338867,0.2419148087501526,0.069497250020504,0.0694972574710846,Nature,21.0 -jena_weather/H/long,TTM-R1-Pretrained,1302.5177001953125,1302.5177001953125,14.308568954467772,1.0584625005722046,3.396106243133545,0.6726809144020081,42.3385009765625,36.09041213989258,0.2173042595386505,0.0861534401774406,0.08615343272686,Nature,21.0 -bitbrains_fast_storage/5T/long,TTM-R1-Pretrained,3354402.25,3354402.25,356.4161071777344,2.0263211727142334,4.350730895996094,0.8388842344284058,81.05284118652344,1831.5028076171875,4.840115070343018,0.9419013857841492,0.9419013261795044,Web/CloudOps,2.0 -bitbrains_rnd/5T/long,TTM-R1-Pretrained,1939892.375,1939892.375,193.2017059326172,6.739154815673828,2.378561496734619,0.7550832629203796,269.5661926269531,1392.8001708984375,5.335503578186035,0.7401121854782104,0.7401122450828552,Web/CloudOps,2.0 -bizitobs_application/10S/long,TTM-R1-Pretrained,63951968.0,63951968.0,5090.4970703125,11.375816345214844,0.1536046713590622,0.169856384396553,455.03265380859375,7996.99755859375,0.3084122240543365,0.1963201314210891,0.1963201314210891,Web/CloudOps,2.0 -bizitobs_service/10S/long,TTM-R1-Pretrained,310560.65625,310560.65625,242.35232543945312,6.615731239318848,0.4960593283176422,0.2466846257448196,264.6292419433594,557.2796630859375,0.4128506183624267,0.1795423477888107,0.1795423477888107,Web/CloudOps,2.0 -bizitobs_l2c/5T/long,TTM-R1-Pretrained,370.515869140625,370.515869140625,14.165494918823242,1.3303046226501465,0.9395979642868042,1.1797410249710083,53.212188720703125,19.248788833618164,1.0807907581329346,0.7953713536262512,0.7953712940216064,Web/CloudOps,7.0 -bizitobs_l2c/H/long,TTM-R1-Pretrained,224.97149658203125,224.97149658203125,11.681208610534668,1.1981686353683472,1.655575394630432,0.9281730651855468,47.92674255371094,14.99905014038086,0.9161852598190308,0.7135218977928162,0.7135219573974609,Web/CloudOps,7.0 diff --git a/results/TTM-R1-Pretrained/config.json b/results/TTM-R1-Pretrained/config.json deleted file mode 100644 index 8c944c89a478fb87c0e8f175c2c9521656383643..0000000000000000000000000000000000000000 --- a/results/TTM-R1-Pretrained/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "TTM-R1-Pretrained", - "model_type": "pretrained", - "model_dtype": "float32", - "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r1", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/ttm.ipynb", - "org": "IBM Research", - "testdata_leakage": "Yes", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/TTM-R1-Zeroshot/all_results.csv b/results/TTM-R1-Zeroshot/all_results.csv new file mode 100644 index 0000000000000000000000000000000000000000..a23c4a62fb4f57799e17df9ec799b3c01287f25e --- /dev/null +++ b/results/TTM-R1-Zeroshot/all_results.csv @@ -0,0 +1,98 @@ +dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates +m4_yearly/A/short,TTM-R1-Zeroshot,4839935.5,4839935.5,1196.179443359375,5.112272262573242,0.232824757695198,0.2032455503940582,204.4908905029297,2199.9853515625,0.3527778387069702,0.1918129175901413,0.1918129175901413,Econ/Fin,1.0 +m4_quarterly/Q/short,TTM-R1-Zeroshot,2172303.25,2172303.25,762.6624145507812,1.9847232103347776,0.1766680777072906,0.1416244357824325,79.38892364501953,1473.87353515625,0.2467004805803299,0.1276562511920929,0.1276562511920929,Econ/Fin,1.0 +m4_monthly/M/short,TTM-R1-Zeroshot,2287464.25,2287464.25,733.611572265625,1.4358307123184204,0.2279524058103561,0.1735519915819168,57.4332275390625,1512.4365234375,0.3143362402915954,0.152469664812088,0.152469664812088,Econ/Fin,1.0 +m4_weekly/W/short,TTM-R1-Zeroshot,422039.78125,422039.78125,342.1238708496094,2.8267741203308105,0.0932142734527587,0.0928001403808593,113.07096099853516,649.6458740234375,0.1183554157614708,0.0623296685516834,0.0623296722769737,Econ/Fin,1.0 +m4_daily/D/short,TTM-R1-Zeroshot,411048.875,411048.875,216.05897521972656,4.402057647705078,0.0544536747038364,0.040205355733633,176.08229064941406,641.1309204101562,0.0990370139479637,0.0333751402795314,0.0333751402795314,Econ/Fin,1.0 +m4_hourly/H/short,TTM-R1-Zeroshot,1733123.125,1733123.125,274.3367919921875,2.284323215484619,0.1546705365180969,0.1287814080715179,91.3729248046875,1316.4813232421875,0.1797287613153457,0.0374530255794525,0.0374530255794525,Econ/Fin,1.0 +electricity/15T/short,TTM-R1-Zeroshot,257502.15625,257502.15625,88.42657470703125,1.5933691263198853,0.2426293641328811,0.2388309836387634,63.73476409912109,507.44671630859375,1.0002775192260742,0.1743062138557434,0.1743062138557434,Energy,1.0 +electricity/H/short,TTM-R1-Zeroshot,1847637.375,1847637.375,209.80812072753903,1.1588753461837769,0.2957945764064789,0.1582918018102646,46.35501480102539,1359.2783203125,0.6416764259338379,0.099044419825077,0.099044419825077,Energy,1.0 +electricity/D/short,TTM-R1-Zeroshot,2313620736.0,2313620736.0,5347.56494140625,1.626549243927002,0.6676267981529236,0.1234149485826492,65.06197357177734,48100.11328125,0.7904446125030518,0.0878782495856285,0.0878782570362091,Energy,1.0 +electricity/W/short,TTM-R1-Zeroshot,401613684736.0,401613684736.0,71897.9140625,2.820096492767334,0.3193942010402679,0.1503859013319015,112.8038558959961,633729.9375,1.44596266746521,0.1640473157167434,0.1640473008155822,Energy,1.0 +solar/10T/short,TTM-R1-Zeroshot,24.345251083374023,24.345251083374023,2.662130355834961,1.1646230220794678,5.660563945770264,1.5489561557769775,46.584922790527344,4.934090614318848,1.4353264570236206,0.7744134068489075,0.7744134068489075,Energy,1.0 +solar/H/short,TTM-R1-Zeroshot,620.20068359375,620.20068359375,12.117817878723145,0.9356493949890136,4.952815055847168,1.4163239002227783,37.42597579956055,24.903827667236328,0.9179633855819702,0.446666806936264,0.446666806936264,Energy,1.0 +solar/D/short,TTM-R1-Zeroshot,118481.9609375,118481.9609375,253.32144165039065,0.9883174300193788,1.0210517644882202,0.4342885315418243,39.53269577026367,344.21209716796875,0.4972692430019378,0.3659631907939911,0.3659631907939911,Energy,1.0 +solar/W/short,TTM-R1-Zeroshot,7674999.5,7674999.5,2448.83447265625,2.70544981956482,0.5475783348083496,0.4141778945922851,108.2179946899414,2770.37890625,0.5655035972595215,0.4998683333396911,0.4998683035373688,Energy,1.0 +hospital/M/short,TTM-R1-Zeroshot,12836.142578125,12836.142578125,32.64528274536133,0.9913527369499208,0.2491465359926223,0.203258529305458,39.654109954833984,113.29669952392578,0.4112933278083801,0.1185099557042121,0.1185099706053733,Healthcare,1.0 +covid_deaths/D/short,TTM-R1-Zeroshot,1131897.75,1131897.75,222.81866455078125,44.56435394287109,0.1296754330396652,0.4075009226799011,1782.57421875,1063.9068603515625,0.4003001153469085,0.0838366076350212,0.0838366150856018,Healthcare,1.0 +us_births/D/short,TTM-R1-Zeroshot,1401044.5,1401044.5,1088.5601806640625,1.6032034158706665,0.1065103560686111,0.1045429110527038,64.12813568115234,1183.6572265625,0.1109582632780075,0.1020436882972717,0.1020436808466911,Healthcare,1.0 +us_births/M/short,TTM-R1-Zeroshot,368808928.0,368808928.0,14531.2119140625,1.643504619598389,0.0464245378971099,0.0449309945106506,65.74018096923828,19204.3984375,0.0596487298607826,0.0451338477432727,0.0451338477432727,Healthcare,1.0 +us_births/W/short,TTM-R1-Zeroshot,6060797.0,6060797.0,1816.890869140625,1.6510907411575315,0.0242670644074678,0.0244892444461584,66.04362487792969,2461.86865234375,0.0334196239709854,0.0246641151607036,0.0246641170233488,Healthcare,1.0 +saugeen/D/short,TTM-R1-Zeroshot,1217.6488037109375,1217.6488037109375,18.195556640625,4.038330078125,0.736946702003479,0.5322720408439636,161.533203125,34.89482498168945,1.1302517652511597,0.589358389377594,0.589358389377594,Nature,1.0 +saugeen/M/short,TTM-R1-Zeroshot,403.4919128417969,403.4919128417969,12.90295696258545,0.7559629678726196,0.430253803730011,0.3774499595165252,30.2385196685791,20.087108612060547,0.6030752658843994,0.3873854875564575,0.3873854577541351,Nature,1.0 +saugeen/W/short,TTM-R1-Zeroshot,1417.9427490234375,1417.9427490234375,22.10640335083008,1.7919658422470093,0.9105833172798156,0.621161162853241,71.67863464355469,37.65558242797852,1.1366649866104126,0.6673001646995544,0.6673002243041992,Nature,1.0 +temperature_rain/D/short,TTM-R1-Zeroshot,164.8345947265625,164.8345947265625,6.6282057762146,1.636677384376526,95.407470703125,1.397904634475708,65.46709442138672,12.838791847229004,1.511465311050415,0.7803150415420532,0.7803150415420532,Nature,1.0 +kdd_cup_2018/H/short,TTM-R1-Zeroshot,4423.197265625,4423.197265625,24.123720169067383,0.9970476627349854,1.096936583518982,0.5092742443084717,39.88190841674805,66.50711822509766,1.392114520072937,0.504953145980835,0.504953145980835,Nature,1.0 +kdd_cup_2018/D/short,TTM-R1-Zeroshot,2660.199462890625,2660.199462890625,20.4514102935791,1.1857279539108276,0.5912871360778809,0.4502138793468475,47.42911911010742,51.57712173461914,1.15538227558136,0.4581333100795746,0.4581332802772522,Nature,1.0 +car_parts/M/short,TTM-R1-Zeroshot,1.949265718460083,1.949265718460083,0.7429021596908569,1.3475180864334106,0.5989811420440674,1.742508053779602,53.900726318359375,1.3961610794067385,3.347851037979126,1.7814031839370728,1.7814030647277832,Sales,1.0 +restaurant/D/short,TTM-R1-Zeroshot,189.1906890869141,189.1906890869141,8.840666770935059,0.8296910524368286,0.9069666266441344,0.4570354223251343,33.18764114379883,13.754660606384276,0.633021354675293,0.4068679809570312,0.4068679809570312,Sales,1.0 +hierarchical_sales/D/short,TTM-R1-Zeroshot,28.03666305541992,28.03666305541992,2.56404709815979,0.8271127343177795,0.8048070073127747,0.9882603287696838,33.084510803222656,5.294965744018555,1.624436855316162,0.7866212129592896,0.7866212129592896,Sales,1.0 +hierarchical_sales/W/short,TTM-R1-Zeroshot,564.8724365234375,564.8724365234375,12.2916898727417,0.9056759476661682,1.0096296072006226,0.5597119331359863,36.22703552246094,23.76704406738281,1.0911566019058228,0.564317524433136,0.564317524433136,Sales,1.0 +loop_seattle/5T/short,TTM-R1-Zeroshot,82.84699249267578,82.84699249267578,5.735431671142578,0.8927099704742432,0.1390590518712997,0.117562897503376,35.708396911621094,9.102032661437988,0.1560797691345214,0.0983499959111213,0.0983499959111213,Transport,1.0 +loop_seattle/H/short,TTM-R1-Zeroshot,78.17028045654297,78.17028045654297,5.784073352813721,1.1600669622421265,0.1653453558683395,0.1292116194963455,46.402679443359375,8.841395378112793,0.1565016955137252,0.102383978664875,0.102383978664875,Transport,1.0 +loop_seattle/D/short,TTM-R1-Zeroshot,39.735931396484375,39.735931396484375,5.312212944030762,1.637539029121399,0.100871056318283,0.0958108827471733,65.50155639648438,6.303644180297852,0.1126545518636703,0.0949363484978675,0.0949363484978675,Transport,1.0 +sz_taxi/15T/short,TTM-R1-Zeroshot,18.08315086364746,18.08315086364746,2.9162304401397705,0.5838834047317505,1241597870080.0,0.4137653708457947,23.355337142944336,4.252429008483887,0.397620290517807,0.2726800143718719,0.2726800143718719,Transport,1.0 +sz_taxi/H/short,TTM-R1-Zeroshot,7.639774799346924,7.639774799346924,1.977967381477356,0.6016649603843689,1.2737908363342283,0.3176651895046234,24.06659889221192,2.76401424407959,0.2574849724769592,0.1842598617076873,0.1842598617076873,Transport,1.0 +m_dense/H/short,TTM-R1-Zeroshot,61866.4765625,61866.4765625,125.69860076904295,1.0748567581176758,0.4138478338718414,0.2934266030788421,42.99427032470703,248.729721069336,0.4411845505237579,0.2229579836130142,0.2229579985141754,Transport,1.0 +m_dense/D/short,TTM-R1-Zeroshot,18425.615234375,18425.615234375,86.44058990478516,1.195383071899414,0.2065937370061874,0.1755319833755493,47.81532287597656,135.74098205566406,0.2349877208471298,0.1496414542198181,0.1496414393186569,Transport,1.0 +ett1/15T/short,TTM-R1-Zeroshot,7.1254191398620605,7.1254191398620605,1.507292866706848,0.929407835006714,0.4586927890777588,0.3414961099624634,37.17631530761719,2.6693480014801025,0.5046178698539734,0.2849410772323608,0.2849410772323608,Energy,7.0 +ett1/H/short,TTM-R1-Zeroshot,106.6740493774414,106.6740493774414,5.530933856964111,0.9026903510093688,0.49075448513031,0.2867715656757355,36.10761260986328,10.328312873840332,0.4820646345615387,0.2581512928009033,0.2581512928009033,Energy,7.0 +ett1/D/short,TTM-R1-Zeroshot,56256.45703125,56256.45703125,158.85206604003906,1.9342973232269287,1.07516610622406,0.5437031984329224,77.37189483642578,237.18443298339844,0.6229727864265442,0.4172302186489105,0.4172302186489105,Energy,7.0 +ett1/W/short,TTM-R1-Zeroshot,1469617.875,1469617.875,879.71875,1.447729229927063,0.7957108616828918,0.4806247055530548,57.9091682434082,1212.2779541015625,0.4823943376541137,0.3500610888004303,0.3500610888004303,Energy,7.0 +ett2/15T/short,TTM-R1-Zeroshot,9.67288589477539,9.67288589477539,2.0208580493927,0.8757458925247192,0.122892290353775,0.1461191624403,35.02983856201172,3.110126256942749,0.1476487666368484,0.0959373190999031,0.0959373190999031,Energy,7.0 +ett2/H/short,TTM-R1-Zeroshot,120.1141357421875,120.1141357421875,6.845362663269043,0.7833001017570496,0.1272993832826614,0.1162665784358978,31.332002639770508,10.959659576416016,0.1389989703893661,0.0868182405829429,0.0868182480335235,Energy,7.0 +ett2/D/short,TTM-R1-Zeroshot,116203.3046875,116203.3046875,213.6118621826172,1.5472772121429443,0.4215363562107086,0.1568193733692169,61.891090393066406,340.88604736328125,0.1950980871915817,0.1222556978464126,0.1222556978464126,Energy,7.0 +ett2/W/short,TTM-R1-Zeroshot,8909286.0,8909286.0,2076.41357421875,1.2180850505828855,0.2469210624694824,0.2033569812774658,48.72340393066406,2984.8427734375,0.2503825426101684,0.1741792559623718,0.1741792559623718,Energy,7.0 +jena_weather/10T/short,TTM-R1-Zeroshot,782.1746826171875,782.1746826171875,7.494993209838867,0.3400033116340637,0.4026773571968078,0.5611201524734497,13.60013198852539,27.96738624572754,0.1737125217914581,0.0465533025562763,0.046553298830986,Nature,21.0 +jena_weather/H/short,TTM-R1-Zeroshot,1094.249755859375,1094.249755859375,9.976097106933594,0.5863804817199707,1.7231212854385376,0.6161960363388062,23.455219268798828,33.079444885253906,0.2027748823165893,0.0611528344452381,0.0611528381705284,Nature,21.0 +jena_weather/D/short,TTM-R1-Zeroshot,718.0636596679688,718.0636596679688,15.289955139160156,2.276055097579956,1.530358076095581,0.4862256646156311,91.04220581054688,26.796709060668945,0.1613599956035614,0.0920705199241638,0.0920705199241638,Nature,21.0 +bitbrains_fast_storage/5T/short,TTM-R1-Zeroshot,1628531.125,1628531.125,187.12472534179688,1.171704649925232,2.403828859329224,0.7738839983940125,46.86818313598633,1276.13916015625,4.00676155090332,0.5875253677368164,0.5875253677368164,Web/CloudOps,2.0 +bitbrains_fast_storage/H/short,TTM-R1-Zeroshot,2192248.25,2192248.25,295.1697998046875,1.3008264303207395,4.625766277313232,0.6319354772567749,52.03305816650391,1480.624267578125,4.220260143280029,0.8413297533988953,0.8413297533988953,Web/CloudOps,2.0 +bitbrains_rnd/5T/short,TTM-R1-Zeroshot,1510926.75,1510926.75,147.6888885498047,2.705741167068481,1.7197303771972656,0.7114022970199585,108.22964477539062,1229.1976318359375,5.029521465301514,0.604300320148468,0.604300320148468,Web/CloudOps,2.0 +bitbrains_rnd/H/short,TTM-R1-Zeroshot,1933281.5,1933281.5,244.4737243652344,5.975783824920654,5.3333740234375,0.6560726165771484,239.0313415527344,1390.4249267578125,6.085179805755615,1.0699366331100464,1.0699366331100464,Web/CloudOps,2.0 +bizitobs_application/10S/short,TTM-R1-Zeroshot,6637856.5,6637856.5,1252.2286376953125,4.147694110870361,0.059621375054121,0.0600129775702953,165.9077606201172,2576.40380859375,0.0993615984916687,0.0482934527099132,0.0482934489846229,Web/CloudOps,2.0 +bizitobs_service/10S/short,TTM-R1-Zeroshot,27536.26171875,27536.26171875,59.36934661865234,2.4597558975219727,0.4024703204631805,0.1219735294580459,98.3902359008789,165.94053649902344,0.1229340583086013,0.0439827106893062,0.0439827106893062,Web/CloudOps,2.0 +bizitobs_l2c/5T/short,TTM-R1-Zeroshot,23.40999794006348,23.40999794006348,2.9665751457214355,0.308409571647644,0.1606444716453552,0.2220607250928878,12.336382865905762,4.838387966156006,0.1661698222160339,0.1018841937184333,0.1018842011690139,Web/CloudOps,7.0 +bizitobs_l2c/H/short,TTM-R1-Zeroshot,205.36241149902344,205.36241149902344,9.765654563903809,0.9571609497070312,1.011512279510498,0.8193023204803467,38.28643798828125,14.33047103881836,0.7724491357803345,0.5263938307762146,0.5263938307762146,Web/CloudOps,7.0 +electricity/15T/medium,TTM-R1-Zeroshot,651701.3125,651701.3125,105.96629333496094,1.6732470989227295,0.2470926195383072,0.2312248647212982,66.92987823486328,807.2802124023438,1.3910250663757324,0.1825906038284301,0.1825905889272689,Energy,1.0 +electricity/H/medium,TTM-R1-Zeroshot,6365218.5,6365218.5,276.15478515625,1.2359539270401,0.3267567455768585,0.1584555804729461,49.43815612792969,2522.9384765625,0.9847859740257264,0.1077923104166984,0.107792317867279,Energy,1.0 +solar/10T/medium,TTM-R1-Zeroshot,26.01571273803711,26.01571273803711,2.7237861156463623,1.1924405097961426,3.2334301471710205,1.5169990062713623,47.6976203918457,5.100560188293457,1.1318345069885254,0.6044189929962158,0.6044189929962158,Energy,1.0 +solar/H/medium,TTM-R1-Zeroshot,660.3209228515625,660.3209228515625,12.42536163330078,0.9503023624420166,5.636295795440674,1.412003517150879,38.01209259033203,25.69671058654785,0.925655484199524,0.4475905001163482,0.447590559720993,Energy,1.0 +kdd_cup_2018/H/medium,TTM-R1-Zeroshot,4856.63623046875,4856.63623046875,25.49256134033203,1.081132888793945,1.1571370363235474,0.5635550618171692,43.24531936645508,69.68956756591797,1.458728909492493,0.5336055159568787,0.5336055159568787,Nature,1.0 +loop_seattle/5T/medium,TTM-R1-Zeroshot,283.7026062011719,283.7026062011719,12.758612632751465,1.9841841459274288,0.320991039276123,0.2810423076152801,79.36736297607422,16.843473434448242,0.2996800243854522,0.2270019352436065,0.2270019352436065,Transport,1.0 +loop_seattle/H/medium,TTM-R1-Zeroshot,90.72705841064452,90.72705841064452,6.5600266456604,1.3280991315841677,0.1687787175178527,0.143565759062767,53.12396240234375,9.52507495880127,0.1683546304702758,0.1159477308392524,0.1159477308392524,Transport,1.0 +sz_taxi/15T/medium,TTM-R1-Zeroshot,18.11094856262207,18.11094856262207,2.979530096054077,0.5815892219543457,12430919860224.0,0.4278164207935333,23.263568878173828,4.255696296691895,0.3960635662078857,0.2772950232028961,0.2772950232028961,Transport,1.0 +m_dense/H/medium,TTM-R1-Zeroshot,56997.91015625,56997.91015625,121.361083984375,1.0112087726593018,0.4383679628372192,0.2912840247154236,40.4483528137207,238.7423553466797,0.415381520986557,0.2111529558897018,0.2111529558897018,Transport,1.0 +ett1/15T/medium,TTM-R1-Zeroshot,15.956341743469238,15.956341743469238,2.393866777420044,1.4170336723327637,0.7636879682540894,0.5661241412162781,56.68134689331055,3.994539022445679,0.7613630890846252,0.4562734067440033,0.4562733471393585,Energy,7.0 +ett1/H/medium,TTM-R1-Zeroshot,147.88861083984375,147.88861083984375,7.473272800445557,1.3740049600601196,4030999822336.0,0.4816672801971435,54.96020126342773,12.160945892333984,0.5824804902076721,0.3579520583152771,0.3579520285129547,Energy,7.0 +ett2/15T/medium,TTM-R1-Zeroshot,15.00575351715088,15.00575351715088,2.521174669265747,1.052714705467224,0.1634148955345153,0.1984668672084808,42.10858917236328,3.873726129531861,0.2064378708600998,0.1343579590320587,0.1343579739332199,Energy,7.0 +ett2/H/medium,TTM-R1-Zeroshot,305.8819885253906,305.8819885253906,11.393686294555664,1.2107949256896973,0.1966899782419204,0.2058309763669967,48.43179702758789,17.48948097229004,0.2356820106506347,0.15353724360466,0.15353724360466,Energy,7.0 +jena_weather/10T/medium,TTM-R1-Zeroshot,1537.4500732421875,1537.4500732421875,11.29836082458496,0.680471658706665,0.9417414665222168,0.6616309285163879,27.218868255615234,39.210330963134766,0.2406331300735473,0.0693378448486328,0.0693378448486328,Nature,21.0 +jena_weather/H/medium,TTM-R1-Zeroshot,1495.2928466796875,1495.2928466796875,12.530884742736816,0.8401451706886292,3.385291337966919,0.6819665431976318,33.60580825805664,38.66901779174805,0.2366225719451904,0.0766787081956863,0.0766787081956863,Nature,21.0 +bitbrains_fast_storage/5T/medium,TTM-R1-Zeroshot,2465359.25,2465359.25,268.6639404296875,2.242617607116699,3.788220882415772,0.819374680519104,89.70470428466797,1570.146240234375,4.770537853240967,0.8162752389907837,0.8162752389907837,Web/CloudOps,2.0 +bitbrains_rnd/5T/medium,TTM-R1-Zeroshot,2117112.5,2117112.5,198.3486785888672,7.39147138595581,2.527876615524292,0.7744432091712952,295.6588439941406,1455.030029296875,6.122053146362305,0.8345540761947632,0.8345540761947632,Web/CloudOps,2.0 +bizitobs_application/10S/medium,TTM-R1-Zeroshot,52785060.0,52785060.0,4526.90380859375,13.303730010986328,0.1894544214010238,0.1915669292211532,532.149169921875,7265.33251953125,0.2852882444858551,0.1777581870555877,0.1777581870555877,Web/CloudOps,2.0 +bizitobs_service/10S/medium,TTM-R1-Zeroshot,191248.296875,191248.296875,196.9491271972656,7.098899841308594,1.7195463180541992,0.2512445151805877,283.9559936523437,437.3194274902344,0.3291369080543518,0.1482285559177398,0.1482285559177398,Web/CloudOps,2.0 +bizitobs_l2c/5T/medium,TTM-R1-Zeroshot,200.64273071289065,200.64273071289065,10.290497779846191,1.0041199922561646,0.6574810743331909,1.018429160118103,40.164798736572266,14.164841651916504,0.7452789545059204,0.5414315462112427,0.5414315462112427,Web/CloudOps,7.0 +bizitobs_l2c/H/medium,TTM-R1-Zeroshot,281.8178405761719,281.8178405761719,13.066701889038086,1.2794169187545776,1.4472025632858276,1.070118427276611,51.17667770385742,16.787431716918945,1.016504406929016,0.7912086248397827,0.7912086844444275,Web/CloudOps,7.0 +electricity/15T/long,TTM-R1-Zeroshot,1342790.5,1342790.5,123.45643615722656,1.7905689477920532,0.2591364085674286,0.2366401106119156,71.62275695800781,1158.788330078125,1.8281307220458984,0.1947676688432693,0.1947676688432693,Energy,1.0 +electricity/H/long,TTM-R1-Zeroshot,9800309.0,9800309.0,320.5233764648437,1.3680378198623655,0.3523074686527252,0.171502023935318,54.72151565551758,3130.544677734375,1.2346858978271484,0.1264143288135528,0.1264143288135528,Energy,1.0 +solar/10T/long,TTM-R1-Zeroshot,27.684722900390625,27.684722900390625,2.788307905197144,1.218901753425598,2.6426501274108887,1.5232243537902832,48.75607299804688,5.261627197265625,1.137216567993164,0.602648138999939,0.602648138999939,Energy,1.0 +solar/H/long,TTM-R1-Zeroshot,689.2293701171875,689.2293701171875,13.81916046142578,1.0523871183395386,6.962733268737793,1.4221395254135132,42.095481872558594,26.253177642822266,0.9102455377578736,0.4791354835033417,0.4791354835033417,Energy,1.0 +kdd_cup_2018/H/long,TTM-R1-Zeroshot,3769.7626953125,3769.7626953125,23.29082298278809,1.0037591457366943,1.1482057571411133,0.5922478437423706,40.15036392211914,61.39839553833008,1.4406543970108032,0.5464968085289001,0.5464968085289001,Nature,1.0 +loop_seattle/5T/long,TTM-R1-Zeroshot,265.4754333496094,265.4754333496094,12.66295337677002,1.972639441490173,0.3322114944458008,0.2708892822265625,78.90557861328125,16.29341697692871,0.288070946931839,0.2238836139440536,0.2238836139440536,Transport,1.0 +loop_seattle/H/long,TTM-R1-Zeroshot,78.83251953125,78.83251953125,5.85994291305542,1.2037372589111328,0.184073269367218,0.1325975954532623,48.14948654174805,8.878767967224121,0.1586754769086837,0.1047250255942344,0.1047250181436538,Transport,1.0 +sz_taxi/15T/long,TTM-R1-Zeroshot,16.81537437438965,16.81537437438965,2.87433123588562,0.5435872673988342,8260150624256.0,0.4148517251014709,21.74349021911621,4.10065507888794,0.3795493543148041,0.2660429775714874,0.2660429775714874,Transport,1.0 +m_dense/H/long,TTM-R1-Zeroshot,60970.38671875,60970.38671875,126.48969268798828,1.0458335876464844,0.45860356092453,0.2954391837120056,41.833343505859375,246.9218292236328,0.4274581670761108,0.2189723551273346,0.2189723551273346,Transport,1.0 +ett1/15T/long,TTM-R1-Zeroshot,16.58780860900879,16.58780860900879,2.484389305114746,1.488469123840332,0.8089041709899902,0.5957595705986023,59.53876495361328,4.072813510894775,0.7762822508811951,0.4735270738601684,0.4735270738601684,Energy,7.0 +ett1/H/long,TTM-R1-Zeroshot,149.5264892578125,149.5264892578125,7.462743759155273,1.3938250541687012,2593034338304.0,0.4885442852973938,55.75300216674805,12.228102684020996,0.5855773687362671,0.3573746383190155,0.3573746681213379,Energy,7.0 +ett2/15T/long,TTM-R1-Zeroshot,15.252047538757324,15.252047538757324,2.600973129272461,1.0827611684799194,0.167321115732193,0.2018365561962127,43.31044769287109,3.905386924743652,0.2081251442432403,0.1386105716228485,0.1386105567216873,Energy,7.0 +ett2/H/long,TTM-R1-Zeroshot,288.520751953125,288.520751953125,11.162341117858888,1.2191311120986938,0.208345890045166,0.214778259396553,48.76524353027344,16.985898971557617,0.2326088100671768,0.1528596729040146,0.1528596729040146,Energy,7.0 +jena_weather/10T/long,TTM-R1-Zeroshot,1562.930419921875,1562.930419921875,11.357297897338867,0.6737896203994751,0.8498238921165466,0.6583142876625061,26.95158576965332,39.53391647338867,0.2419148087501526,0.069497250020504,0.0694972574710846,Nature,21.0 +jena_weather/H/long,TTM-R1-Zeroshot,1302.5177001953125,1302.5177001953125,14.308568954467772,1.0584625005722046,3.396106243133545,0.6726809144020081,42.3385009765625,36.09041213989258,0.2173042595386505,0.0861534401774406,0.08615343272686,Nature,21.0 +bitbrains_fast_storage/5T/long,TTM-R1-Zeroshot,3354402.25,3354402.25,356.4161071777344,2.0263211727142334,4.350730895996094,0.8388842344284058,81.05284118652344,1831.5028076171875,4.840115070343018,0.9419013857841492,0.9419013261795044,Web/CloudOps,2.0 +bitbrains_rnd/5T/long,TTM-R1-Zeroshot,1939892.375,1939892.375,193.2017059326172,6.739154815673828,2.378561496734619,0.7550832629203796,269.5661926269531,1392.8001708984375,5.335503578186035,0.7401121854782104,0.7401122450828552,Web/CloudOps,2.0 +bizitobs_application/10S/long,TTM-R1-Zeroshot,63951968.0,63951968.0,5090.4970703125,11.375816345214844,0.1536046713590622,0.169856384396553,455.03265380859375,7996.99755859375,0.3084122240543365,0.1963201314210891,0.1963201314210891,Web/CloudOps,2.0 +bizitobs_service/10S/long,TTM-R1-Zeroshot,310560.65625,310560.65625,242.35232543945312,6.615731239318848,0.4960593283176422,0.2466846257448196,264.6292419433594,557.2796630859375,0.4128506183624267,0.1795423477888107,0.1795423477888107,Web/CloudOps,2.0 +bizitobs_l2c/5T/long,TTM-R1-Zeroshot,370.515869140625,370.515869140625,14.165494918823242,1.3303046226501465,0.9395979642868042,1.1797410249710083,53.212188720703125,19.248788833618164,1.0807907581329346,0.7953713536262512,0.7953712940216064,Web/CloudOps,7.0 +bizitobs_l2c/H/long,TTM-R1-Zeroshot,224.97149658203125,224.97149658203125,11.681208610534668,1.1981686353683472,1.655575394630432,0.9281730651855468,47.92674255371094,14.99905014038086,0.9161852598190308,0.7135218977928162,0.7135219573974609,Web/CloudOps,7.0 diff --git a/results/TTM-R1-Zeroshot/config.json b/results/TTM-R1-Zeroshot/config.json new file mode 100644 index 0000000000000000000000000000000000000000..fda1b1c935fab9a886d905d91089f4eddf00dd93 --- /dev/null +++ b/results/TTM-R1-Zeroshot/config.json @@ -0,0 +1,6 @@ +{ + "model": "TTM-R1-Zeroshot", + "model_type": "pretrained", + "model_dtype": "float32", + "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r1" +} \ No newline at end of file diff --git a/results/TTM-R2-Finetuned/config.json b/results/TTM-R2-Finetuned/config.json index 14441ce5860a4c3025cdfc572cc491d3c3e4825b..4faddb89b51b64a924b4dc0dfc722d4c3e564650 100644 --- a/results/TTM-R2-Finetuned/config.json +++ b/results/TTM-R2-Finetuned/config.json @@ -3,8 +3,5 @@ "model_type": "fine-tuned", "model_dtype": "float32", "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r2", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/ttm.ipynb", - "org": "IBM Research", - "testdata_leakage": "Yes", - "replication_code_available": "Yes" -} \ No newline at end of file + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/ttm.ipynb" +} diff --git a/results/TTM-R2-Pretrained/all_results.csv b/results/TTM-R2-Pretrained/all_results.csv deleted file mode 100644 index ce6371340c6e83622ccf1eb797761770eced3a1c..0000000000000000000000000000000000000000 --- a/results/TTM-R2-Pretrained/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,TTM-R2-Pretrained,5041324.0,5041324.0,1227.967529296875,5.130594253540039,0.26075807213783264,0.20997147262096405,205.22377014160156,2245.289306640625,0.360042542219162,0.1969102919101715,0.1969102919101715,Econ/Fin,1 -m4_quarterly/Q/short,TTM-R2-Pretrained,2419099.25,2419099.25,829.3945922851562,2.027404546737671,0.2134242057800293,0.1531578153371811,81.09618377685547,1555.3453369140625,0.2603374123573303,0.13882605731487274,0.13882605731487274,Econ/Fin,1 -m4_monthly/M/short,TTM-R2-Pretrained,2809195.75,2809195.75,853.7319946289062,1.533287525177002,0.2921540439128876,0.20111285150051117,61.331504821777344,1676.0655517578125,0.34834396839141846,0.17743481695652008,0.17743481695652008,Econ/Fin,1 -m4_weekly/W/short,TTM-R2-Pretrained,485578.84375,485578.84375,380.5999450683594,3.483421564102173,0.10566350817680359,0.10109246522188187,139.33685302734375,696.8348999023438,0.12695251405239105,0.06933940947055817,0.06933940947055817,Econ/Fin,1 -m4_daily/D/short,TTM-R2-Pretrained,484801.0625,484801.0625,227.68121337890625,4.401748180389404,0.061012763530015945,0.04191255196928978,176.06993103027344,696.2765502929688,0.10755548626184464,0.035170454531908035,0.035170454531908035,Econ/Fin,1 -m4_hourly/H/short,TTM-R2-Pretrained,2191172.0,2191172.0,294.4319152832031,2.782658100128174,0.1602412611246109,0.13437475264072418,111.30632781982422,1480.2607421875,0.20208826661109924,0.040196456015110016,0.040196459740400314,Econ/Fin,1 -electricity/15T/short,TTM-R2-Pretrained,217371.640625,217371.640625,77.01490783691406,1.4320731163024902,0.2330285906791687,0.22012728452682495,57.282928466796875,466.2313232421875,0.9190338253974915,0.15181156992912292,0.15181156992912292,Energy,1 -electricity/H/short,TTM-R2-Pretrained,1608349.0,1608349.0,205.38311767578125,1.159241795539856,0.32240381836891174,0.1580224633216858,46.36967086791992,1268.20703125,0.5986843109130859,0.09695550054311752,0.09695549309253693,Energy,1 -electricity/D/short,TTM-R2-Pretrained,2721178624.0,2721178624.0,5633.8505859375,1.6594325304031372,0.6992169618606567,0.12481007725000381,66.3772964477539,52164.91796875,0.8572428822517395,0.09258288145065308,0.09258287400007248,Energy,1 -electricity/W/short,TTM-R2-Pretrained,435031867392.0,435031867392.0,69856.296875,2.5212364196777344,0.3161992132663727,0.14358647167682648,100.84944915771484,659569.4375,1.5049197673797607,0.15938901901245117,0.15938901901245117,Energy,1 -solar/10T/short,TTM-R2-Pretrained,25.674766540527344,25.674766540527344,2.6975016593933105,1.1790286302566528,5.4343438148498535,1.5516341924667358,47.16114807128906,5.067027568817139,1.473997712135315,0.7847028970718384,0.7847028970718384,Energy,1 -solar/H/short,TTM-R2-Pretrained,619.8744506835938,619.8744506835938,12.693642616271973,0.9796592593193054,5.864431858062744,1.4270025491714478,39.186370849609375,24.89727783203125,0.9177219867706299,0.4678918719291687,0.4678918719291687,Energy,1 -solar/D/short,TTM-R2-Pretrained,174468.765625,174468.765625,274.03997802734375,1.068386435508728,1.2658942937850952,0.4365691542625427,42.73545837402344,417.694580078125,0.6034263968467712,0.3958944082260132,0.3958944082260132,Energy,1 -solar/W/short,TTM-R2-Pretrained,8813000.0,8813000.0,2602.568359375,2.877580165863037,0.5861544609069824,0.434007465839386,115.10320281982422,2968.669677734375,0.6059797406196594,0.5312492847442627,0.5312492847442627,Energy,1 -hospital/M/short,TTM-R2-Pretrained,12283.0078125,12283.0078125,33.84686279296875,1.050584316253662,0.28865745663642883,0.22228078544139862,42.023372650146484,110.8287353515625,0.4023340344429016,0.12287196516990662,0.12287196516990662,Healthcare,1 -covid_deaths/D/short,TTM-R2-Pretrained,2181495.0,2181495.0,326.9200134277344,53.48585891723633,0.18292130529880524,0.4471057057380676,2139.434326171875,1476.988525390625,0.5557240843772888,0.12300524860620499,0.12300524860620499,Healthcare,1 -us_births/D/short,TTM-R2-Pretrained,1462623.625,1462623.625,1105.77490234375,1.6285947561264038,0.1084594875574112,0.10614823549985886,65.14379119873047,1209.3897705078125,0.11337047815322876,0.10365742444992065,0.10365742444992065,Healthcare,1 -us_births/M/short,TTM-R2-Pretrained,203729680.0,203729680.0,11695.4287109375,1.3240991830825806,0.03641016408801079,0.036412689834833145,52.963966369628906,14273.390625,0.04433305561542511,0.03632592037320137,0.03632592037320137,Healthcare,1 -us_births/W/short,TTM-R2-Pretrained,7337176.0,7337176.0,1957.2852783203125,1.778560757637024,0.026068169623613358,0.026361528784036636,71.1424331665039,2708.72216796875,0.03677063807845116,0.026569955050945282,0.026569955050945282,Healthcare,1 -saugeen/D/short,TTM-R2-Pretrained,1256.2291259765625,1256.2291259765625,18.179595947265625,4.034393787384033,0.7102912664413452,0.54073566198349,161.37574768066406,35.44332504272461,1.1480176448822021,0.588841438293457,0.5888414978981018,Nature,1 -saugeen/M/short,TTM-R2-Pretrained,395.8756408691406,395.8756408691406,13.489892959594727,0.7903389930725098,0.46876707673072815,0.39558157324790955,31.61355972290039,19.896623611450195,0.5973563194274902,0.4050070643424988,0.4050070345401764,Nature,1 -saugeen/W/short,TTM-R2-Pretrained,1452.2073974609375,1452.2073974609375,23.04207420349121,1.867565631866455,0.9696307182312012,0.6474501490592957,74.70262908935547,38.10783767700195,1.150316834449768,0.6955441832542419,0.6955441236495972,Nature,1 -temperature_rain/D/short,TTM-R2-Pretrained,166.6985321044922,166.6985321044922,6.7175984382629395,1.6619073152542114,104.13045501708984,1.4001303911209106,66.4762954711914,12.911178588867188,1.5199871063232422,0.7908389568328857,0.7908389568328857,Nature,1 -kdd_cup_2018/H/short,TTM-R2-Pretrained,4468.35791015625,4468.35791015625,24.576093673706055,1.0117350816726685,1.1454356908798218,0.515670120716095,40.46940231323242,66.84577941894531,1.3992031812667847,0.5144221186637878,0.5144221186637878,Nature,1 -kdd_cup_2018/D/short,TTM-R2-Pretrained,2629.543701171875,2629.543701171875,20.171777725219727,1.172845721244812,0.6205669641494751,0.4447740316390991,46.91382598876953,51.279075622558594,1.1487058401107788,0.45186924934387207,0.4518692195415497,Nature,1 -car_parts/M/short,TTM-R2-Pretrained,2.1810801029205322,2.1810801029205322,0.9530109167098999,1.5749661922454834,0.5873419642448425,1.7117667198181152,62.99864959716797,1.4768480062484741,3.541330099105835,2.2852225303649902,2.2852225303649902,Sales,1 -restaurant/D/short,TTM-R2-Pretrained,202.3482666015625,202.3482666015625,9.526232719421387,0.896973729133606,1.08562433719635,0.4797813296318054,35.87895202636719,14.2249174118042,0.654663622379303,0.43841928243637085,0.43841931223869324,Sales,1 -hierarchical_sales/D/short,TTM-R2-Pretrained,28.789072036743164,28.789072036743164,2.583070755004883,0.8340859413146973,0.8086299300193787,0.9911177754402161,33.36343765258789,5.36554479598999,1.646089792251587,0.7924574613571167,0.7924575209617615,Sales,1 -hierarchical_sales/W/short,TTM-R2-Pretrained,792.2537231445312,792.2537231445312,15.791238784790039,1.086988925933838,1.4281177520751953,0.6353736519813538,43.47955322265625,28.147001266479492,1.2922426462173462,0.7249835133552551,0.7249835133552551,Sales,1 -loop_seattle/5T/short,TTM-R2-Pretrained,45.9228630065918,45.9228630065918,3.994657039642334,0.6305468678474426,0.11244776844978333,0.08397196233272552,25.221874237060547,6.776640892028809,0.11620443314313889,0.06849955022335052,0.06849955022335052,Transport,1 -loop_seattle/H/short,TTM-R2-Pretrained,73.20549011230469,73.20549011230469,5.383979320526123,1.0818450450897217,0.1589573621749878,0.1218806728720665,43.2738037109375,8.556020736694336,0.1514502614736557,0.09530191123485565,0.09530191123485565,Transport,1 -loop_seattle/D/short,TTM-R2-Pretrained,46.15348434448242,46.15348434448242,5.638345241546631,1.7389246225357056,0.10922148823738098,0.10129295289516449,69.5569839477539,6.793635368347168,0.12141135334968567,0.10076476633548737,0.10076476633548737,Transport,1 -sz_taxi/15T/short,TTM-R2-Pretrained,17.630447387695312,17.630447387695312,2.8711466789245605,0.5735141634941101,1263836594176.0,0.41091153025627136,22.940567016601562,4.198862552642822,0.39261162281036377,0.26846447587013245,0.26846450567245483,Transport,1 -sz_taxi/H/short,TTM-R2-Pretrained,7.52171516418457,7.52171516418457,1.9656306505203247,0.5973286032676697,1.2311612367630005,0.31563815474510193,23.893144607543945,2.742574453353882,0.25548774003982544,0.18311062455177307,0.18311062455177307,Transport,1 -m_dense/H/short,TTM-R2-Pretrained,62694.859375,62694.859375,127.04000854492188,1.0879967212677002,0.42501795291900635,0.30154624581336975,43.51987075805664,250.38941955566406,0.4441284239292145,0.22533731162548065,0.22533732652664185,Transport,1 -m_dense/D/short,TTM-R2-Pretrained,18615.037109375,18615.037109375,87.37030792236328,1.20741868019104,0.2061319649219513,0.1770661622285843,48.29674530029297,136.4369354248047,0.2361925095319748,0.1512509286403656,0.1512509435415268,Transport,1 -ett1/15T/short,TTM-R2-Pretrained,5.624592304229736,5.624592304229736,1.2432845830917358,0.8118457794189453,0.4656950831413269,0.2875490188598633,32.47383117675781,2.371622323989868,0.44833534955978394,0.23503252863883972,0.23503251373767853,Energy,7 -ett1/H/short,TTM-R2-Pretrained,104.39521789550781,104.39521789550781,5.366086959838867,0.8821736574172974,0.507716715335846,0.2827994227409363,35.28694534301758,10.217397689819336,0.4768877625465393,0.250457227230072,0.250457227230072,Energy,7 -ett1/D/short,TTM-R2-Pretrained,55083.7578125,55083.7578125,158.2764434814453,1.962099552154541,1.2102364301681519,0.5308582782745361,78.4839859008789,234.6992950439453,0.6164454817771912,0.41571831703186035,0.41571834683418274,Energy,7 -ett1/W/short,TTM-R2-Pretrained,2633276.5,2633276.5,1125.6827392578125,1.5393426418304443,1.0096418857574463,0.45859840512275696,61.573707580566406,1622.7374267578125,0.6457259058952332,0.4479359984397888,0.4479359984397888,Energy,7 -ett2/15T/short,TTM-R2-Pretrained,9.754693031311035,9.754693031311035,1.9594718217849731,0.8321967720985413,0.11617407947778702,0.13838815689086914,33.287872314453125,3.1232504844665527,0.1482718139886856,0.09302309900522232,0.09302308410406113,Energy,7 -ett2/H/short,TTM-R2-Pretrained,124.45040130615234,124.45040130615234,6.933920383453369,0.7901540994644165,0.12842239439487457,0.11721692979335785,31.606163024902344,11.155734062194824,0.14148575067520142,0.08794140070676804,0.08794140070676804,Energy,7 -ett2/D/short,TTM-R2-Pretrained,108511.2109375,108511.2109375,208.45578002929688,1.564162015914917,0.4740043878555298,0.17024555802345276,62.56648254394531,329.410400390625,0.18853026628494263,0.11930473893880844,0.11930473893880844,Energy,7 -ett2/W/short,TTM-R2-Pretrained,12633047.0,12633047.0,2386.069580078125,1.356560468673706,0.2729461193084717,0.21735575795173645,54.262420654296875,3554.2998046875,0.29815125465393066,0.200154647231102,0.20015466213226318,Energy,7 -jena_weather/10T/short,TTM-R2-Pretrained,766.1596069335938,766.1596069335938,7.308579921722412,0.3458213806152344,0.43337202072143555,0.5557089447975159,13.832855224609375,27.679588317871094,0.17192493379116058,0.045395441353321075,0.04539543762803078,Nature,21 -jena_weather/H/short,TTM-R2-Pretrained,1081.8076171875,1081.8076171875,9.729592323303223,0.5753880739212036,1.828482747077942,0.6190334558486938,23.01552391052246,32.890846252441406,0.2016187608242035,0.059641774743795395,0.05964178219437599,Nature,21 -jena_weather/D/short,TTM-R2-Pretrained,1248.5050048828125,1248.5050048828125,20.558490753173828,2.5306453704833984,1.7924801111221313,0.5474997162818909,101.22581481933594,35.334190368652344,0.21276958286762238,0.12379571795463562,0.12379571795463562,Nature,21 -bitbrains_fast_storage/5T/short,TTM-R2-Pretrained,1646846.625,1646846.625,189.82254028320312,0.965898871421814,2.6614601612091064,0.7760849595069885,38.635955810546875,1283.295166015625,4.029229640960693,0.5959958434104919,0.5959958434104919,Web/CloudOps,2 -bitbrains_fast_storage/H/short,TTM-R2-Pretrained,2551485.5,2551485.5,324.970458984375,1.2754968404769897,5.218166351318359,0.635557234287262,51.019874572753906,1597.3369140625,4.552928924560547,0.9262713193893433,0.9262713193893433,Web/CloudOps,2 -bitbrains_rnd/5T/short,TTM-R2-Pretrained,1557151.0,1557151.0,147.8542022705078,2.053879737854004,1.7888224124908447,0.7139753103256226,82.15518951416016,1247.8585205078125,5.105876922607422,0.6049767136573792,0.6049767136573792,Web/CloudOps,2 -bitbrains_rnd/H/short,TTM-R2-Pretrained,1869584.0,1869584.0,242.30137634277344,6.163490295410156,5.455203533172607,0.6784248352050781,246.53961181640625,1367.3272705078125,5.984093189239502,1.060429334640503,1.060429334640503,Web/CloudOps,2 -bizitobs_application/10S/short,TTM-R2-Pretrained,9943514.0,9943514.0,1497.74755859375,4.214677333831787,0.06536141782999039,0.06266415864229202,168.5870819091797,3153.333984375,0.12161148339509964,0.057762134820222855,0.05776213854551315,Web/CloudOps,2 -bizitobs_service/10S/short,TTM-R2-Pretrained,44515.95703125,44515.95703125,71.65859985351562,2.732959747314453,0.35066136717796326,0.13076719641685486,109.31838989257812,210.98805236816406,0.1563066989183426,0.05308698117733002,0.05308698117733002,Web/CloudOps,2 -bizitobs_l2c/5T/short,TTM-R2-Pretrained,24.209394454956055,24.209394454956055,3.1097848415374756,0.32445377111434937,0.17801183462142944,0.22899238765239716,12.978150367736816,4.920304298400879,0.16898316144943237,0.10680259764194489,0.10680260509252548,Web/CloudOps,7 -bizitobs_l2c/H/short,TTM-R2-Pretrained,213.2822723388672,213.2822723388672,10.186047554016113,0.9834961295127869,0.9913681149482727,0.8249644041061401,39.33984375,14.60418701171875,0.7872030735015869,0.5490540862083435,0.5490540862083435,Web/CloudOps,7 -electricity/15T/medium,TTM-R2-Pretrained,479186.78125,479186.78125,82.32447814941406,1.3208063840866089,0.22963841259479523,0.19777949154376984,52.83225631713867,692.233154296875,1.192787528038025,0.14185337722301483,0.14185337722301483,Energy,1 -electricity/H/medium,TTM-R2-Pretrained,6960439.5,6960439.5,280.2503662109375,1.2517058849334717,0.36617472767829895,0.15959197282791138,50.0682373046875,2638.264404296875,1.0298014879226685,0.10939095169305801,0.10939095169305801,Energy,1 -solar/10T/medium,TTM-R2-Pretrained,23.291641235351562,23.291641235351562,2.581629991531372,1.1273674964904785,3.8910577297210693,1.4938726425170898,45.09469985961914,4.826141357421875,1.0709398984909058,0.5728739500045776,0.5728738903999329,Energy,1 -solar/H/medium,TTM-R2-Pretrained,673.9286499023438,673.9286499023438,13.676705360412598,1.0454044342041016,6.807587623596191,1.4293445348739624,41.81617736816406,25.96013641357422,0.9351446628570557,0.4926668405532837,0.4926668703556061,Energy,1 -kdd_cup_2018/H/medium,TTM-R2-Pretrained,4815.142578125,4815.142578125,25.404191970825195,1.0737661123275757,1.1820731163024902,0.559693455696106,42.95064163208008,69.39122772216797,1.452484130859375,0.5317557454109192,0.5317556858062744,Nature,1 -loop_seattle/5T/medium,TTM-R2-Pretrained,116.44446563720703,116.44446563720703,6.789214134216309,1.0654295682907104,0.237552210688591,0.14933940768241882,42.617183685302734,10.790943145751953,0.19199305772781372,0.12079407274723053,0.12079407274723053,Transport,1 -loop_seattle/H/medium,TTM-R2-Pretrained,80.56887817382812,80.56887817382812,5.882454872131348,1.1938070058822632,0.15960149466991425,0.13077092170715332,47.752281188964844,8.976016998291016,0.15865008533000946,0.10397172719240189,0.10397172719240189,Transport,1 -sz_taxi/15T/medium,TTM-R2-Pretrained,17.407360076904297,17.407360076904297,2.9041669368743896,0.5661407113075256,12433696489472.0,0.4201682507991791,22.645627975463867,4.172213077545166,0.3882940411567688,0.27028119564056396,0.27028119564056396,Transport,1 -m_dense/H/medium,TTM-R2-Pretrained,58433.80859375,58433.80859375,122.33402252197266,1.0201516151428223,0.43751060962677,0.29099923372268677,40.80606460571289,241.73086547851562,0.4205811619758606,0.21284574270248413,0.21284572780132294,Transport,1 -ett1/15T/medium,TTM-R2-Pretrained,9.845744132995605,9.845744132995605,1.7487417459487915,1.137412428855896,0.7454333305358887,0.4337901473045349,45.496498107910156,3.1377928256988525,0.5980664491653442,0.33331191539764404,0.33331191539764404,Energy,7 -ett1/H/medium,TTM-R2-Pretrained,138.2218780517578,138.2218780517578,7.0701584815979,1.3201416730880737,3522723315712.0,0.4641319215297699,52.8056640625,11.756780624389648,0.5631219148635864,0.3386438190937042,0.3386438190937042,Energy,7 -ett2/15T/medium,TTM-R2-Pretrained,14.557876586914062,14.557876586914062,2.3989028930664062,0.9867962598800659,0.1484927535057068,0.17977158725261688,39.47185134887695,3.815478563308716,0.2033337652683258,0.12784188985824585,0.12784187495708466,Energy,7 -ett2/H/medium,TTM-R2-Pretrained,269.994140625,269.994140625,10.284690856933594,1.097788691520691,0.17862552404403687,0.1798628270626068,43.91154861450195,16.43149757385254,0.22142501175403595,0.13859282433986664,0.13859283924102783,Energy,7 -jena_weather/10T/medium,TTM-R2-Pretrained,1497.083984375,1497.083984375,11.306266784667969,0.6561654806137085,0.9674649834632874,0.6653227210044861,26.246618270874023,38.692169189453125,0.2374531775712967,0.06938636302947998,0.06938636302947998,Nature,21 -jena_weather/H/medium,TTM-R2-Pretrained,1447.2547607421875,1447.2547607421875,11.986462593078613,0.8147535920143127,3.532595634460449,0.6771256327629089,32.59014129638672,38.04280090332031,0.23279064893722534,0.0733472928404808,0.0733472928404808,Nature,21 -bitbrains_fast_storage/5T/medium,TTM-R2-Pretrained,2803018.75,2803018.75,298.1320495605469,1.2348387241363525,5.41544771194458,0.8186819553375244,49.393550872802734,1674.2218017578125,5.0867486000061035,0.9058074951171875,0.9058074951171875,Web/CloudOps,2 -bitbrains_rnd/5T/medium,TTM-R2-Pretrained,2112217.75,2112217.75,198.39056396484375,4.8130364418029785,2.431964635848999,0.7751559019088745,192.52145385742188,1453.3470458984375,6.114972114562988,0.8347302675247192,0.8347302675247192,Web/CloudOps,2 -bizitobs_application/10S/medium,TTM-R2-Pretrained,34985988.0,34985988.0,3205.0234375,9.018702507019043,0.12491956353187561,0.1292904019355774,360.74810791015625,5914.8955078125,0.2322605401277542,0.1258518397808075,0.1258518248796463,Web/CloudOps,2 -bizitobs_service/10S/medium,TTM-R2-Pretrained,182072.703125,182072.703125,155.6486053466797,5.1217827796936035,0.48198986053466797,0.20569010078907013,204.87130737304688,426.6997985839844,0.32114431262016296,0.11714481562376022,0.11714481562376022,Web/CloudOps,2 -bizitobs_l2c/5T/medium,TTM-R2-Pretrained,195.24891662597656,195.24891662597656,10.268324851989746,0.9921779036521912,0.6358968019485474,1.0157976150512695,39.68711471557617,13.973150253295898,0.7351931929588318,0.540264904499054,0.540264904499054,Web/CloudOps,7 -bizitobs_l2c/H/medium,TTM-R2-Pretrained,289.9211730957031,289.9211730957031,12.913840293884277,1.2396725416183472,1.3855247497558594,1.0475947856903076,49.5869026184082,17.027070999145508,1.0310150384902954,0.7819526791572571,0.7819526791572571,Web/CloudOps,7 -electricity/15T/long,TTM-R2-Pretrained,718197.1875,718197.1875,90.5474853515625,1.3471070528030396,0.2316545844078064,0.19732432067394257,53.88428497314453,847.4651489257812,1.3369802236557007,0.14284975826740265,0.14284975826740265,Energy,1 -electricity/H/long,TTM-R2-Pretrained,10661256.0,10661256.0,325.055908203125,1.3821061849594116,0.39065104722976685,0.1722196787595749,55.28424835205078,3265.157958984375,1.287777304649353,0.12820196151733398,0.1282019466161728,Energy,1 -solar/10T/long,TTM-R2-Pretrained,22.359167098999023,22.359167098999023,2.523888111114502,1.10085928440094,3.154728651046753,1.4919425249099731,44.03437423706055,4.728548049926758,1.0220000743865967,0.5454980731010437,0.5454980134963989,Energy,1 -solar/H/long,TTM-R2-Pretrained,705.2949829101562,705.2949829101562,14.768033027648926,1.1248584985733032,8.415304183959961,1.436413049697876,44.99434280395508,26.557390213012695,0.9207931160926819,0.5120345950126648,0.5120344758033752,Energy,1 -kdd_cup_2018/H/long,TTM-R2-Pretrained,3760.123779296875,3760.123779296875,23.107446670532227,1.0000991821289062,1.153912901878357,0.589838981628418,40.003963470458984,61.319847106933594,1.4388114213943481,0.5421940684318542,0.5421940684318542,Nature,1 -loop_seattle/5T/long,TTM-R2-Pretrained,124.74163818359375,124.74163818359375,7.093927383422852,1.1124976873397827,0.2543075382709503,0.15363603830337524,44.499908447265625,11.168779373168945,0.19746631383895874,0.12542209029197693,0.12542209029197693,Transport,1 -loop_seattle/H/long,TTM-R2-Pretrained,74.29878997802734,74.29878997802734,5.402324199676514,1.114170789718628,0.17675289511680603,0.1240486353635788,44.56683349609375,8.619674682617188,0.15404511988162994,0.09654676914215088,0.09654677659273148,Transport,1 -sz_taxi/15T/long,TTM-R2-Pretrained,16.31110191345215,16.31110191345215,2.8116378784179688,0.5310652256011963,8224755941376.0,0.40968406200408936,21.242610931396484,4.038700580596924,0.3738149106502533,0.26024019718170166,0.26024019718170166,Transport,1 -m_dense/H/long,TTM-R2-Pretrained,62940.5234375,62940.5234375,128.40859985351562,1.0597881078720093,0.470628559589386,0.2987874746322632,42.39152526855469,250.87950134277344,0.4343095123767853,0.22229425609111786,0.22229427099227905,Transport,1 -ett1/15T/long,TTM-R2-Pretrained,10.124308586120605,10.124308586120605,1.8443189859390259,1.1956290006637573,0.7989325523376465,0.45920833945274353,47.82516098022461,3.1818718910217285,0.6064679026603699,0.3515290319919586,0.3515290319919586,Energy,7 -ett1/H/long,TTM-R2-Pretrained,142.861328125,142.861328125,7.135627269744873,1.355435848236084,3003883454464.0,0.4777284562587738,54.217430114746094,11.952461242675781,0.5723775029182434,0.3417097330093384,0.34170976281166077,Energy,7 -ett2/15T/long,TTM-R2-Pretrained,13.01650333404541,13.01650333404541,2.3665223121643066,0.986086905002594,0.14943784475326538,0.1789281815290451,39.443477630615234,3.6078391075134277,0.1922682821750641,0.12611626088619232,0.12611627578735352,Energy,7 -ett2/H/long,TTM-R2-Pretrained,278.0689392089844,278.0689392089844,10.509333610534668,1.131842017173767,0.19035466015338898,0.1890520304441452,45.273681640625,16.675399780273438,0.2283567637205124,0.1439172327518463,0.1439172327518463,Energy,7 -jena_weather/10T/long,TTM-R2-Pretrained,1487.92626953125,1487.92626953125,11.109465599060059,0.6489807367324829,0.8593838214874268,0.6524072885513306,25.959230422973633,38.573646545410156,0.23603875935077667,0.06798072904348373,0.06798072159290314,Nature,21 -jena_weather/H/long,TTM-R2-Pretrained,1266.5963134765625,1266.5963134765625,13.908403396606445,1.0716230869293213,3.8741116523742676,0.6689540147781372,42.864925384521484,35.589271545410156,0.21428684890270233,0.08374399691820145,0.08374399691820145,Nature,21 -bitbrains_fast_storage/5T/long,TTM-R2-Pretrained,3219523.0,3219523.0,355.3838195800781,1.1635642051696777,4.895505428314209,0.8371726274490356,46.54256820678711,1794.302978515625,4.741806983947754,0.9391733407974243,0.9391733407974243,Web/CloudOps,2 -bitbrains_rnd/5T/long,TTM-R2-Pretrained,1991291.25,1991291.25,203.39283752441406,3.782886266708374,2.9021401405334473,0.7515886425971985,151.31544494628906,1411.1312255859375,5.405725955963135,0.7791521549224854,0.7791520953178406,Web/CloudOps,2 -bizitobs_application/10S/long,TTM-R2-Pretrained,41283780.0,41283780.0,3723.089111328125,9.590826034545898,0.13113968074321747,0.13875478506088257,383.633056640625,6425.24560546875,0.24779604375362396,0.14358466863632202,0.14358466863632202,Web/CloudOps,2 -bizitobs_service/10S/long,TTM-R2-Pretrained,267102.25,267102.25,188.78057861328125,5.33817720413208,0.43358832597732544,0.21543104946613312,213.52708435058594,516.8193359375,0.38287630677223206,0.13985468447208405,0.13985469937324524,Web/CloudOps,2 -bizitobs_l2c/5T/long,TTM-R2-Pretrained,358.9239196777344,358.9239196777344,13.98556137084961,1.316333532333374,0.9028239250183105,1.1863360404968262,52.653343200683594,18.945287704467773,1.0637495517730713,0.7852683663368225,0.7852683663368225,Web/CloudOps,7 -bizitobs_l2c/H/long,TTM-R2-Pretrained,248.46641540527344,248.46641540527344,12.290800094604492,1.2374670505523682,1.677781105041504,0.9531240463256836,49.498680114746094,15.7628173828125,0.9628384113311768,0.7507575750350952,0.7507575154304504,Web/CloudOps,7 diff --git a/results/TTM-R2-Pretrained/config.json b/results/TTM-R2-Pretrained/config.json deleted file mode 100644 index 250a90807e4c54e27c4dbbdde58373906c7a4a00..0000000000000000000000000000000000000000 --- a/results/TTM-R2-Pretrained/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "TTM-R2-Pretrained", - "model_type": "pretrained", - "model_dtype": "float32", - "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r2", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/ttm.ipynb", - "org": "IBM Research", - "testdata_leakage": "Yes", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/TTM-R2-Zeroshot/all_results.csv b/results/TTM-R2-Zeroshot/all_results.csv new file mode 100644 index 0000000000000000000000000000000000000000..f2846f953d9ce7932881acff448db1c391c7bc0c --- /dev/null +++ b/results/TTM-R2-Zeroshot/all_results.csv @@ -0,0 +1,98 @@ +dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates +m4_yearly/A/short,TTM-R2-Zeroshot,5041324.0,5041324.0,1227.967529296875,5.130594253540039,0.26075807213783264,0.20997147262096405,205.22377014160156,2245.289306640625,0.360042542219162,0.1969102919101715,0.1969102919101715,Econ/Fin,1 +m4_quarterly/Q/short,TTM-R2-Zeroshot,2419099.25,2419099.25,829.3945922851562,2.027404546737671,0.2134242057800293,0.1531578153371811,81.09618377685547,1555.3453369140625,0.2603374123573303,0.13882605731487274,0.13882605731487274,Econ/Fin,1 +m4_monthly/M/short,TTM-R2-Zeroshot,2809195.75,2809195.75,853.7319946289062,1.533287525177002,0.2921540439128876,0.20111285150051117,61.331504821777344,1676.0655517578125,0.34834396839141846,0.17743481695652008,0.17743481695652008,Econ/Fin,1 +m4_weekly/W/short,TTM-R2-Zeroshot,485578.84375,485578.84375,380.5999450683594,3.483421564102173,0.10566350817680359,0.10109246522188187,139.33685302734375,696.8348999023438,0.12695251405239105,0.06933940947055817,0.06933940947055817,Econ/Fin,1 +m4_daily/D/short,TTM-R2-Zeroshot,484801.0625,484801.0625,227.68121337890625,4.401748180389404,0.061012763530015945,0.04191255196928978,176.06993103027344,696.2765502929688,0.10755548626184464,0.035170454531908035,0.035170454531908035,Econ/Fin,1 +m4_hourly/H/short,TTM-R2-Zeroshot,2191172.0,2191172.0,294.4319152832031,2.782658100128174,0.1602412611246109,0.13437475264072418,111.30632781982422,1480.2607421875,0.20208826661109924,0.040196456015110016,0.040196459740400314,Econ/Fin,1 +electricity/15T/short,TTM-R2-Zeroshot,217371.640625,217371.640625,77.01490783691406,1.4320731163024902,0.2330285906791687,0.22012728452682495,57.282928466796875,466.2313232421875,0.9190338253974915,0.15181156992912292,0.15181156992912292,Energy,1 +electricity/H/short,TTM-R2-Zeroshot,1608349.0,1608349.0,205.38311767578125,1.159241795539856,0.32240381836891174,0.1580224633216858,46.36967086791992,1268.20703125,0.5986843109130859,0.09695550054311752,0.09695549309253693,Energy,1 +electricity/D/short,TTM-R2-Zeroshot,2721178624.0,2721178624.0,5633.8505859375,1.6594325304031372,0.6992169618606567,0.12481007725000381,66.3772964477539,52164.91796875,0.8572428822517395,0.09258288145065308,0.09258287400007248,Energy,1 +electricity/W/short,TTM-R2-Zeroshot,435031867392.0,435031867392.0,69856.296875,2.5212364196777344,0.3161992132663727,0.14358647167682648,100.84944915771484,659569.4375,1.5049197673797607,0.15938901901245117,0.15938901901245117,Energy,1 +solar/10T/short,TTM-R2-Zeroshot,25.674766540527344,25.674766540527344,2.6975016593933105,1.1790286302566528,5.4343438148498535,1.5516341924667358,47.16114807128906,5.067027568817139,1.473997712135315,0.7847028970718384,0.7847028970718384,Energy,1 +solar/H/short,TTM-R2-Zeroshot,619.8744506835938,619.8744506835938,12.693642616271973,0.9796592593193054,5.864431858062744,1.4270025491714478,39.186370849609375,24.89727783203125,0.9177219867706299,0.4678918719291687,0.4678918719291687,Energy,1 +solar/D/short,TTM-R2-Zeroshot,174468.765625,174468.765625,274.03997802734375,1.068386435508728,1.2658942937850952,0.4365691542625427,42.73545837402344,417.694580078125,0.6034263968467712,0.3958944082260132,0.3958944082260132,Energy,1 +solar/W/short,TTM-R2-Zeroshot,8813000.0,8813000.0,2602.568359375,2.877580165863037,0.5861544609069824,0.434007465839386,115.10320281982422,2968.669677734375,0.6059797406196594,0.5312492847442627,0.5312492847442627,Energy,1 +hospital/M/short,TTM-R2-Zeroshot,12283.0078125,12283.0078125,33.84686279296875,1.050584316253662,0.28865745663642883,0.22228078544139862,42.023372650146484,110.8287353515625,0.4023340344429016,0.12287196516990662,0.12287196516990662,Healthcare,1 +covid_deaths/D/short,TTM-R2-Zeroshot,2181495.0,2181495.0,326.9200134277344,53.48585891723633,0.18292130529880524,0.4471057057380676,2139.434326171875,1476.988525390625,0.5557240843772888,0.12300524860620499,0.12300524860620499,Healthcare,1 +us_births/D/short,TTM-R2-Zeroshot,1462623.625,1462623.625,1105.77490234375,1.6285947561264038,0.1084594875574112,0.10614823549985886,65.14379119873047,1209.3897705078125,0.11337047815322876,0.10365742444992065,0.10365742444992065,Healthcare,1 +us_births/M/short,TTM-R2-Zeroshot,203729680.0,203729680.0,11695.4287109375,1.3240991830825806,0.03641016408801079,0.036412689834833145,52.963966369628906,14273.390625,0.04433305561542511,0.03632592037320137,0.03632592037320137,Healthcare,1 +us_births/W/short,TTM-R2-Zeroshot,7337176.0,7337176.0,1957.2852783203125,1.778560757637024,0.026068169623613358,0.026361528784036636,71.1424331665039,2708.72216796875,0.03677063807845116,0.026569955050945282,0.026569955050945282,Healthcare,1 +saugeen/D/short,TTM-R2-Zeroshot,1256.2291259765625,1256.2291259765625,18.179595947265625,4.034393787384033,0.7102912664413452,0.54073566198349,161.37574768066406,35.44332504272461,1.1480176448822021,0.588841438293457,0.5888414978981018,Nature,1 +saugeen/M/short,TTM-R2-Zeroshot,395.8756408691406,395.8756408691406,13.489892959594727,0.7903389930725098,0.46876707673072815,0.39558157324790955,31.61355972290039,19.896623611450195,0.5973563194274902,0.4050070643424988,0.4050070345401764,Nature,1 +saugeen/W/short,TTM-R2-Zeroshot,1452.2073974609375,1452.2073974609375,23.04207420349121,1.867565631866455,0.9696307182312012,0.6474501490592957,74.70262908935547,38.10783767700195,1.150316834449768,0.6955441832542419,0.6955441236495972,Nature,1 +temperature_rain/D/short,TTM-R2-Zeroshot,166.6985321044922,166.6985321044922,6.7175984382629395,1.6619073152542114,104.13045501708984,1.4001303911209106,66.4762954711914,12.911178588867188,1.5199871063232422,0.7908389568328857,0.7908389568328857,Nature,1 +kdd_cup_2018/H/short,TTM-R2-Zeroshot,4468.35791015625,4468.35791015625,24.576093673706055,1.0117350816726685,1.1454356908798218,0.515670120716095,40.46940231323242,66.84577941894531,1.3992031812667847,0.5144221186637878,0.5144221186637878,Nature,1 +kdd_cup_2018/D/short,TTM-R2-Zeroshot,2629.543701171875,2629.543701171875,20.171777725219727,1.172845721244812,0.6205669641494751,0.4447740316390991,46.91382598876953,51.279075622558594,1.1487058401107788,0.45186924934387207,0.4518692195415497,Nature,1 +car_parts/M/short,TTM-R2-Zeroshot,2.1810801029205322,2.1810801029205322,0.9530109167098999,1.5749661922454834,0.5873419642448425,1.7117667198181152,62.99864959716797,1.4768480062484741,3.541330099105835,2.2852225303649902,2.2852225303649902,Sales,1 +restaurant/D/short,TTM-R2-Zeroshot,202.3482666015625,202.3482666015625,9.526232719421387,0.896973729133606,1.08562433719635,0.4797813296318054,35.87895202636719,14.2249174118042,0.654663622379303,0.43841928243637085,0.43841931223869324,Sales,1 +hierarchical_sales/D/short,TTM-R2-Zeroshot,28.789072036743164,28.789072036743164,2.583070755004883,0.8340859413146973,0.8086299300193787,0.9911177754402161,33.36343765258789,5.36554479598999,1.646089792251587,0.7924574613571167,0.7924575209617615,Sales,1 +hierarchical_sales/W/short,TTM-R2-Zeroshot,792.2537231445312,792.2537231445312,15.791238784790039,1.086988925933838,1.4281177520751953,0.6353736519813538,43.47955322265625,28.147001266479492,1.2922426462173462,0.7249835133552551,0.7249835133552551,Sales,1 +loop_seattle/5T/short,TTM-R2-Zeroshot,45.9228630065918,45.9228630065918,3.994657039642334,0.6305468678474426,0.11244776844978333,0.08397196233272552,25.221874237060547,6.776640892028809,0.11620443314313889,0.06849955022335052,0.06849955022335052,Transport,1 +loop_seattle/H/short,TTM-R2-Zeroshot,73.20549011230469,73.20549011230469,5.383979320526123,1.0818450450897217,0.1589573621749878,0.1218806728720665,43.2738037109375,8.556020736694336,0.1514502614736557,0.09530191123485565,0.09530191123485565,Transport,1 +loop_seattle/D/short,TTM-R2-Zeroshot,46.15348434448242,46.15348434448242,5.638345241546631,1.7389246225357056,0.10922148823738098,0.10129295289516449,69.5569839477539,6.793635368347168,0.12141135334968567,0.10076476633548737,0.10076476633548737,Transport,1 +sz_taxi/15T/short,TTM-R2-Zeroshot,17.630447387695312,17.630447387695312,2.8711466789245605,0.5735141634941101,1263836594176.0,0.41091153025627136,22.940567016601562,4.198862552642822,0.39261162281036377,0.26846447587013245,0.26846450567245483,Transport,1 +sz_taxi/H/short,TTM-R2-Zeroshot,7.52171516418457,7.52171516418457,1.9656306505203247,0.5973286032676697,1.2311612367630005,0.31563815474510193,23.893144607543945,2.742574453353882,0.25548774003982544,0.18311062455177307,0.18311062455177307,Transport,1 +m_dense/H/short,TTM-R2-Zeroshot,62694.859375,62694.859375,127.04000854492188,1.0879967212677002,0.42501795291900635,0.30154624581336975,43.51987075805664,250.38941955566406,0.4441284239292145,0.22533731162548065,0.22533732652664185,Transport,1 +m_dense/D/short,TTM-R2-Zeroshot,18615.037109375,18615.037109375,87.37030792236328,1.20741868019104,0.2061319649219513,0.1770661622285843,48.29674530029297,136.4369354248047,0.2361925095319748,0.1512509286403656,0.1512509435415268,Transport,1 +ett1/15T/short,TTM-R2-Zeroshot,5.624592304229736,5.624592304229736,1.2432845830917358,0.8118457794189453,0.4656950831413269,0.2875490188598633,32.47383117675781,2.371622323989868,0.44833534955978394,0.23503252863883972,0.23503251373767853,Energy,7 +ett1/H/short,TTM-R2-Zeroshot,104.39521789550781,104.39521789550781,5.366086959838867,0.8821736574172974,0.507716715335846,0.2827994227409363,35.28694534301758,10.217397689819336,0.4768877625465393,0.250457227230072,0.250457227230072,Energy,7 +ett1/D/short,TTM-R2-Zeroshot,55083.7578125,55083.7578125,158.2764434814453,1.962099552154541,1.2102364301681519,0.5308582782745361,78.4839859008789,234.6992950439453,0.6164454817771912,0.41571831703186035,0.41571834683418274,Energy,7 +ett1/W/short,TTM-R2-Zeroshot,2633276.5,2633276.5,1125.6827392578125,1.5393426418304443,1.0096418857574463,0.45859840512275696,61.573707580566406,1622.7374267578125,0.6457259058952332,0.4479359984397888,0.4479359984397888,Energy,7 +ett2/15T/short,TTM-R2-Zeroshot,9.754693031311035,9.754693031311035,1.9594718217849731,0.8321967720985413,0.11617407947778702,0.13838815689086914,33.287872314453125,3.1232504844665527,0.1482718139886856,0.09302309900522232,0.09302308410406113,Energy,7 +ett2/H/short,TTM-R2-Zeroshot,124.45040130615234,124.45040130615234,6.933920383453369,0.7901540994644165,0.12842239439487457,0.11721692979335785,31.606163024902344,11.155734062194824,0.14148575067520142,0.08794140070676804,0.08794140070676804,Energy,7 +ett2/D/short,TTM-R2-Zeroshot,108511.2109375,108511.2109375,208.45578002929688,1.564162015914917,0.4740043878555298,0.17024555802345276,62.56648254394531,329.410400390625,0.18853026628494263,0.11930473893880844,0.11930473893880844,Energy,7 +ett2/W/short,TTM-R2-Zeroshot,12633047.0,12633047.0,2386.069580078125,1.356560468673706,0.2729461193084717,0.21735575795173645,54.262420654296875,3554.2998046875,0.29815125465393066,0.200154647231102,0.20015466213226318,Energy,7 +jena_weather/10T/short,TTM-R2-Zeroshot,766.1596069335938,766.1596069335938,7.308579921722412,0.3458213806152344,0.43337202072143555,0.5557089447975159,13.832855224609375,27.679588317871094,0.17192493379116058,0.045395441353321075,0.04539543762803078,Nature,21 +jena_weather/H/short,TTM-R2-Zeroshot,1081.8076171875,1081.8076171875,9.729592323303223,0.5753880739212036,1.828482747077942,0.6190334558486938,23.01552391052246,32.890846252441406,0.2016187608242035,0.059641774743795395,0.05964178219437599,Nature,21 +jena_weather/D/short,TTM-R2-Zeroshot,1248.5050048828125,1248.5050048828125,20.558490753173828,2.5306453704833984,1.7924801111221313,0.5474997162818909,101.22581481933594,35.334190368652344,0.21276958286762238,0.12379571795463562,0.12379571795463562,Nature,21 +bitbrains_fast_storage/5T/short,TTM-R2-Zeroshot,1646846.625,1646846.625,189.82254028320312,0.965898871421814,2.6614601612091064,0.7760849595069885,38.635955810546875,1283.295166015625,4.029229640960693,0.5959958434104919,0.5959958434104919,Web/CloudOps,2 +bitbrains_fast_storage/H/short,TTM-R2-Zeroshot,2551485.5,2551485.5,324.970458984375,1.2754968404769897,5.218166351318359,0.635557234287262,51.019874572753906,1597.3369140625,4.552928924560547,0.9262713193893433,0.9262713193893433,Web/CloudOps,2 +bitbrains_rnd/5T/short,TTM-R2-Zeroshot,1557151.0,1557151.0,147.8542022705078,2.053879737854004,1.7888224124908447,0.7139753103256226,82.15518951416016,1247.8585205078125,5.105876922607422,0.6049767136573792,0.6049767136573792,Web/CloudOps,2 +bitbrains_rnd/H/short,TTM-R2-Zeroshot,1869584.0,1869584.0,242.30137634277344,6.163490295410156,5.455203533172607,0.6784248352050781,246.53961181640625,1367.3272705078125,5.984093189239502,1.060429334640503,1.060429334640503,Web/CloudOps,2 +bizitobs_application/10S/short,TTM-R2-Zeroshot,9943514.0,9943514.0,1497.74755859375,4.214677333831787,0.06536141782999039,0.06266415864229202,168.5870819091797,3153.333984375,0.12161148339509964,0.057762134820222855,0.05776213854551315,Web/CloudOps,2 +bizitobs_service/10S/short,TTM-R2-Zeroshot,44515.95703125,44515.95703125,71.65859985351562,2.732959747314453,0.35066136717796326,0.13076719641685486,109.31838989257812,210.98805236816406,0.1563066989183426,0.05308698117733002,0.05308698117733002,Web/CloudOps,2 +bizitobs_l2c/5T/short,TTM-R2-Zeroshot,24.209394454956055,24.209394454956055,3.1097848415374756,0.32445377111434937,0.17801183462142944,0.22899238765239716,12.978150367736816,4.920304298400879,0.16898316144943237,0.10680259764194489,0.10680260509252548,Web/CloudOps,7 +bizitobs_l2c/H/short,TTM-R2-Zeroshot,213.2822723388672,213.2822723388672,10.186047554016113,0.9834961295127869,0.9913681149482727,0.8249644041061401,39.33984375,14.60418701171875,0.7872030735015869,0.5490540862083435,0.5490540862083435,Web/CloudOps,7 +electricity/15T/medium,TTM-R2-Zeroshot,479186.78125,479186.78125,82.32447814941406,1.3208063840866089,0.22963841259479523,0.19777949154376984,52.83225631713867,692.233154296875,1.192787528038025,0.14185337722301483,0.14185337722301483,Energy,1 +electricity/H/medium,TTM-R2-Zeroshot,6960439.5,6960439.5,280.2503662109375,1.2517058849334717,0.36617472767829895,0.15959197282791138,50.0682373046875,2638.264404296875,1.0298014879226685,0.10939095169305801,0.10939095169305801,Energy,1 +solar/10T/medium,TTM-R2-Zeroshot,23.291641235351562,23.291641235351562,2.581629991531372,1.1273674964904785,3.8910577297210693,1.4938726425170898,45.09469985961914,4.826141357421875,1.0709398984909058,0.5728739500045776,0.5728738903999329,Energy,1 +solar/H/medium,TTM-R2-Zeroshot,673.9286499023438,673.9286499023438,13.676705360412598,1.0454044342041016,6.807587623596191,1.4293445348739624,41.81617736816406,25.96013641357422,0.9351446628570557,0.4926668405532837,0.4926668703556061,Energy,1 +kdd_cup_2018/H/medium,TTM-R2-Zeroshot,4815.142578125,4815.142578125,25.404191970825195,1.0737661123275757,1.1820731163024902,0.559693455696106,42.95064163208008,69.39122772216797,1.452484130859375,0.5317557454109192,0.5317556858062744,Nature,1 +loop_seattle/5T/medium,TTM-R2-Zeroshot,116.44446563720703,116.44446563720703,6.789214134216309,1.0654295682907104,0.237552210688591,0.14933940768241882,42.617183685302734,10.790943145751953,0.19199305772781372,0.12079407274723053,0.12079407274723053,Transport,1 +loop_seattle/H/medium,TTM-R2-Zeroshot,80.56887817382812,80.56887817382812,5.882454872131348,1.1938070058822632,0.15960149466991425,0.13077092170715332,47.752281188964844,8.976016998291016,0.15865008533000946,0.10397172719240189,0.10397172719240189,Transport,1 +sz_taxi/15T/medium,TTM-R2-Zeroshot,17.407360076904297,17.407360076904297,2.9041669368743896,0.5661407113075256,12433696489472.0,0.4201682507991791,22.645627975463867,4.172213077545166,0.3882940411567688,0.27028119564056396,0.27028119564056396,Transport,1 +m_dense/H/medium,TTM-R2-Zeroshot,58433.80859375,58433.80859375,122.33402252197266,1.0201516151428223,0.43751060962677,0.29099923372268677,40.80606460571289,241.73086547851562,0.4205811619758606,0.21284574270248413,0.21284572780132294,Transport,1 +ett1/15T/medium,TTM-R2-Zeroshot,9.845744132995605,9.845744132995605,1.7487417459487915,1.137412428855896,0.7454333305358887,0.4337901473045349,45.496498107910156,3.1377928256988525,0.5980664491653442,0.33331191539764404,0.33331191539764404,Energy,7 +ett1/H/medium,TTM-R2-Zeroshot,138.2218780517578,138.2218780517578,7.0701584815979,1.3201416730880737,3522723315712.0,0.4641319215297699,52.8056640625,11.756780624389648,0.5631219148635864,0.3386438190937042,0.3386438190937042,Energy,7 +ett2/15T/medium,TTM-R2-Zeroshot,14.557876586914062,14.557876586914062,2.3989028930664062,0.9867962598800659,0.1484927535057068,0.17977158725261688,39.47185134887695,3.815478563308716,0.2033337652683258,0.12784188985824585,0.12784187495708466,Energy,7 +ett2/H/medium,TTM-R2-Zeroshot,269.994140625,269.994140625,10.284690856933594,1.097788691520691,0.17862552404403687,0.1798628270626068,43.91154861450195,16.43149757385254,0.22142501175403595,0.13859282433986664,0.13859283924102783,Energy,7 +jena_weather/10T/medium,TTM-R2-Zeroshot,1497.083984375,1497.083984375,11.306266784667969,0.6561654806137085,0.9674649834632874,0.6653227210044861,26.246618270874023,38.692169189453125,0.2374531775712967,0.06938636302947998,0.06938636302947998,Nature,21 +jena_weather/H/medium,TTM-R2-Zeroshot,1447.2547607421875,1447.2547607421875,11.986462593078613,0.8147535920143127,3.532595634460449,0.6771256327629089,32.59014129638672,38.04280090332031,0.23279064893722534,0.0733472928404808,0.0733472928404808,Nature,21 +bitbrains_fast_storage/5T/medium,TTM-R2-Zeroshot,2803018.75,2803018.75,298.1320495605469,1.2348387241363525,5.41544771194458,0.8186819553375244,49.393550872802734,1674.2218017578125,5.0867486000061035,0.9058074951171875,0.9058074951171875,Web/CloudOps,2 +bitbrains_rnd/5T/medium,TTM-R2-Zeroshot,2112217.75,2112217.75,198.39056396484375,4.8130364418029785,2.431964635848999,0.7751559019088745,192.52145385742188,1453.3470458984375,6.114972114562988,0.8347302675247192,0.8347302675247192,Web/CloudOps,2 +bizitobs_application/10S/medium,TTM-R2-Zeroshot,34985988.0,34985988.0,3205.0234375,9.018702507019043,0.12491956353187561,0.1292904019355774,360.74810791015625,5914.8955078125,0.2322605401277542,0.1258518397808075,0.1258518248796463,Web/CloudOps,2 +bizitobs_service/10S/medium,TTM-R2-Zeroshot,182072.703125,182072.703125,155.6486053466797,5.1217827796936035,0.48198986053466797,0.20569010078907013,204.87130737304688,426.6997985839844,0.32114431262016296,0.11714481562376022,0.11714481562376022,Web/CloudOps,2 +bizitobs_l2c/5T/medium,TTM-R2-Zeroshot,195.24891662597656,195.24891662597656,10.268324851989746,0.9921779036521912,0.6358968019485474,1.0157976150512695,39.68711471557617,13.973150253295898,0.7351931929588318,0.540264904499054,0.540264904499054,Web/CloudOps,7 +bizitobs_l2c/H/medium,TTM-R2-Zeroshot,289.9211730957031,289.9211730957031,12.913840293884277,1.2396725416183472,1.3855247497558594,1.0475947856903076,49.5869026184082,17.027070999145508,1.0310150384902954,0.7819526791572571,0.7819526791572571,Web/CloudOps,7 +electricity/15T/long,TTM-R2-Zeroshot,718197.1875,718197.1875,90.5474853515625,1.3471070528030396,0.2316545844078064,0.19732432067394257,53.88428497314453,847.4651489257812,1.3369802236557007,0.14284975826740265,0.14284975826740265,Energy,1 +electricity/H/long,TTM-R2-Zeroshot,10661256.0,10661256.0,325.055908203125,1.3821061849594116,0.39065104722976685,0.1722196787595749,55.28424835205078,3265.157958984375,1.287777304649353,0.12820196151733398,0.1282019466161728,Energy,1 +solar/10T/long,TTM-R2-Zeroshot,22.359167098999023,22.359167098999023,2.523888111114502,1.10085928440094,3.154728651046753,1.4919425249099731,44.03437423706055,4.728548049926758,1.0220000743865967,0.5454980731010437,0.5454980134963989,Energy,1 +solar/H/long,TTM-R2-Zeroshot,705.2949829101562,705.2949829101562,14.768033027648926,1.1248584985733032,8.415304183959961,1.436413049697876,44.99434280395508,26.557390213012695,0.9207931160926819,0.5120345950126648,0.5120344758033752,Energy,1 +kdd_cup_2018/H/long,TTM-R2-Zeroshot,3760.123779296875,3760.123779296875,23.107446670532227,1.0000991821289062,1.153912901878357,0.589838981628418,40.003963470458984,61.319847106933594,1.4388114213943481,0.5421940684318542,0.5421940684318542,Nature,1 +loop_seattle/5T/long,TTM-R2-Zeroshot,124.74163818359375,124.74163818359375,7.093927383422852,1.1124976873397827,0.2543075382709503,0.15363603830337524,44.499908447265625,11.168779373168945,0.19746631383895874,0.12542209029197693,0.12542209029197693,Transport,1 +loop_seattle/H/long,TTM-R2-Zeroshot,74.29878997802734,74.29878997802734,5.402324199676514,1.114170789718628,0.17675289511680603,0.1240486353635788,44.56683349609375,8.619674682617188,0.15404511988162994,0.09654676914215088,0.09654677659273148,Transport,1 +sz_taxi/15T/long,TTM-R2-Zeroshot,16.31110191345215,16.31110191345215,2.8116378784179688,0.5310652256011963,8224755941376.0,0.40968406200408936,21.242610931396484,4.038700580596924,0.3738149106502533,0.26024019718170166,0.26024019718170166,Transport,1 +m_dense/H/long,TTM-R2-Zeroshot,62940.5234375,62940.5234375,128.40859985351562,1.0597881078720093,0.470628559589386,0.2987874746322632,42.39152526855469,250.87950134277344,0.4343095123767853,0.22229425609111786,0.22229427099227905,Transport,1 +ett1/15T/long,TTM-R2-Zeroshot,10.124308586120605,10.124308586120605,1.8443189859390259,1.1956290006637573,0.7989325523376465,0.45920833945274353,47.82516098022461,3.1818718910217285,0.6064679026603699,0.3515290319919586,0.3515290319919586,Energy,7 +ett1/H/long,TTM-R2-Zeroshot,142.861328125,142.861328125,7.135627269744873,1.355435848236084,3003883454464.0,0.4777284562587738,54.217430114746094,11.952461242675781,0.5723775029182434,0.3417097330093384,0.34170976281166077,Energy,7 +ett2/15T/long,TTM-R2-Zeroshot,13.01650333404541,13.01650333404541,2.3665223121643066,0.986086905002594,0.14943784475326538,0.1789281815290451,39.443477630615234,3.6078391075134277,0.1922682821750641,0.12611626088619232,0.12611627578735352,Energy,7 +ett2/H/long,TTM-R2-Zeroshot,278.0689392089844,278.0689392089844,10.509333610534668,1.131842017173767,0.19035466015338898,0.1890520304441452,45.273681640625,16.675399780273438,0.2283567637205124,0.1439172327518463,0.1439172327518463,Energy,7 +jena_weather/10T/long,TTM-R2-Zeroshot,1487.92626953125,1487.92626953125,11.109465599060059,0.6489807367324829,0.8593838214874268,0.6524072885513306,25.959230422973633,38.573646545410156,0.23603875935077667,0.06798072904348373,0.06798072159290314,Nature,21 +jena_weather/H/long,TTM-R2-Zeroshot,1266.5963134765625,1266.5963134765625,13.908403396606445,1.0716230869293213,3.8741116523742676,0.6689540147781372,42.864925384521484,35.589271545410156,0.21428684890270233,0.08374399691820145,0.08374399691820145,Nature,21 +bitbrains_fast_storage/5T/long,TTM-R2-Zeroshot,3219523.0,3219523.0,355.3838195800781,1.1635642051696777,4.895505428314209,0.8371726274490356,46.54256820678711,1794.302978515625,4.741806983947754,0.9391733407974243,0.9391733407974243,Web/CloudOps,2 +bitbrains_rnd/5T/long,TTM-R2-Zeroshot,1991291.25,1991291.25,203.39283752441406,3.782886266708374,2.9021401405334473,0.7515886425971985,151.31544494628906,1411.1312255859375,5.405725955963135,0.7791521549224854,0.7791520953178406,Web/CloudOps,2 +bizitobs_application/10S/long,TTM-R2-Zeroshot,41283780.0,41283780.0,3723.089111328125,9.590826034545898,0.13113968074321747,0.13875478506088257,383.633056640625,6425.24560546875,0.24779604375362396,0.14358466863632202,0.14358466863632202,Web/CloudOps,2 +bizitobs_service/10S/long,TTM-R2-Zeroshot,267102.25,267102.25,188.78057861328125,5.33817720413208,0.43358832597732544,0.21543104946613312,213.52708435058594,516.8193359375,0.38287630677223206,0.13985468447208405,0.13985469937324524,Web/CloudOps,2 +bizitobs_l2c/5T/long,TTM-R2-Zeroshot,358.9239196777344,358.9239196777344,13.98556137084961,1.316333532333374,0.9028239250183105,1.1863360404968262,52.653343200683594,18.945287704467773,1.0637495517730713,0.7852683663368225,0.7852683663368225,Web/CloudOps,7 +bizitobs_l2c/H/long,TTM-R2-Zeroshot,248.46641540527344,248.46641540527344,12.290800094604492,1.2374670505523682,1.677781105041504,0.9531240463256836,49.498680114746094,15.7628173828125,0.9628384113311768,0.7507575750350952,0.7507575154304504,Web/CloudOps,7 diff --git a/results/TTM-R2-Zeroshot/config.json b/results/TTM-R2-Zeroshot/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b7223b7139389853ddb6d52225051d3655915940 --- /dev/null +++ b/results/TTM-R2-Zeroshot/config.json @@ -0,0 +1,6 @@ +{ + "model": "TTM-R2-Zeroshot", + "model_type": "pretrained", + "model_dtype": "float32", + "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r2" +} \ No newline at end of file diff --git a/results/TempoPFN/all_results.csv b/results/TempoPFN/all_results.csv deleted file mode 100644 index 2d8615d84c53e8e573d1ab47867926d46fc9e99e..0000000000000000000000000000000000000000 --- a/results/TempoPFN/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/5T/long,TempoPFN,3618256.9829756585,3618256.9829756585,338.4312167667779,0.9595844510578648,4.588266457410265,0.8399897172520914,19.241605706495108,1902.1716491882796,5.026871684094993,0.8943726510195391,0.6678268449333727,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,TempoPFN,3490642.119405824,3490642.119405824,301.8811999515851,1.087872593968248,4.769953463921024,0.8210839637401958,26.30283433825968,1868.3260206414252,5.676490438562655,0.9171984579643542,0.6614682834018946,Web/CloudOps,2 -bitbrains_fast_storage/5T/short,TempoPFN,1736889.3352509716,1736889.3352509716,160.0157453841179,0.801255387878816,1.9218905715057013,0.7575231846392683,20.131648115903275,1317.9109739473952,4.137914492539307,0.502409862994168,0.4044679618361981,Web/CloudOps,2 -bitbrains_fast_storage/H/short,TempoPFN,3363884.832010599,3363884.832010599,329.19714123375235,1.1776469011711326,4.457941706914934,0.5853265142362003,29.800183081711054,1834.0896466668685,5.227751330329551,0.9383187981857776,0.6703564200719261,Web/CloudOps,2 -bitbrains_rnd/5T/long,TempoPFN,2493635.7704445343,2493635.7704445343,184.85815909796736,3.49402765035714,1.8313951411684228,0.7196198477764139,123.6137271904707,1579.125001526014,6.049272265365491,0.7081499778530569,0.6656210632794134,Web/CloudOps,2 -bitbrains_rnd/5T/medium,TempoPFN,2275676.336229122,2275676.336229122,151.01203167676363,4.531806050958238,0.5467116146354555,0.7299014896040513,167.39248662886962,1508.5344995157127,6.347173534687507,0.6353845876152348,0.6279025492873883,Web/CloudOps,2 -bitbrains_rnd/5T/short,TempoPFN,2011734.3695229236,2011734.3695229236,134.30605096927297,1.832568262660189,0.6586633552117653,0.656137064195758,62.93383434592541,1418.356220955414,5.803504227963083,0.5495415912627092,0.5202928683440972,Web/CloudOps,2 -bitbrains_rnd/H/short,TempoPFN,1833864.2438088744,1833864.2438088744,153.2905187898267,6.01363937647747,1.5291568048300928,0.5582824469142056,205.55868673901568,1354.2024382672166,5.926652438420676,0.6708743104428145,0.618212914212475,Web/CloudOps,2 -bizitobs_application/10S/long,TempoPFN,5657311.004444445,5657311.004444445,1429.5381944444443,3.9935939909083578,0.0552200571695963,0.0560428445956551,41.407884466219066,2378.5102489677115,0.0917296331255114,0.0551315741322563,0.0479135525414879,Web/CloudOps,2 -bizitobs_application/10S/medium,TempoPFN,2650328.1066666665,2650328.1066666665,935.765625,2.364984377318028,0.0336558278401692,0.0338731847769077,15.434342587999888,1627.982833652329,0.0639260944377557,0.0367447619709549,0.0270434065068804,Web/CloudOps,2 -bizitobs_application/10S/short,TempoPFN,293439.68,293439.68,295.42597222222224,0.9466275803814688,0.0135210143195258,0.0134548451269552,8.580196796530167,541.7007291854055,0.0208912318849855,0.0113933988972522,0.00953574516053,Web/CloudOps,2 -bizitobs_l2c/5T/long,TempoPFN,367.06944444444446,367.06944444444446,13.761483134920637,1.3155973821539892,0.9571163607658632,1.2181312289631128,7.885543197729365,19.159056460182075,1.0757523773983624,0.7726867045715494,0.5896921543935266,Web/CloudOps,7 -bizitobs_l2c/5T/medium,TempoPFN,163.58042091836734,163.58042091836734,8.885337611607143,0.8610525859698702,0.6099801922560945,0.9228806695040844,6.307505529874752,12.789856172700588,0.6729345176643731,0.4674994229187991,0.3627533422032696,Web/CloudOps,7 -bizitobs_l2c/5T/short,TempoPFN,20.047846330915178,20.047846330915178,2.600427972702753,0.2686845079651391,0.1518286962104781,0.2076780035227841,2.5235380675937336,4.477482141886797,0.1537748540925176,0.0893092187547975,0.0720421741602355,Web/CloudOps,7 -bizitobs_l2c/H/long,TempoPFN,98.5917472718254,98.5917472718254,6.016043526785714,0.6413105932372005,0.6509262616598414,0.8261580792115836,4.751776965922631,9.929337705598767,0.6065126048913725,0.3674773019869454,0.2938886036823789,Web/CloudOps,7 -bizitobs_l2c/H/medium,TempoPFN,82.46735491071429,82.46735491071429,5.365966796875,0.54873251131501,0.4955350006673371,0.8101830879498161,3.850019847551049,9.081153831464055,0.5498776579757226,0.3249174399863038,0.2576779421135526,Web/CloudOps,7 -bizitobs_l2c/H/short,TempoPFN,71.66493055555556,71.66493055555556,5.0741470579117065,0.4983927757107515,0.4166772417027315,0.6602853650606105,2.8558255329955027,8.465514193216828,0.4563128955709352,0.2735095215366474,0.2101358130746266,Web/CloudOps,7 -bizitobs_service/10S/long,TempoPFN,139263.86455026455,139263.86455026455,86.99005291005291,1.9752761964322127,0.1301014669105489,0.1029230492969231,25.03013689093193,373.1807397900709,0.2764642223248632,0.0644450121978571,0.0570826431375106,Web/CloudOps,2 -bizitobs_service/10S/medium,TempoPFN,19165.80317460317,19165.80317460317,39.86685763888889,1.4935984152782629,0.0765924653552827,0.0708193369150006,22.81562139347309,138.44061244664866,0.1041936672955716,0.0300047365259027,0.0239326877791516,Web/CloudOps,2 -bizitobs_service/10S/short,TempoPFN,2715.061951058201,2715.061951058201,19.888859333664023,1.1616366321025038,0.0593373701307508,0.0525101472354606,24.95355863856096,52.1062563523633,0.0386019821559286,0.0147343034569216,0.0120944994148936,Web/CloudOps,2 -car_parts/M/short,TempoPFN,1.380084632385992,1.380084632385992,0.4645809090215085,0.8366520149396539,0.8199656181552258,1.8907443630253995,26.291321420640703,1.174770033830448,2.816978032698879,1.114017362919686,1.0292306216598837,Sales,1 -covid_deaths/D/short,TempoPFN,370936.4616541353,370936.4616541353,94.56139567669172,36.04520009694931,0.0940949679211966,0.3625045082139037,843.8589477254002,609.0455333176127,0.2291563350625127,0.0355791836344794,0.0296139962990822,Healthcare,1 -electricity/15T/long,TempoPFN,515609.28629851725,515609.28629851725,79.64843210984422,1.216131395655562,0.1857030765674225,0.1813470064237671,9.465589736859284,718.0593891166087,1.1328267007922177,0.1256551643747329,0.0994305516316144,Energy,1 -electricity/15T/medium,TempoPFN,343459.75924866274,343459.75924866274,67.58744384765625,1.067375896117608,0.1677188596322956,0.1682547257327376,8.490327113278452,586.0543995642919,1.0098307304867264,0.1164599700013375,0.0925882940722339,Energy,1 -electricity/15T/short,TempoPFN,171757.78046281144,171757.78046281144,62.98562438414977,1.1931012587642071,0.180947034795355,0.1919673872999833,12.877546976003789,414.4367026010262,0.8169364520422242,0.1241570840398924,0.1034183194974842,Energy,1 -electricity/D/short,TempoPFN,1744382968.951928,1744382968.951928,4712.234637387387,1.5059817070474473,0.3781292603528917,0.0993942963334438,12.209207841718015,41765.81100555726,0.6863510119895466,0.0774376681365695,0.0605576668549789,Energy,1 -electricity/H/long,TempoPFN,12085989.068084083,12085989.068084083,328.1874481981982,1.351688949542974,0.3323946402737576,0.1600357384061194,10.881146001754864,3476.490912987417,1.3711270438535197,0.129437037789103,0.1018763160058587,Energy,1 -electricity/H/medium,TempoPFN,7633591.743260135,7633591.743260135,279.1331670537725,1.1869987828552766,0.2284607838742088,0.1445345259241482,9.684205926922072,2762.895536074452,1.0784491704274983,0.1089548730733587,0.086722229134305,Energy,1 -electricity/H/short,TempoPFN,2208858.590561656,2208858.590561656,202.0431417159347,1.080026929488082,0.1953288202749636,0.1440642829713808,10.826930992269196,1486.2229276126968,0.7016033646927585,0.0953787923785388,0.0775416499233381,Energy,1 -electricity/W/short,TempoPFN,76037392012.22342,76037392012.22342,34668.69622747748,1.6493742919995291,0.1544844579847553,0.1038489037121781,12.494169358924385,275748.78424432524,0.6291677495879527,0.0791025267667675,0.0615897744737159,Energy,1 -ett1/15T/long,TempoPFN,11.978993055555556,11.978993055555556,1.8880533854166663,1.212444818759412,0.8287945896041671,0.4472149557037699,8.46201695858504,3.461068195738933,0.6596830202135081,0.3598648420592849,0.2797980199158211,Energy,7 -ett1/15T/medium,TempoPFN,13.43643074156746,13.43643074156746,1.8905306280226932,1.1797517678385212,0.7970212191155484,0.43954221337977,8.223282897531906,3.665573726112661,0.698661997148076,0.360337017595522,0.2811686910905668,Energy,7 -ett1/15T/short,TempoPFN,5.099480674380348,5.099480674380348,1.1698426564534503,0.741546008875918,0.4654608062507852,0.2595709059467272,5.147750148415939,2.2582029745752146,0.426894371974916,0.2211489630289341,0.1726175763221381,Energy,7 -ett1/D/short,TempoPFN,42729.2,42729.2,136.59449404761904,1.6581498701514856,1.2102758014012898,0.4676384699468007,10.56560792016809,206.71042547486567,0.5429317891645985,0.3587699695017287,0.2758014109082333,Energy,7 -ett1/H/long,TempoPFN,160.8376488095238,160.8376488095238,7.619004732969577,1.430575614677362,5479841931429.26,0.503485097120815,10.20657408167272,12.68217839369577,0.6073220555826839,0.3648576350433162,0.2829238662092257,Energy,7 -ett1/H/medium,TempoPFN,157.39791666666667,157.39791666666667,7.223530505952381,1.299741057732705,5054848608259.2,0.4486699790563304,9.650358222119532,12.545832641425864,0.6009156437329185,0.3459899879164509,0.2689166253737378,Energy,7 -ett1/H/short,TempoPFN,100.80698649088542,100.80698649088542,5.060569835844494,0.8403265022553135,0.4968278794061569,0.2622573751688414,5.98443659457911,10.040268247954604,0.4686203701424435,0.2361974850710038,0.1834135589681844,Energy,7 -ett1/W/short,TempoPFN,1279605.7142857143,1279605.7142857143,879.8480747767857,1.507155874023796,0.5954761505126953,0.5393904652465016,9.10318356177116,1131.1965851635666,0.4501301203526759,0.3501125489466342,0.2625724585314818,Energy,7 -ett2/15T/long,TempoPFN,13.533348214285716,13.533348214285716,2.424822048611111,1.0135265058563276,0.1584284454575112,0.185030629210618,7.045551229295476,3.678769932230842,0.196048308745081,0.129223156216707,0.101380599232709,Energy,7 -ett2/15T/medium,TempoPFN,12.825265066964286,12.825265066964286,2.290014009021577,0.9483801637803936,0.1461737087731561,0.1714602937799933,6.526389275058007,3.5812379238140943,0.1908506388582445,0.1220389837016552,0.0954723860923936,Energy,7 -ett2/15T/short,TempoPFN,8.133874802362351,8.133874802362351,1.7851745242164248,0.7703858853307536,0.1104452733682924,0.1309166272176335,5.293028437280506,2.851994881194977,0.1353943540652381,0.0847485923602811,0.0674543308068449,Energy,7 -ett2/D/short,TempoPFN,112962.61587301588,112962.61587301588,207.83965773809524,1.5496984874944315,0.4820026700458829,0.1511797712495964,19.462870152067044,336.0991161443539,0.1923583918454782,0.1189521197879121,0.1066884506136202,Energy,7 -ett2/H/long,TempoPFN,229.53148148148148,229.53148148148148,9.273859126984126,1.0113857673562383,0.1748218196029372,0.1693670062333175,9.479883980716991,15.150296415631,0.2074716488827785,0.1269983630549228,0.1072915845761625,Energy,7 -ett2/H/medium,TempoPFN,255.46834077380956,255.46834077380956,9.96962425595238,1.061375295248805,0.1751025049528656,0.1699638172677547,8.471492513095246,15.983377014067132,0.2153862837896859,0.1343470980744068,0.109065760500553,Energy,7 -ett2/H/short,TempoPFN,109.75648716517856,109.75648716517856,6.45131588890439,0.7442834948534669,0.1227267156291921,0.1111503639073804,5.304663232134608,10.476473030804716,0.1328708278042311,0.0818206356342305,0.0646933229648542,Energy,7 -ett2/W/short,TempoPFN,3040794.0,3040794.0,1185.226283482143,0.7919453687947827,0.1242664967264447,0.1334569111973995,10.314707020923404,1743.787257666485,0.1462770170301953,0.0994223145577723,0.0897493054435351,Energy,7 -hierarchical_sales/D/short,TempoPFN,29.341291517026665,29.341291517026665,2.3308575078641742,0.7515303852414364,0.643765657992357,1.0687853866383117,7.160107297447228,5.416760241789059,1.661802055179867,0.7150812707229562,0.5794574773220535,Sales,1 -hierarchical_sales/W/short,TempoPFN,458.0132622153072,458.0132622153072,8.93583847304522,0.7286080065136529,0.5309887622276143,0.4589218333357173,6.849441306693008,21.401244408101768,0.9825415989032096,0.4102487151412057,0.3436783258405172,Sales,1 -hospital/M/short,TempoPFN,2887.856787809648,2887.856787809648,17.96532290648088,0.7630582976954873,0.1943136875860487,0.1736909272797743,5.266586994518253,53.73878290219874,0.1950842608418231,0.0652182939530641,0.0524560110459564,Healthcare,1 -jena_weather/10T/long,TempoPFN,1718.90582010582,1718.90582010582,10.550130983382935,0.6640362455912168,0.7972626043441335,0.664936377234079,6.621161578850359,41.45968909803618,0.2536989403275045,0.064558058900818,0.0533160206667286,Nature,21 -jena_weather/10T/medium,TempoPFN,1660.2598022637085,1660.2598022637085,10.15341374120671,0.6136259378470442,0.7647985972503782,0.6674678211249887,5.93342577688109,40.74628574807412,0.250059272905106,0.0623113300026583,0.0510033972767451,Nature,21 -jena_weather/10T/short,TempoPFN,1050.4386885446215,1050.4386885446215,6.41726545303587,0.292725754515535,0.3439445336407977,0.568207526297445,2.755726611829969,32.41047189635815,0.2013096684158334,0.0398592647653549,0.03274036960386,Nature,21 -jena_weather/D/short,TempoPFN,427.30902777777777,427.30902777777777,10.311205667162698,1.2874194691385978,0.7325579906276438,0.486565054600677,8.388405658068676,20.671454418540023,0.1244759415252602,0.0620903110005419,0.0506077846040417,Nature,21 -jena_weather/H/long,TempoPFN,1290.070634920635,1290.070634920635,12.581970279431218,1.140988291353168,3.88548675463336,0.6861124363452801,8.200180878390375,35.91755329808303,0.2162634648338188,0.0757574009700474,0.0618072701020758,Nature,21 -jena_weather/H/medium,TempoPFN,1742.8521825396826,1742.8521825396826,12.35874488467262,0.8782377911156792,3.545601453541025,0.7001498803384193,7.342938930567742,41.747481152036976,0.255460241214288,0.0756253518109623,0.0600757598751637,Nature,21 -jena_weather/H/short,TempoPFN,1235.4418716876828,1235.4418716876828,8.964551015804485,0.5611385029075185,1.478170493854511,0.6203076482334885,5.110643040937704,35.148853063616215,0.2154602151292791,0.0549521228162633,0.0439211763031603,Nature,21 -kdd_cup_2018/D/short,TempoPFN,3007.9840201933307,3007.9840201933307,21.784901216472587,1.213689940300646,0.5259014022850484,0.4765536007317,9.068395818072355,54.845091122117125,1.2285882717386165,0.4880049169022034,0.3817844300375371,Nature,1 -kdd_cup_2018/H/long,TempoPFN,4026.646239183037,4026.646239183037,23.803200396499903,1.016140337194536,1.052279311582525,0.6002688257398056,7.894461230299585,63.45586055821036,1.488930964694955,0.5585192891155629,0.4361924024242362,Nature,1 -kdd_cup_2018/H/medium,TempoPFN,5010.101625543566,5010.101625543566,25.41210362428859,1.0732953214395051,1.052224683751761,0.5718608229379591,8.895520812896255,70.78207135669008,1.4815969585566695,0.5319213569004195,0.4250349871439885,Nature,1 -kdd_cup_2018/H/short,TempoPFN,4745.364931404211,4745.364931404211,24.703448951029102,1.0192517924056437,1.1250171321773017,0.508873616577222,7.690231024699732,68.88660923143344,1.4419214870539816,0.5170879252767191,0.407936825867448,Nature,1 -loop_seattle/5T/long,TempoPFN,143.79048381779612,143.79048381779612,7.197657165982256,1.124932179969269,0.2746223631846687,0.1540881178804492,10.301217356464582,11.991266981340884,0.2120080646688819,0.1272560579532165,0.103843879033806,Transport,1 -loop_seattle/5T/medium,TempoPFN,135.90292908281734,135.90292908281734,6.9837141397986,1.0929173620658874,0.2533539062043338,0.152741208437223,9.459404812612885,11.65774116554392,0.2074151689186729,0.1242546242377878,0.1007216468880828,Transport,1 -loop_seattle/5T/short,TempoPFN,55.07204281203641,55.07204281203641,4.052846149960293,0.6386709392602476,0.1157079082032352,0.0849917301121413,5.351151924838335,7.421054022983285,0.1272546927022406,0.0694973638226292,0.0555923827352211,Transport,1 -loop_seattle/D/short,TempoPFN,19.22348884763609,19.22348884763609,2.9199862233133387,0.882942169035729,0.0537918834863432,0.0539515098306408,7.512909316602903,4.384459926562917,0.0783561571785183,0.0521840553466773,0.0425073760052162,Transport,1 -loop_seattle/H/long,TempoPFN,92.96811078323871,92.96811078323871,5.41151807933974,1.116537919624389,0.1772630915266154,0.1214857343071853,8.075697052499791,9.641997240366685,0.1723153713454961,0.0967110572776757,0.0762777215333773,Transport,1 -loop_seattle/H/medium,TempoPFN,73.21742100828818,73.21742100828818,4.869904792714784,0.9948131743249428,0.1324426963971495,0.1104619028789109,7.238022698815357,8.55671788761837,0.1512390320842758,0.0860750227909825,0.0676708871822447,Transport,1 -loop_seattle/H/short,TempoPFN,76.14078191912911,76.14078191912911,4.676374909801503,0.9436139302609656,0.1369242086188304,0.1066310293785478,7.169357926178901,8.725868548123396,0.1544567497276496,0.0827765929652023,0.0655274010829642,Transport,1 -m4_daily/D/short,TempoPFN,1112905.8005677783,1112905.8005677783,196.52268612543932,4.372761062940581,0.0376063521746595,0.0322795529975073,73.70884513793729,1054.9435058654935,0.1629596160533486,0.0303573236848358,0.0251042328812495,Econ/Fin,1 -m4_hourly/H/short,TempoPFN,3069025.55805526,3069025.55805526,318.0727330083432,0.8084784636610177,0.1193998255399498,0.1061841264303322,5.710559329458197,1751.8634530280208,0.2391680663866767,0.0434239554415285,0.0323819182333365,Econ/Fin,1 -m4_monthly/M/short,TempoPFN,1888124.477016204,1888124.477016204,549.9968148871528,0.9217615703803564,0.157565068465692,0.1293010993297977,7.671573883821074,1374.0904180643297,0.2855831600688605,0.1143082189922729,0.0912326621749872,Econ/Fin,1 -m4_quarterly/Q/short,TempoPFN,1753619.374916667,1753619.374916667,552.2895295410157,1.1583565915207,0.1138860057890415,0.1004795715377529,8.745826363405456,1324.2429440690505,0.2216549501355986,0.092443541933982,0.0737285029120814,Econ/Fin,1 -m4_weekly/W/short,TempoPFN,607561.4484679665,607561.4484679665,284.2463205753161,2.4931339160639845,0.0628478183259998,0.0631521139918567,35.216213640568455,779.4622816198142,0.1420059370329369,0.0517852705041478,0.0427721929788442,Econ/Fin,1 -m4_yearly/A/short,TempoPFN,3981871.368097269,3981871.368097269,945.8485271393748,3.594287580979644,0.1813482780491965,0.1574376427947597,30.47808014725305,1995.4626952406977,0.3199816919998426,0.1516711952628658,0.1220860237418128,Econ/Fin,1 -m_dense/D/short,TempoPFN,10432.774814814817,10432.774814814817,49.92540509259259,0.716391519986865,0.1296904092368768,0.1042102598672269,7.594356199341907,102.14095561925596,0.176821102803412,0.0864282611500752,0.0723087394241984,Transport,1 -m_dense/H/long,TempoPFN,69720.16197530864,69720.16197530864,130.88989197530864,1.1207445037710384,0.4772352302896374,0.3084701081167127,8.331446128485986,264.04575735146483,0.4571022390360082,0.2265897520536809,0.1803573058785147,Transport,1 -m_dense/H/medium,TempoPFN,59202.59777777778,59202.59777777778,116.75945746527778,0.9974534903596788,0.4046579227656174,0.2835007501023778,7.860410512848903,243.31583955381487,0.4233388211334883,0.2031467049994665,0.1623521118844338,Transport,1 -m_dense/H/short,TempoPFN,56761.099027777775,56761.099027777775,105.46545247395834,0.9083752568964248,0.3625613449923406,0.2503984492754204,8.850123567663994,238.24587935109764,0.4225888717008309,0.1870694539850069,0.1532706642869601,Transport,1 -restaurant/D/short,TempoPFN,141.35223064313942,141.35223064313942,7.147021993998888,0.6863793596182625,0.6477974089763852,0.39361027068903,4.6517007613714565,11.889164421570568,0.5471668935978649,0.3289225116474076,0.2574888292666817,Sales,1 -saugeen/D/short,TempoPFN,1332.268125,1332.268125,14.217711588541668,3.159507034675852,0.343343505859375,0.370225833163857,29.900541686949477,36.50024828682676,1.1822517387572482,0.4605150659418648,0.3798277414404716,Nature,1 -saugeen/M/short,TempoPFN,436.14378720238096,436.14378720238096,13.104008265904016,0.7677055406440635,0.3685969625200544,0.3755904385589027,5.058659432159418,20.88405581304505,0.6270020070882578,0.3934216397990439,0.2987275977752708,Nature,1 -saugeen/W/short,TempoPFN,1022.00615234375,1022.00615234375,15.432559204101562,1.2518516198694998,0.3968208312988281,0.4121084413075842,11.22219787148846,31.968830950532897,0.9650057451344712,0.4658446321396566,0.3813628841565235,Nature,1 -solar/10T/long,TempoPFN,37.78305248251217,37.78305248251217,2.7866745948018043,1.214705032724981,1.8266880795155165,1.6107086534619184,9.20458964921274,6.146792048094044,1.3285307434179614,0.6022951227451945,0.4667998641968494,Energy,1 -solar/10T/medium,TempoPFN,30.12696470291418,30.12696470291418,2.3539968185576683,1.0259451011656044,2.551014183871936,1.535072505812766,6.89999148291423,5.488803576638007,1.217987341465626,0.5223612553848703,0.4057535032704926,Energy,1 -solar/10T/short,TempoPFN,25.65273684016689,25.65273684016689,2.24543791165317,0.9911362110444696,2.5917711639699004,1.5956671600988714,5.05982280366222,5.064853091666815,1.4733651872342612,0.653197632635979,0.4935287986242122,Energy,1 -solar/D/short,TempoPFN,140204.03698296836,140204.03698296836,249.91001368613135,0.9749128655715203,1.1146321020567214,0.4203658745642584,5.600866745464962,374.43829529438943,0.5409358132622821,0.3610348572637601,0.2822855355828816,Energy,1 -solar/H/long,TempoPFN,783.437510137875,783.437510137875,12.81624784570154,0.9761773348447412,3.3414418534142247,1.4562960320266205,7.556206423076039,27.98995373590094,0.9704628341834614,0.4443627283307781,0.3505849847892496,Energy,1 -solar/H/medium,TempoPFN,833.4198296836983,833.4198296836983,12.512497148722629,0.9572939417619708,3.160871762852397,1.450740022192459,7.168044400156375,28.869011581342686,1.0399291777627522,0.4507293516082588,0.3587224356592087,Energy,1 -solar/H/short,TempoPFN,741.199458357344,741.199458357344,11.236547508440031,0.8685164089622858,3.3186240137958527,1.43583049195247,6.191705384186862,27.224978574047476,1.0035217818558928,0.4141828852834149,0.3232686160844196,Energy,1 -solar/W/short,TempoPFN,2347435.0364963505,2347435.0364963505,1216.2723540145985,1.3307958944424054,0.2787131358237162,0.232816148029339,9.018767002302416,1532.1341444195905,0.3127468768391526,0.2482715899840278,0.182517771146522,Energy,1 -sz_taxi/15T/long,TempoPFN,16.503143919159545,16.503143919159545,2.79878986934651,0.528993560771414,8097110420372.057,0.4090025440315828,4.217670297139487,4.062406173582295,0.3760090499440382,0.2590509847610789,0.207437608211451,Transport,1 -sz_taxi/15T/medium,TempoPFN,17.080281366853633,17.080281366853633,2.8379674145299147,0.5548316711810012,11413195380557.455,0.4150869350122109,4.003654382317505,4.132829704555177,0.3846287796808951,0.2641202327358593,0.2086089818709996,Transport,1 -sz_taxi/15T/short,TempoPFN,17.24009844322344,17.24009844322344,2.7841574420859088,0.5558349832307647,956856373595.1888,0.4016793761734464,3.9498539926371095,4.152119752996467,0.3882409656073763,0.2603306354394822,0.20470716461367,Transport,1 -sz_taxi/H/short,TempoPFN,7.39170645037268,7.39170645037268,1.8780369391808145,0.5727259087699795,1.1904760874234712,0.3007927448972708,4.13172781105312,2.7187692896552806,0.2532701334745672,0.1749507279143944,0.1380282714985613,Transport,1 -temperature_rain/D/short,TempoPFN,185.9097353343313,185.9097353343313,5.882616175640533,1.3631103893097485,27.98747634396327,1.5210877307257795,25.653777081769206,13.634872032194924,1.605184930238957,0.6925394542187013,0.5670617778904212,Nature,1 -us_births/D/short,TempoPFN,173711.7066666667,173711.7066666667,262.520234375,0.3865791284094845,0.0251130040486653,0.0248028352333846,3.0451996411307154,416.7873638519607,0.0390704333267364,0.024609141743824,0.0195924956071727,Healthcare,1 -us_births/M/short,TempoPFN,42345229.333333336,42345229.333333336,5471.84765625,0.6194267273791539,0.016852726538976,0.0170317398110582,3.755457269583364,6507.32121024722,0.0202116953126723,0.0169955215137264,0.0125850097572043,Healthcare,1 -us_births/W/short,TempoPFN,1749907.142857143,1749907.142857143,1002.00244140625,0.911081904499245,0.0135813121284757,0.0135829704909661,5.83050142925883,1322.8405583656493,0.0179574308694424,0.0136020848913159,0.0107284204594915,Healthcare,1 diff --git a/results/TempoPFN/config.json b/results/TempoPFN/config.json deleted file mode 100644 index 355295966731ccc4c5f42b0e8459c5ef65df4709..0000000000000000000000000000000000000000 --- a/results/TempoPFN/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "TempoPFN", - "model_type": "zero-shot", - "model_dtype": "bfloat16", - "model_link": "https://github.com/automl/TempoPFN", - "code_link": "https://github.com/automl/TempoPFN/blob/main/examples/gift_eval/gift_eval_submission.ipynb", - "org": "University of Freiburg", - "testdata_leakage": "No", - "replication_code_available": "Yes" - } diff --git a/results/TiRex/all_results.csv b/results/TiRex/all_results.csv deleted file mode 100644 index 4ddae834d03c4db3c919945d67d9dade007a8f5d..0000000000000000000000000000000000000000 --- a/results/TiRex/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -hospital/M/short,TiRex,2644.19643633203,2644.19643633203,17.5862532594524,0.755865392174249,0.191281222095183,0.172168637192389,5.25967332209665,51.421750615202,0.186672895592297,0.0638421831090076,0.0507120184694813,Healthcare,1 -bitbrains_rnd/H/short,TiRex,1860029.63488342,1860029.63488342,148.293652231363,5.81352395077296,1.21926167009735,0.540491088435329,201.119558566922,1363.82903433071,5.968783393746,0.649005605948899,0.610826124077467,Web/CloudOps,2 -electricity/15T/short,TiRex,146322.356103604,146322.356103604,56.3978325591216,1.05886695554428,0.17361034036139,0.174779273368217,10.3802298347494,382.521053150808,0.754024396079158,0.111171244785838,0.0908779875964044,Energy,1 -electricity/15T/medium,TiRex,255671.678666667,255671.678666667,56.1211638513514,0.886755694424925,0.146496468651524,0.141831616650831,7.61869001719984,505.639870527104,0.871268413985601,0.0967024166207872,0.0785193705902253,Energy,1 -electricity/15T/long,TiRex,342905.674258258,342905.674258258,61.5929007132132,0.931412999226833,0.150075429994154,0.142895412135769,7.7065866990373,585.581483875864,0.923826556435333,0.097170349359635,0.078357523139035,Energy,1 -ett2/15T/short,TiRex,8.27063685825893,8.27063685825893,1.75263817196801,0.753517418470681,0.108876788573041,0.127567155020578,5.91709698916717,2.87587149543559,0.136527850674307,0.0832039689563025,0.0664633034282213,Energy,7 -ett2/15T/medium,TiRex,11.5075992063492,11.5075992063492,2.11123031374008,0.875448046886407,0.132833499862704,0.159348328605531,6.58646624550195,3.39228524837597,0.180781021367569,0.112511285023224,0.08887998139281,Energy,7 -ett2/15T/long,TiRex,11.4011954365079,11.4011954365079,2.152353515625,0.895088792806266,0.136160108385271,0.162007572234623,6.98905559555637,3.37656562745461,0.179943294314093,0.114702814890065,0.0917998952067343,Energy,7 -kdd_cup_2018/H/short,TiRex,4603.77870809985,4603.77870809985,22.6580666420206,0.938996642282699,0.871673945737486,0.492577772758213,7.38520178415405,67.8511511184582,1.42024748114532,0.474274371849209,0.378479913803786,Nature,1 -kdd_cup_2018/H/medium,TiRex,5263.3172234338,5263.3172234338,25.2960950972606,1.03550057716719,0.875891458239159,0.56021339373083,8.43783042836189,72.548723099954,1.51857616634071,0.529493083748186,0.42078225967313,Nature,1 -kdd_cup_2018/H/long,TiRex,4159.45667063118,4159.45667063118,24.0769403155594,1.01017684177464,0.944022770224736,0.607863225110001,7.55887128426801,64.4938498667212,1.51328633070932,0.564942312176045,0.441401874867448,Nature,1 -bitbrains_rnd/5T/short,TiRex,1822980.9044414,1822980.9044414,124.569639880882,1.65443881126234,1.03871106305341,0.652076655293488,53.0002449835185,1350.17810100794,5.52453899259735,0.509703003108993,0.404343446178176,Web/CloudOps,2 -bitbrains_rnd/5T/medium,TiRex,2271289.45280659,2271289.45280659,144.008904336876,4.41284012548038,0.323056186110018,0.706112744305564,160.75371303913,1507.07977652365,6.34105204289741,0.605918791603154,0.60358127879194,Web/CloudOps,2 -bitbrains_rnd/5T/long,TiRex,2663408.83434101,2663408.83434101,181.691378848576,3.35080382531189,1.6193283618618,0.689429497932538,115.778865334641,1631.99535365178,6.25180678895661,0.696018768214446,0.631590464570861,Web/CloudOps,2 -m4_weekly/W/short,TiRex,251983.208056567,251983.208056567,251.707306620956,1.9898576085037,0.0601451702947557,0.0600645180966427,13.6699438957489,501.979290465819,0.0914528473795872,0.0458571704728537,0.0366506530197832,Econ/Fin,1 -us_births/D/short,TiRex,224253.226666667,224253.226666667,300.685078125,0.442530892940835,0.0289126841227214,0.0283680566151937,4.61466207089751,473.553826578,0.0443918285740266,0.0281867861555422,0.0229982063460591,Healthcare,1 -covid_deaths/D/short,TiRex,358791.05764411,358791.05764411,101.769666353383,34.0894655192164,0.0877009080495907,0.364382146773183,590.76244317065,598.991700814052,0.225373542460132,0.038291332233613,0.0320417471338689,Healthcare,1 -hierarchical_sales/W/short,TiRex,466.043531514831,466.043531514831,8.91578803628178,0.722002990448829,0.521284520966481,0.456024978120448,6.72950422672361,21.5880414006188,0.991117540198871,0.409328189781998,0.347713604359324,Sales,1 -bitbrains_fast_storage/H/short,TiRex,3243963.03668098,3243963.03668098,334.448431569116,1.01909825389839,4.01457071055922,0.576481967492728,19.1859552323854,1801.10050710142,5.13372137052289,0.953286645422924,0.700457273184491,Web/CloudOps,2 -jena_weather/10T/short,TiRex,759.07251984127,759.07251984127,5.28411962115575,0.280013898626847,0.333174281482271,0.548704795231895,2.31333516965091,27.5512707482118,0.171127921943386,0.0328210055475335,0.0268340697020524,Nature,21 -jena_weather/10T/medium,TiRex,1490.73217893218,1490.73217893218,9.60529288419913,0.605888380590751,0.844295391597424,0.657101427218615,5.60158569649755,38.6100010221727,0.236948916222463,0.0589475182247028,0.0484508173393888,Nature,21 -jena_weather/10T/long,TiRex,1481.66798941799,1481.66798941799,9.74874235284392,0.64869173167028,0.87222681475266,0.6455078125,5.83779066010593,38.4924406788916,0.235541844869428,0.0596542260882094,0.0490159690415373,Nature,21 -sz_taxi/15T/short,TiRex,16.6106371384692,16.6106371384692,2.719239305955,0.542612799483091,1058725744525.09,0.393572170364697,3.86039157025905,4.07561493991633,0.381087427565312,0.254260504811613,0.199707467245458,Transport,1 -sz_taxi/15T/medium,TiRex,16.406492053953,16.406492053953,2.75556891025641,0.53719561769997,9773193493407.73,0.406178296733106,3.82328875190454,4.05049281618336,0.376965965360914,0.256451681687493,0.20224328876413,Transport,1 -sz_taxi/15T/long,TiRex,15.6001958689459,15.6001958689459,2.69801237535613,0.508947046551112,6486175791256.87,0.398897367732817,3.8126544811249,3.9497083270725,0.365577967070455,0.249723219447115,0.19804045792907,Transport,1 -bizitobs_l2c/H/short,TiRex,70.4708116319444,70.4708116319444,5.00163244822669,0.494049349450387,0.483508286144522,0.645069182865203,3.21610501906271,8.39468948990637,0.452495262677301,0.2696007983615,0.211869589661229,Web/CloudOps,7 -bizitobs_l2c/H/medium,TiRex,83.9045200892857,83.9045200892857,5.14754580543155,0.519407285636078,0.512528478223083,0.789695667085194,3.74675345707516,9.15994105271894,0.554648343893623,0.311691717191968,0.252226751198328,Web/CloudOps,7 -bizitobs_l2c/H/long,TiRex,85.6205667162698,85.6205667162698,5.50080760168651,0.584526991382213,0.67236099600234,0.844378565228175,4.61491715806473,9.25313820907641,0.565208388011856,0.336005204619431,0.267920303958168,Web/CloudOps,7 -m4_daily/D/short,TiRex,355053.278988813,355053.278988813,171.296116430515,3.23342416752454,0.0366710728721357,0.0300325869853187,26.7510831760996,595.863473447411,0.0920444396869217,0.0264605161416795,0.0209845012924179,Econ/Fin,1 -temperature_rain/D/short,TiRex,184.264866911557,184.264866911557,5.73554940948832,1.32775200467932,19.7353202546275,1.53731190474545,18.9116728320248,13.5744195791775,1.59806806307989,0.675225801151743,0.547114671834489,Nature,1 -solar/W/short,TiRex,1436996.67153285,1436996.67153285,949.579664689781,1.03917831602574,0.210086015019104,0.185790514423899,7.25307866673056,1198.74795997025,0.244694428808035,0.193832950226465,0.143733119722635,Energy,1 -solar/H/short,TiRex,588.254377961327,588.254377961327,9.66517399795108,0.744055301637309,3.52267769565368,1.39275235930697,4.10300131920922,24.2539559239586,0.894008917992685,0.356261542455586,0.27294675614805,Energy,1 -solar/H/medium,TiRex,669.71800486618,669.71800486618,10.2703286572384,0.783587840190401,3.66420620378017,1.39278866076384,4.57372507515444,25.8789104265651,0.932218755237414,0.369961209298128,0.285002422784048,Energy,1 -solar/H/long,TiRex,684.682725060827,684.682725060827,10.864412256691,0.825937782882037,4.45946881094626,1.39144387517171,4.89489661061675,26.1664427284418,0.907238256381854,0.376688972768173,0.286549909649834,Energy,1 -bizitobs_l2c/5T/short,TiRex,21.9927083333333,21.9927083333333,2.70683274042039,0.279404233539459,0.14863716297864,0.209346516927083,3.04083128718608,4.68963840112789,0.161061164918434,0.0929635927381121,0.0757868723998155,Web/CloudOps,7 -bizitobs_l2c/5T/medium,TiRex,110.856505102041,110.856505102041,6.12811636373299,0.619657460159213,0.422435374290898,0.716299625318878,3.41074281082924,10.528841584051,0.553971900641935,0.322429038586215,0.251045131938078,Web/CloudOps,7 -bizitobs_l2c/5T/long,TiRex,118.173561507937,118.173561507937,6.07899677579365,0.606914542915138,0.451283226613999,0.788613978794643,3.45909915059771,10.8707663716932,0.610377279945655,0.341326580844308,0.269298286344833,Web/CloudOps,7 -ett2/H/short,TiRex,108.625651041667,108.625651041667,6.31626848493304,0.723450509844985,0.117928764649165,0.107504563104539,5.24903193204832,10.422363025805,0.132184562384975,0.0801078587955249,0.0637035663238856,Energy,7 -ett2/H/medium,TiRex,239.600446428571,239.600446428571,9.7535888671875,1.13485833908813,0.19244802425261,0.186294846307664,8.502277426429,15.479032477147,0.208589916822852,0.13143588228364,0.107098715176229,Energy,7 -ett2/H/long,TiRex,269.734623015873,269.734623015873,10.3482804232804,1.18447623253982,0.201857746503876,0.199055101505663,9.03218502617122,16.4235995754851,0.224908555650492,0.141711735771999,0.114179095263771,Energy,7 -jena_weather/D/short,TiRex,349.400992063492,349.400992063492,9.24314856150794,1.0148134039666,0.580600590068475,0.448921421595982,6.74877929327845,18.6922709177749,0.112558022025216,0.0556588615661044,0.0443666263226254,Nature,21 -kdd_cup_2018/D/short,TiRex,2898.74858700712,2898.74858700712,21.1785981448235,1.20081279350788,0.532790038867694,0.46727168216803,9.09800562951533,53.8400277396577,1.20607375328813,0.474423071946037,0.376010434817368,Nature,1 -electricity/D/short,TiRex,1217102237.29009,1217102237.29009,4187.82082882883,1.41100095624963,0.462302094433848,0.0951578468116554,11.6408151311046,34886.9923795401,0.57330914482646,0.0688198040101141,0.0544045662328728,Energy,1 -m_dense/D/short,TiRex,9682.07407407408,9682.07407407408,46.8870775462963,0.676466322469793,0.12428814186,0.0975540500217014,7.52986235294712,98.3975308332179,0.17034068077372,0.0811684667399199,0.0677391601735547,Transport,1 -us_births/W/short,TiRex,2231215.85714286,2231215.85714286,1151.05894252232,1.04645785562603,0.0155978894659451,0.0156046194689614,6.41147759786562,1493.72549591378,0.0202771771406339,0.015625512278317,0.0123022329409069,Healthcare,1 -restaurant/D/short,TiRex,137.628537405436,137.628537405436,7.06953832772952,0.677866081421558,0.684489547944686,0.390664766624638,4.65087208860166,11.731518972641,0.539911684777291,0.325356533797841,0.254774507499283,Sales,1 -loop_seattle/H/short,TiRex,59.0568889692304,59.0568889692304,4.06274919290777,0.824491105530812,0.111127780771134,0.0933387121911557,6.96009703685967,7.68484801210996,0.136029624639534,0.0719147921787039,0.0579328784553887,Transport,1 -loop_seattle/H/medium,TiRex,70.4620098039216,70.4620098039216,4.4874209881321,0.918036210957686,0.115524874629128,0.102947544534878,7.54305505942023,8.39416522376833,0.148365913951787,0.0793146546967499,0.0639476714240825,Transport,1 -loop_seattle/H/long,TiRex,61.2371517027864,61.2371517027864,4.18108364508084,0.874260418832198,0.121539569089697,0.0971182503977468,7.31816185892794,7.82541703059884,0.139850679080602,0.0747215624127076,0.0599976849474176,Transport,1 -bitbrains_fast_storage/5T/short,TiRex,1793533.6468832,1793533.6468832,152.617672373269,0.690323257895346,1.76809231187727,0.749194142434284,14.7245390050409,1339.22875076784,4.20484711707723,0.479181752427164,0.379986469449896,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,TiRex,3488059.71099448,3488059.71099448,289.036957313442,1.00319651211573,4.27677880587811,0.807874056337002,21.1835531721736,1867.63479058259,5.67439064160822,0.878174155851259,0.63844764801188,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,TiRex,4253620.75945946,4253620.75945946,351.517821910294,0.915079747536741,4.62810728117076,0.826736469454913,16.7047656520911,2062.43078901074,5.45038881902059,0.928956654659582,0.67175241451944,Web/CloudOps,2 -ett1/D/short,TiRex,42060.2920634921,42060.2920634921,137.44466765873,1.67920212923972,1.16163775731647,0.478167724609375,10.8968733055242,205.086060139377,0.538665339730188,0.361002978691859,0.27701612446829,Energy,7 -saugeen/D/short,TiRex,1260.25520833333,1260.25520833333,13.4994775390625,2.99962314573788,0.30044189453125,0.341775614420573,29.1906511120771,35.5000733567317,1.14985583446437,0.437251293948898,0.35754300109357,Nature,1 -solar/10T/short,TiRex,29.0384784633516,29.0384784633516,2.45663135834854,1.08840250635685,6.14568620138451,1.54691539309611,5.14650642147991,5.38873625847022,1.56758279402509,0.714633796106937,0.54092029681464,Energy,1 -solar/10T/medium,TiRex,21.0330122622207,21.0330122622207,1.9521716797307,0.856162079212004,3.54719020038737,1.46772919949956,5.08960526936923,4.58617621360331,1.01769067602589,0.433194631852752,0.334213226821628,Energy,1 -solar/10T/long,TiRex,19.4040389136,19.4040389136,1.89603823919366,0.830923895840091,2.94041326833921,1.47183405248885,5.39216370608081,4.40500157929597,0.95207052332571,0.409798290906215,0.320531392485542,Energy,1 -m4_monthly/M/short,TiRex,1816001.82014815,1816001.82014815,551.921800636574,0.921364086798541,0.158591870095995,0.130974720036542,7.46045679873852,1347.59111756799,0.28007569768415,0.114708297914098,0.0914518007923697,Econ/Fin,1 -bizitobs_service/10S/short,TiRex,2880.53544973545,2880.53544973545,20.1323511904762,0.825489803577478,0.0583618809937169,0.0484123916222305,7.63762366043384,53.6706199865015,0.0397609110925938,0.0149146893769948,0.0117277966973059,Web/CloudOps,2 -bizitobs_service/10S/medium,TiRex,13943.826031746,13943.826031746,37.5231001984127,1.1744109098841,0.0685378059508309,0.0618882291279142,11.7837792617469,118.083978725931,0.0888727850510518,0.0282407694452991,0.0230622873768297,Web/CloudOps,2 -bizitobs_service/10S/long,TiRex,131327.972910053,131327.972910053,78.7745436507937,1.4943902945243,0.0881985845010747,0.0853091026992394,19.8262119398491,362.39201551642,0.268471590476097,0.0583587002953685,0.0530869721056591,Web/CloudOps,2 -jena_weather/H/short,TiRex,1122.3045112782,1122.3045112782,8.26004137949039,0.523626147293653,1.04545920790151,0.60624114811769,4.45395155685419,33.5008135912875,0.205357830639192,0.050633521901189,0.0408131511606026,Nature,21 -jena_weather/H/medium,TiRex,1594.23343253968,1594.23343253968,10.6992854042659,0.757858239977878,1.925273364389,0.66883777436756,5.79843837700476,39.9278528415902,0.244325612865952,0.0654708249400793,0.0527998392140855,Nature,21 -jena_weather/H/long,TiRex,1371.43439153439,1371.43439153439,11.8446531911376,0.984803991760474,2.25631090728243,0.662045497995205,7.15663575501418,37.0328825712284,0.222978982760088,0.0713179352059895,0.0586325972922798,Nature,21 -bizitobs_application/10S/short,TiRex,528159.253333333,528159.253333333,380.366631944444,1.20508471840302,0.0164307202233209,0.0167000071207682,8.07649955082409,726.745659315096,0.0280276751943635,0.0146692206252183,0.0114647240446569,Web/CloudOps,2 -bizitobs_application/10S/medium,TiRex,4210563.84,4210563.84,1142.3096875,2.64176880560029,0.0376736831665039,0.038354237874349,33.4502951134848,2051.96584766901,0.0805746595416203,0.0448551394098318,0.038027088981558,Web/CloudOps,2 -bizitobs_application/10S/long,TiRex,6242166.89777778,6242166.89777778,1524.97083333333,3.3952556662572,0.0489295069376628,0.0498919762505425,58.5291787797066,2498.43288838779,0.096354569983455,0.0588120295590415,0.0518335702120556,Web/CloudOps,2 -solar/D/short,TiRex,128172.371776156,128172.371776156,245.218080900243,0.956101721293655,1.07360655698753,0.418233443582725,5.29267763868974,358.011692233865,0.517204954929065,0.354256604549535,0.27272924798333,Energy,1 -m4_hourly/H/short,TiRex,793757.938808374,793757.938808374,179.838315217391,0.722167134003322,0.0859587910478626,0.0785340940318822,5.45090485663555,890.93093941583,0.121631738496967,0.0245518995474546,0.0201566007817003,Econ/Fin,1 -m4_quarterly/Q/short,TiRex,1734981.68666667,1734981.68666667,545.696161458333,1.14570153997989,0.110111211458842,0.0996874338785807,8.75125965271807,1317.18703556734,0.22047391484399,0.0913399280309771,0.0728452027046033,Econ/Fin,1 -car_parts/M/short,TiRex,1.39280723185594,1.39280723185594,0.459582583590999,0.846948909033782,0.830748690126757,1.89617928841587,15.7911825986166,1.18017254325626,2.82993269611018,1.10203189126775,0.995178384556839,Sales,1 -us_births/M/short,TiRex,43087882.6666667,43087882.6666667,5335.27994791667,0.601660107977772,0.0164558365941048,0.0166467999418577,3.876819976161,6564.13609446564,0.0203881619556958,0.01657134314275,0.0124333765361167,Healthcare,1 -loop_seattle/5T/short,TiRex,43.4963103775639,43.4963103775639,3.5392723248839,0.560244137718282,0.0996245857608823,0.0750905394185069,4.30820887489137,6.59517326365001,0.113092664433702,0.0606907083978658,0.0479319004409381,Transport,1 -loop_seattle/5T/medium,TiRex,88.6736284184727,88.6736284184727,5.58500180396994,0.880264287103265,0.175820255811,0.126898798356976,6.41590284012957,9.41666758564157,0.167541861632186,0.0993686557315688,0.0780106683606492,Transport,1 -loop_seattle/5T/long,TiRex,106.183226694186,106.183226694186,6.06466739766082,0.951881420955391,0.184712493550052,0.136631232262642,6.93582870994652,10.3045245739038,0.182186101859072,0.10722455988409,0.0843039723354861,Transport,1 -sz_taxi/H/short,TiRex,7.25931073050214,7.25931073050214,1.84754461304754,0.562878902792566,1.06078162560096,0.295248667399089,3.94262819039669,2.69431080807359,0.250991678061704,0.17211018169586,0.135355970215781,Transport,1 -ett2/D/short,TiRex,108861.206349206,108861.206349206,199.126165674603,1.31020561529259,0.413035026429192,0.135618349105593,12.4410612455298,329.941216505617,0.188834063411527,0.113965158382242,0.0936081122506195,Energy,7 -saugeen/W/short,TiRex,922.7759765625,922.7759765625,14.4323623657227,1.17070485274757,0.363051891326904,0.385186767578125,10.3896555914127,30.3772279275529,0.916961884427595,0.435652858884186,0.350148571341342,Nature,1 -electricity/W/short,TiRex,39204819807.4811,39204819807.4811,26513.3567567568,1.4415945973124,0.139054505346481,0.0937778112050649,11.1714338490138,198002.070210089,0.451775383322202,0.0604947306825267,0.0480246177985173,Energy,1 -ett2/W/short,TiRex,2954849.71428571,2954849.71428571,1180.03362165179,0.765193274498532,0.120308228901454,0.128949105739594,9.2935123605046,1718.96763037752,0.144195030808729,0.0989867298385632,0.086646890601686,Energy,7 -loop_seattle/D/short,TiRex,18.1092379385965,18.1092379385965,2.90261908217234,0.874606980295278,0.0540785404673794,0.0535481770833333,7.49266430412173,4.25549502861847,0.0760513876149473,0.0518736850665518,0.0420953143590836,Transport,1 -m4_yearly/A/short,TiRex,4869937.2150547,4869937.2150547,896.544617466121,3.36403414140292,0.164740760944692,0.149613472171497,29.2070103808379,2206.79342373832,0.353869554742539,0.14376508520318,0.11602893484346,Econ/Fin,1 -ett1/15T/short,TiRex,4.74445452008929,4.74445452008929,1.10784432547433,0.709254882775933,0.455971513921977,0.242423720586868,4.81084525137784,2.17817687989045,0.41176609725318,0.209428691708265,0.160948264490076,Energy,7 -ett1/15T/medium,TiRex,9.39411644345238,9.39411644345238,1.60454582093254,1.01440816611886,0.747097280139764,0.395592254154266,6.9625330088379,3.06498229088724,0.584188655121756,0.305828019952596,0.237291025559533,Energy,7 -ett1/15T/long,TiRex,8.89391245039683,8.89391245039683,1.58203528025794,1.02004163567787,0.775535500830036,0.397329605344742,7.37650035900291,2.98226632787832,0.568422910787457,0.301537488642885,0.234493783525133,Energy,7 -ett1/H/short,TiRex,99.027734375,99.027734375,4.89497244698661,0.816757639069635,0.458388446626209,0.257867576962426,5.61107263994962,9.95126797825282,0.464466388868604,0.2284683902626,0.176046429435933,Energy,7 -ett1/H/medium,TiRex,135.180989583333,135.180989583333,6.75149158296131,1.23502868249193,3822758029070.15,0.438204338437035,8.85834579393572,11.6267359814925,0.55689309243306,0.323380442469499,0.252026548063122,Energy,7 -ett1/H/long,TiRex,141.562020502646,141.562020502646,7.00184720155423,1.32065939110372,5616380927470.63,0.467359593305638,9.13125184655765,11.8979838839463,0.569768678958172,0.335303297534248,0.257826785729453,Energy,7 -electricity/H/short,TiRex,1617627.05900901,1617627.05900901,165.516425957207,0.908995957160898,0.189200266734236,0.123060811017011,10.3719161033991,1271.8596852676,0.600408599879982,0.0781355732218182,0.0638134616449604,Energy,1 -electricity/H/medium,TiRex,6710335.6209009,6710335.6209009,250.399177927928,1.08821158390077,0.251182429125225,0.131136623073269,9.24825714086903,2590.43155109354,1.01113080018223,0.0977390508683225,0.0786019893653698,Energy,1 -electricity/H/long,TiRex,10662922.2584505,10662922.2584505,296.024432432432,1.23396553888477,0.30542091744701,0.145356618337087,10.5566407749733,3265.41303029961,1.2878779333268,0.116751948595103,0.0938948461663266,Energy,1 -m_dense/H/short,TiRex,43959.8266666667,43959.8266666667,88.3184201388889,0.782375729444844,0.28625055021442,0.210613640679253,8.0076608767733,209.665988340185,0.371895217899888,0.156654869786674,0.128349506926327,Transport,1 -m_dense/H/medium,TiRex,37094.3244444444,37094.3244444444,80.6302864583333,0.69800879470509,0.259518864511837,0.194461992051866,7.82987362265595,192.59886927094,0.335097700249675,0.140286511883156,0.11608219052137,Transport,1 -m_dense/H/long,TiRex,34514.3664197531,34514.3664197531,80.4806481481482,0.694068153401345,0.265958946339283,0.193699529200424,7.42600281429142,185.780425286824,0.321613379512223,0.139323899147602,0.115084326051384,Transport,1 -hierarchical_sales/D/short,TiRex,28.9036378668871,28.9036378668871,2.30875232534105,0.745221034554874,0.644201467243194,1.06602345687784,6.80702982300087,5.37621036296824,1.64936180142046,0.708299645524991,0.570194378757521,Sales,1 -ett1/W/short,TiRex,1525286.85714286,1525286.85714286,960.383998325893,1.5982252820467,0.752675669533866,0.557370356151036,8.31930983349622,1235.02504312376,0.491445941926606,0.382159715138028,0.277767861011278,Energy,7 -saugeen/M/short,TiRex,417.812081473214,417.812081473214,12.7812064034598,0.748811134342036,0.37057926541283,0.363821211315337,4.69356041675726,20.4404520858325,0.61368369239862,0.383730159491952,0.297045254137591,Nature,1 diff --git a/results/TiRex/config.json b/results/TiRex/config.json deleted file mode 100644 index 63b2f75f2b20ddb0008433b3e1a7c98ba20c5ace..0000000000000000000000000000000000000000 --- a/results/TiRex/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "TiRex", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/NX-AI/TiRex-1.1-gifteval", - "code_link":"https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/tirex.ipynb", - "org": "NXAI", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} diff --git a/results/TimeCopilot/all_results.csv b/results/TimeCopilot/all_results.csv deleted file mode 100644 index 5d6f7ca0ac8b788a00b2811f17acd064165b1495..0000000000000000000000000000000000000000 --- a/results/TimeCopilot/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,TimeCopilot,3623318.292956872,3623318.292956872,896.1415311682032,3.352789096613149,0.164011590722566,0.1507799592310456,28.617921130926973,1903.5015873271216,0.3052353032671677,0.1437004486140326,0.1155226987213568,Econ/Fin,1 -m4_quarterly/Q/short,TimeCopilot,1759815.6624919923,1759815.6624919923,550.2454226355738,1.1590601775264513,0.1107441604242053,0.1003228954583047,8.824084356821759,1326.5804395105456,0.2220462048644621,0.092101394081576,0.0733132251866769,Econ/Fin,1 -m4_monthly/M/short,TimeCopilot,1817081.272341091,1817081.272341091,549.841198444464,0.911253241052204,0.157745402393644,0.1300634206629717,7.464709178246987,1347.991569833095,0.2801589239299355,0.1142758767457428,0.0909937816657743,Econ/Fin,1 -m4_weekly/W/short,TimeCopilot,220623.14482081163,220623.14482081163,240.2263895131104,1.9198034914621185,0.0586079585284285,0.0582746544101475,14.63148685238026,469.7053808727463,0.0855730412114503,0.0437655252994698,0.0349978393293822,Econ/Fin,1 -m4_daily/D/short,TimeCopilot,353490.1571844316,353490.1571844316,170.39705484441762,3.175756961031712,0.0376861601452085,0.0293684759592992,24.86309668044434,594.5503823768273,0.0918416029249228,0.0263216358351925,0.0207889066555366,Econ/Fin,1 -m4_hourly/H/short,TimeCopilot,852726.6250467548,852726.6250467548,185.3077787252216,0.6860634673852392,0.0873776644826867,0.078425752047952,5.539411902208129,923.4319818193188,0.126068848171362,0.0252986020421949,0.0197096877526468,Econ/Fin,1 -electricity/15T/short,TimeCopilot,128293.92468774492,128293.92468774492,49.04157622972897,0.9321482968110442,0.1567505974177776,0.1503585689406013,10.097822596196648,358.1814130964153,0.7060461707827386,0.0966706139406217,0.0801835324305198,Energy,1 -electricity/H/short,TimeCopilot,1496151.039929627,1496151.039929627,158.53752573243503,0.8669619484111293,0.176574345363005,0.1128698777119714,9.957074831866423,1223.172530728853,0.57742478868508,0.0748410342735203,0.0612837704789279,Energy,1 -electricity/D/short,TimeCopilot,1218673437.5753818,1218673437.5753818,4116.138977795666,1.377821987404302,0.3136554862575263,0.0906473868141199,11.139877312635267,34909.50354237914,0.573679105169932,0.067641836346091,0.0532369653564484,Energy,1 -electricity/W/short,TimeCopilot,46800263075.57075,46800263075.57075,28387.150774844056,1.4572521701129888,0.2035680074497389,0.0954241856309926,11.263974781479208,216333.6845606129,0.493602077817544,0.0647701102777502,0.0499949942858169,Energy,1 -solar/10T/short,TimeCopilot,24.37663747553509,24.37663747553509,2.241486510167475,0.9930148490188802,5.453790219123076,1.525427904111361,4.818265082483623,4.937270245341559,1.4362513796966991,0.6520481830698053,0.4976428465742399,Energy,1 -solar/H/short,TimeCopilot,654.0650849339982,654.0650849339982,10.889763498247271,0.8397061913589596,4.044817670924707,1.305372400309357,4.080803341755496,25.574696184588355,0.9426918072029532,0.4014003043508964,0.3013797124116734,Energy,1 -solar/D/short,TimeCopilot,132232.75812333074,132232.75812333074,248.9278371060272,0.9707494856784976,1.1013781669415048,0.4204974404829871,5.580218344148475,363.6382242330016,0.5253333772463988,0.3596159387076965,0.2774434347937445,Energy,1 -solar/W/short,TimeCopilot,1878244.8709359532,1878244.8709359532,1108.7167350045086,1.2161073536974203,0.2528584586107666,0.2158924147477143,7.377999107631626,1370.4907409158054,0.2797514241804544,0.2263168049008143,0.1616292900652415,Energy,1 -hospital/M/short,TimeCopilot,2861.3043412096604,2861.3043412096604,17.815051922483168,0.756820374897648,0.1918730465508048,0.1724875792676216,5.129152455270326,53.49116133726824,0.1941853370598893,0.0646727753861007,0.0508432960045565,Healthcare,1 -covid_deaths/D/short,TimeCopilot,432023.4929762012,432023.4929762012,113.50011927932432,36.29551561,0.0943375167289825,0.2930030792290609,665.8345345309949,657.2849404757432,0.2473066575703719,0.0427049722339471,0.035212059008402,Healthcare,1 -us_births/D/short,TimeCopilot,150367.2099304533,150367.2099304533,225.4844449869792,0.3319835430317066,0.0217841119232014,0.0214096680897595,3.4161792355552234,387.7721108208445,0.0363504888002673,0.0211373370167937,0.0172893963106227,Healthcare,1 -us_births/M/short,TimeCopilot,43992063.817342125,43992063.817342125,5570.623697916667,0.6282748881799558,0.017173388660799,0.0173568175516474,4.223405234303391,6632.651341457813,0.0206009698457224,0.0173023192256966,0.0129348949461795,Healthcare,1 -us_births/W/short,TimeCopilot,2163478.104384831,2163478.104384831,1132.98193359375,1.03020678537136,0.0153375293882701,0.0153549572601015,6.147698046075086,1470.8766448566755,0.0199670062279647,0.0153801186546423,0.0120682786004195,Healthcare,1 -saugeen/D/short,TimeCopilot,1310.9408078477593,1310.9408078477593,13.540265766779582,3.0085533936225217,0.2832238947087966,0.3398187935873074,33.38935095357659,36.20691657470654,1.1727506547108713,0.4385724343630787,0.3611090832066537,Nature,1 -saugeen/M/short,TimeCopilot,403.8431601034509,403.8431601034509,12.41271462326958,0.727292821544807,0.356969898623353,0.3554020929246113,4.8921690305678185,20.095849325257465,0.6033376837275577,0.3726669307856539,0.2952004063922404,Nature,1 -saugeen/W/short,TimeCopilot,976.6330734191066,976.6330734191066,14.712926697731016,1.1935373084045853,0.3624084211066235,0.3849981303798641,9.952326469459296,31.251129154305875,0.943341319628137,0.444121926542206,0.3526089345644552,Nature,1 -temperature_rain/D/short,TimeCopilot,182.97082425198397,182.97082425198397,5.724791233387727,1.330618454426622,18.00570456381298,1.5172500198157917,18.74885913350685,13.526670848807695,1.5924467974747458,0.6739592888536465,0.5444793711800684,Nature,1 -kdd_cup_2018/H/short,TimeCopilot,4200.141746069787,4200.141746069787,21.443265815157083,0.8915718317390194,0.8162245764901869,0.4695068771671895,7.246412088703758,64.80850056952241,1.3565592827952695,0.4488463864982175,0.3590324633205042,Nature,1 -kdd_cup_2018/D/short,TimeCopilot,2908.3090880968493,2908.3090880968493,21.231837754770275,1.1951957709349497,0.5254420086581945,0.4645847928469535,9.25007430881293,53.92874083544738,1.208061077003669,0.4756157178425276,0.3768255724738475,Nature,1 -car_parts/M/short,TimeCopilot,1.3613521799160238,1.3613521799160238,0.4524161639078284,0.8300575655061477,0.8324655191852736,1.889632060031151,15.229305483944689,1.1667699772945923,2.7977947177752145,1.0848475520019831,0.9531399271136942,Sales,1 -restaurant/D/short,TimeCopilot,140.86394165124764,140.86394165124764,7.146748193552639,0.6850637366419515,0.6778804399905437,0.393663886113605,4.7079549404016285,11.868611614306351,0.5462210057871897,0.328909910716482,0.257865182979218,Sales,1 -hierarchical_sales/D/short,TimeCopilot,28.83104995923876,28.83104995923876,2.2949516654281727,0.7407932320969244,0.6306875058992247,1.0532199960486268,6.875123309921721,5.369455275839325,1.6472894154973845,0.704065755896809,0.5684040271207136,Sales,1 -hierarchical_sales/W/short,TimeCopilot,466.5947955950688,466.5947955950688,8.886685136421505,0.7155719050966906,0.5194323676636545,0.4541925009709089,6.595446802894897,21.60080543857263,0.9917035434246873,0.4079920614141257,0.3432383281258184,Sales,1 -loop_seattle/5T/short,TimeCopilot,41.05880358408309,41.05880358408309,3.4619420538542043,0.5484964553129523,0.0968595706477452,0.0734875439823379,4.220341372318343,6.407714380657357,0.109878155316302,0.0593646632936229,0.0469132315884219,Transport,1 -loop_seattle/H/short,TimeCopilot,56.69559325731581,56.69559325731581,3.996556123755533,0.8128023049231333,0.1091552347737474,0.0918297966758414,6.983060020091911,7.529647618402591,0.1332824208466624,0.0707431080734554,0.0571968441901989,Transport,1 -loop_seattle/D/short,TimeCopilot,17.849240444502655,17.849240444502655,2.8742780019132224,0.8663526708790623,0.0535685031839717,0.0530851012748574,6.715346474731288,4.224836144101054,0.0755034701685097,0.0513671905776679,0.0416252415962686,Transport,1 -sz_taxi/15T/short,TimeCopilot,16.587558292802978,16.587558292802978,2.7148265982800317,0.5415867631608944,1065053949483.8048,0.3919517879679624,3.849297032361695,4.072782622827172,0.3808225938041576,0.2538478977716339,0.1995179385365098,Transport,1 -sz_taxi/H/short,TimeCopilot,7.118109626052236,7.118109626052236,1.835493657050812,0.5591594403657988,1.1356446911404448,0.2940101467272279,3.903943788673277,2.667978565515892,0.2485386671741405,0.1709875607796671,0.1345639324832183,Transport,1 -m_dense/H/short,TimeCopilot,43238.41671006195,43238.41671006195,86.04011993004768,0.7708026475594886,0.2925660113158236,0.2054638496608183,8.42983888949354,207.93849261274823,0.3688310709437598,0.1526137329322135,0.1261792340095731,Transport,1 -m_dense/D/short,TimeCopilot,7942.468547215981,7942.468547215981,42.838499071333146,0.6449696296594424,0.1183839490247463,0.093503758955741,7.365426418862357,89.12052820319222,0.1542808169726993,0.0741597785365545,0.0617285355059594,Transport,1 -ett1/15T/short,TimeCopilot,5.004077911322244,5.004077911322244,1.083807275260222,0.679642015114924,0.4801456951841858,0.237790329911371,4.528747375771124,2.2369796403459383,0.4228822666533839,0.204884688671817,0.1581378355031145,Energy,7 -ett1/H/short,TimeCopilot,98.73134309317892,98.73134309317892,4.879974661319561,0.803668040017943,0.4568973705975855,0.2561611292178454,5.484562912085751,9.936364681973933,0.4637707910593578,0.2277683822470367,0.1740989812702882,Energy,7 -ett1/D/short,TimeCopilot,41313.56931695536,41313.56931695536,137.69056003240365,1.6990245493301293,1.0867150456288373,0.4931039006217519,10.776630787435236,203.2573967090875,0.5338622945731478,0.3616488231676456,0.2758487954267685,Energy,7 -ett1/W/short,TimeCopilot,1288137.5144635793,1288137.5144635793,896.4227324894497,1.5704922299986386,0.6782030172720718,0.5608639209297829,8.469957902472162,1134.961459461765,0.4516282536950035,0.3567080008502722,0.2700317578526821,Energy,7 -ett2/15T/short,TimeCopilot,7.845169278633598,7.845169278633598,1.697435298757184,0.73048926379873,0.1050850003650307,0.1239507280483686,5.3468864133844365,2.800922933362073,0.1329697758064737,0.0805832921831985,0.0642716857192217,Energy,7 -ett2/H/short,TimeCopilot,111.00168618506125,111.00168618506125,6.447304992909943,0.736526386469788,0.1191696938136581,0.1097750079327987,5.117158924184468,10.535733775350495,0.1336224189323656,0.0817697663130905,0.0649558725663057,Energy,7 -ett2/D/short,TimeCopilot,104667.7976687638,104667.7976687638,191.95323600693357,1.26609597419066,0.4280713550391729,0.13173137724248,12.17382192946875,323.5240295074908,0.1851613379807318,0.1098599014820692,0.0913283421234515,Energy,7 -ett2/W/short,TimeCopilot,2997038.487662636,2997038.487662636,1187.314319065639,0.7885949778805613,0.1216001834193879,0.1282851838986621,8.607380365760186,1731.1956815053102,0.1452207768309006,0.0995974687316911,0.0838890272508537,Energy,7 -jena_weather/10T/short,TimeCopilot,905.905882028894,905.905882028894,6.0942343897399,0.3010985667231378,0.3518341513188411,0.5286519694290023,2.6589290599225808,30.0982704159042,0.1869479820889232,0.03785283359461,0.0300771704835859,Nature,21 -jena_weather/H/short,TimeCopilot,1096.616969477417,1096.616969477417,8.29910067857465,0.521866958871569,1.1456281352471576,0.5516464715641278,4.506464492029438,33.115207525809296,0.2029940902818978,0.0508729528900625,0.0409198306297826,Nature,21 -jena_weather/D/short,TimeCopilot,366.4712700248433,366.4712700248433,9.472486530442811,1.002191012190578,0.6191777053052017,0.4509569993161887,6.66281943555144,19.143439346806083,0.1152747933685886,0.057039850974622,0.0450673652964732,Nature,21 -bitbrains_fast_storage/5T/short,TimeCopilot,1755468.383227564,1755468.383227564,157.09262135237188,0.6559882303327758,1.9376995972529845,0.6411501834555586,13.081417517313175,1324.940898013026,4.159986740605117,0.4932319795190913,0.3847375246720173,Web/CloudOps,2 -bitbrains_fast_storage/H/short,TimeCopilot,3388387.3500898136,3388387.3500898136,343.59436846359625,1.0343822914762917,4.029740171712447,0.5373171666718571,18.667532673515375,1840.75727625611,5.246756009508833,0.979355529826488,0.7219866683522643,Web/CloudOps,2 -bitbrains_rnd/5T/short,TimeCopilot,1680850.729530806,1680850.729530806,120.8159167208182,1.6160098499977276,0.8830180564826331,0.5854488885988877,52.40390428015042,1296.476274187386,5.3048066367348,0.4943438531071508,0.3964612062226265,Web/CloudOps,2 -bitbrains_rnd/H/short,TimeCopilot,1914024.875839788,1914024.875839788,151.02454572911958,5.835419338595843,1.3730293584041908,0.5253995623722053,200.2494106028372,1383.4828787664082,6.054797954393498,0.6609573017334569,0.6144106009404606,Web/CloudOps,2 -bizitobs_application/10S/short,TimeCopilot,346574.9770875603,346574.9770875603,318.26204059176973,1.0735370691318118,0.0148554065634032,0.0150235622547423,6.781292090509864,588.7061891024761,0.022704044587794,0.0122740947758917,0.0096656862275339,Web/CloudOps,2 -bizitobs_service/10S/short,TimeCopilot,2708.533394533042,2708.533394533042,19.35327236487016,0.7723213505738389,0.0567313334571394,0.0462849090866936,6.33391890489669,52.043572076991815,0.0385555431192742,0.0143375232979576,0.0111084990569613,Web/CloudOps,2 -bizitobs_l2c/5T/short,TimeCopilot,20.460949984401623,20.460949984401623,2.6485812178679877,0.2761542839387261,0.153747881954706,0.2055201076943256,2.732507818293984,4.523378160667272,0.1553511152903504,0.0909629996692935,0.0734726620542566,Web/CloudOps,7 -bizitobs_l2c/H/short,TimeCopilot,54.67622858513749,54.67622858513749,4.46167671522643,0.4445604087944049,0.3855683845484163,0.5798314149388096,2.6922606810147367,7.394337602864605,0.3985737340202903,0.2404958015022543,0.1864666623644873,Web/CloudOps,7 -electricity/15T/medium,TimeCopilot,236515.55416131,236515.55416131,51.66783375652916,0.8084229413476077,0.1208535879629089,0.1243342652554423,6.957830953526775,486.3286483041175,0.8379932505621451,0.0890288822386279,0.0717043028530855,Energy,1 -electricity/15T/long,TimeCopilot,311473.5633320522,311473.5633320522,56.974143114661246,0.8577928201229896,0.125683278926977,0.1276541540267709,7.0874667507914415,558.0981663937378,0.880468300267221,0.0898836978295258,0.0719803551543305,Energy,1 -electricity/H/medium,TimeCopilot,5821732.860188879,5821732.860188879,239.53363844340507,1.0561643779123162,0.2251346440304366,0.1230119898471822,8.891307587479334,2412.8267364626245,0.9418057467957052,0.0934978686317197,0.0749768199070903,Energy,1 -electricity/H/long,TimeCopilot,9190427.475796772,9190427.475796772,281.45836713785,1.1933290428598704,0.2900489822659057,0.1368732590852085,9.973042839370002,3031.571783051949,1.195651031569732,0.1110070983288789,0.0887727604544426,Energy,1 -solar/10T/medium,TimeCopilot,21.59732207933846,21.59732207933846,1.9032655957504807,0.8322017990009988,3.910097387929556,1.4427682106320747,4.649474245041393,4.647291908126545,1.0312525303760354,0.4223421942062973,0.3240567774003175,Energy,1 -solar/10T/long,TimeCopilot,19.93343756384208,19.93343756384208,1.833922522548152,0.8015567173366982,3.502350142788309,1.440766538713434,4.951801307452327,4.464687846181643,0.9649708528520436,0.396373014557189,0.3063514234442692,Energy,1 -solar/H/medium,TimeCopilot,678.1193236707683,678.1193236707683,11.308250251485232,0.8639561278695133,3.9021283354365295,1.2853552873341596,4.019540582454916,26.040724330762544,0.9380476697421062,0.4073495676437541,0.3009266290528963,Energy,1 -solar/H/long,TimeCopilot,668.2065897135462,668.2065897135462,11.526002033222165,0.8776931978366276,4.496302288547843,1.2775662209728849,4.38242465661728,25.849692255683557,0.896255940229991,0.3996274960336139,0.2999546072251047,Energy,1 -kdd_cup_2018/H/medium,TimeCopilot,5124.208760826031,5124.208760826031,24.46011936010273,0.996231967613259,0.9014741970833042,0.5319989986464125,8.449002143365693,71.58357884896529,1.498373936307538,0.5119945931381596,0.4063545663479109,Nature,1 -kdd_cup_2018/H/long,TimeCopilot,4196.6286474372,4196.6286474372,23.61998950106404,0.9830015005517432,0.8852672465451198,0.5865132418455189,7.352097596586821,64.78139121257894,1.520033179956784,0.554220387673254,0.4339078339310311,Nature,1 -loop_seattle/5T/medium,TimeCopilot,86.35745061394746,86.35745061394746,5.465495242610135,0.8633634276715643,0.1783955716004149,0.1237903432363743,6.096880241946944,9.292870956488498,0.1653392719168335,0.0972423924005013,0.0761478343100576,Transport,1 -loop_seattle/5T/long,TimeCopilot,102.67591405819309,102.67591405819309,5.939598011951224,0.9355166824810184,0.190080530939235,0.1324900580563883,6.473930465258974,10.13291241737503,0.1791519647429992,0.1050133080988704,0.0821922278459704,Transport,1 -loop_seattle/H/medium,TimeCopilot,67.96142113175956,67.96142113175956,4.428790436596151,0.9078164858292448,0.1145776272598142,0.1012648418842439,7.312828659656807,8.24387173188421,0.145709506064622,0.0782783730712721,0.0629617122630184,Transport,1 -loop_seattle/H/long,TimeCopilot,59.455076613020445,59.455076613020445,4.153927649542212,0.8683485306773391,0.1206798146712537,0.0963736048354253,6.951896375187444,7.71071180975015,0.1378007458629442,0.0742362498419159,0.0594142595772402,Transport,1 -sz_taxi/15T/medium,TimeCopilot,16.262880556044415,16.262880556044415,2.740155295511712,0.5342110293525478,9448683242235.854,0.403277543303855,3.787384124871976,4.032726194033562,0.3753124846182396,0.2550171875590937,0.2010435449898101,Transport,1 -sz_taxi/15T/long,TimeCopilot,15.428155577018597,15.428155577018597,2.683348678885309,0.5062548609697384,6247949985489.982,0.3972602670482759,3.730605634571505,3.927869088579531,0.363556566058157,0.2483659738224705,0.1966549636231346,Transport,1 -m_dense/H/medium,TimeCopilot,35817.04093653174,35817.04093653174,78.26090722870495,0.6797904579599956,0.2697071143941927,0.1891598612604073,7.538636089795069,189.2539060007263,0.329277886750712,0.1361640913628625,0.1130111406014334,Transport,1 -m_dense/H/long,TimeCopilot,33686.05392581881,33686.05392581881,77.89664007019358,0.6769353594248038,0.2710686322244779,0.1896201891342978,7.062686741625563,183.53760902283437,0.3177307330107253,0.1348505991788093,0.1120602970919229,Transport,1 -ett1/15T/medium,TimeCopilot,9.981713481655284,9.981713481655284,1.582522697512334,0.9915852904641798,0.7507864977623967,0.3841926425385567,6.81333905724897,3.159384984717008,0.6021819019056779,0.3016303908535034,0.2359594548502602,Energy,7 -ett1/15T/long,TimeCopilot,9.145720823685098,9.145720823685098,1.568936365631091,0.9985537659401948,0.7849079115673763,0.3855993666194035,7.220664056698961,3.0241892837064777,0.576413467619318,0.299040822563553,0.2346078852235413,Energy,7 -ett1/H/medium,TimeCopilot,141.60279704337782,141.60279704337782,6.772101792756335,1.2451768949683812,2649496750856.2017,0.4314962852014963,8.584838749920038,11.899697350915185,0.5699672949757607,0.3243676226623505,0.2538509370996076,Energy,7 -ett1/H/long,TimeCopilot,144.7921488308064,144.7921488308064,6.903936538522541,1.3330098256563383,3183558657266.194,0.4626126295964976,9.71693506556066,12.032960933652465,0.576232437528607,0.3306145679414334,0.2618494542142582,Energy,7 -ett2/15T/medium,TimeCopilot,11.290997577809248,11.290997577809248,2.087360550588558,0.8600154757133787,0.1292444814411573,0.1556607743146997,6.5852381900381705,3.360207966452262,0.1790715649497639,0.1112392221374743,0.0880873921819831,Energy,7 -ett2/15T/long,TimeCopilot,10.98179065324542,10.98179065324542,2.103303477570178,0.8690296184660402,0.1322920322874429,0.1584818980965289,7.153389096823767,3.3138784910200645,0.1766025833415498,0.1120888495751155,0.0907130832548878,Energy,7 -ett2/H/medium,TimeCopilot,255.57112842633688,255.57112842633688,9.803149774111809,1.040973538853912,0.1719793888128407,0.1686040900806265,7.614867239781896,15.986592145492953,0.2154296097532225,0.1321037473758718,0.1067271107480788,Energy,7 -ett2/H/long,TimeCopilot,231.46263653003027,231.46263653003027,9.342868735613646,1.0256298120569345,0.1774689796262005,0.1746506497693186,7.934005849875703,15.213896165349304,0.2083425985051882,0.1279433965313867,0.1053284863541385,Energy,7 -jena_weather/10T/medium,TimeCopilot,1484.1858446263088,1484.1858446263088,9.668029277152618,0.6051370744394549,0.7386146189499935,0.6360915704415862,5.695157703303464,38.52513263606381,0.2364280803877654,0.0593325303957591,0.0482467353304328,Nature,21 -jena_weather/10T/long,TimeCopilot,1460.857589198122,1460.857589198122,9.831072311026984,0.6259346894044211,0.7114689440887886,0.6277360018598593,6.234256427084278,38.221166769188535,0.2338818733002198,0.0601580172195703,0.0490729331929777,Nature,21 -jena_weather/H/medium,TimeCopilot,1232.0504053622226,1232.0504053622226,9.862688004942656,0.7861309202407549,1.975104371877037,0.5916587777230587,6.066633010404215,35.10057557024133,0.2147866471100061,0.0603515370804828,0.0485601206880911,Nature,21 -jena_weather/H/long,TimeCopilot,1116.5053196232564,1116.5053196232564,10.608046140454295,0.9661327401085368,2.0508442810132834,0.5853766316627341,7.231680579914377,33.414148494661006,0.2011901943307702,0.0638721906921791,0.0528682099485974,Nature,21 -bitbrains_fast_storage/5T/medium,TimeCopilot,3279629.4524840857,3279629.4524840857,291.8520878303471,0.9536450858635028,4.81938886008226,0.7497755624722655,20.74132918296856,1810.9747244188927,5.5022414152740335,0.8867272541915866,0.6180918878491857,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,TimeCopilot,4189381.479679128,4189381.479679128,371.41306038680415,0.8749791633710378,5.595117742899964,0.7750577286889536,15.438412570532794,2046.79785999476,5.409075720451288,0.981533841940754,0.6757619889754686,Web/CloudOps,2 -bitbrains_rnd/5T/medium,TimeCopilot,2317643.576247831,2317643.576247831,151.99639333909803,4.357324961139689,0.6313503620150362,0.6743120630571986,159.8787002978397,1522.38089066036,6.405432768165085,0.6395263396368895,0.6109418602731045,Web/CloudOps,2 -bitbrains_rnd/5T/long,TimeCopilot,2866197.0630570524,2866197.0630570524,196.1143766018546,3.282503295928003,2.203904915328427,0.6544766751822441,115.90692691094009,1692.9846611995788,6.485442812511457,0.7512699927602743,0.6308730484232111,Web/CloudOps,2 -bizitobs_application/10S/medium,TimeCopilot,2943142.7222554926,2943142.7222554926,954.1212809801102,2.0792345306733995,0.0291460284196298,0.0298180574147895,15.10135770327809,1715.5590115922837,0.0673649531933652,0.0374655345573703,0.0306248530840539,Web/CloudOps,2 -bizitobs_application/10S/long,TimeCopilot,6323794.845872988,6323794.845872988,1528.558036365509,3.2227093329644263,0.0466780897417334,0.047776901029403,49.72871421310417,2514.7156590503405,0.0969825313638149,0.0589503736415321,0.0532368092817842,Web/CloudOps,2 -bizitobs_service/10S/medium,TimeCopilot,18632.675050976544,18632.675050976544,37.04852859742466,1.013201423904092,0.0657354188418835,0.0586519529335255,10.06382721313094,136.50155695440452,0.1027342884380532,0.0278835956750636,0.0213210606829475,Web/CloudOps,2 -bizitobs_service/10S/long,TimeCopilot,131285.69489545247,131285.69489545247,76.69929972157539,1.3354987722201337,0.0799915856988493,0.0770978551825089,20.746346496920047,362.3336789417352,0.2684283728765971,0.0568212932487228,0.0524026559916625,Web/CloudOps,2 -bizitobs_l2c/5T/medium,TimeCopilot,143.0089914421698,143.0089914421698,8.102515888612826,0.7954148218483699,0.5639831367880619,0.8651895706464133,5.559167473960077,11.95863668827554,0.6292001491717296,0.4263114883973163,0.3314397617783311,Web/CloudOps,7 -bizitobs_l2c/5T/long,TimeCopilot,292.6456219502893,292.6456219502893,11.65753647049682,1.0958397882129156,0.8664108339345789,1.117734287778356,6.675549714650306,17.10688814338509,0.9605261944073564,0.6545532447700658,0.508043300889487,Web/CloudOps,7 -bizitobs_l2c/H/medium,TimeCopilot,71.80546363602481,71.80546363602481,4.851299580712137,0.4886258222798499,0.4706340605166543,0.7595045998282356,3.370014704709077,8.47381045551674,0.5131020940604996,0.2937535583130325,0.2319735359977425,Web/CloudOps,7 -bizitobs_l2c/H/long,TimeCopilot,83.7434645241067,83.7434645241067,5.339594684263307,0.566927678162445,0.6195787550312101,0.7829352069783353,4.582971253588229,9.151145530703063,0.5589783808478149,0.3261578543340893,0.2610668053551979,Web/CloudOps,7 \ No newline at end of file diff --git a/results/TimeCopilot/config.json b/results/TimeCopilot/config.json deleted file mode 100644 index a031c515ede086e8d537d5bcffc25406e1126b3a..0000000000000000000000000000000000000000 --- a/results/TimeCopilot/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "model": "TimeCopilot", - "model_type": "agentic", - "model_dtype": "float32", - "model_link": "https://github.com/AzulGarza/TimeCopilot", - "code_link": "https://github.com/AzulGarza/timecopilot/tree/main/experiments/gift-eval", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/Timer/all_results.csv b/results/Timer/all_results.csv new file mode 100644 index 0000000000000000000000000000000000000000..a4eab49f6317b0b93ddb8d3ed28043448ead9a90 --- /dev/null +++ b/results/Timer/all_results.csv @@ -0,0 +1,98 @@ +dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates +m4_yearly/A/short,Timer,15960074.0,15960074.0,2515.56494140625,11.507473945617676,0.3822980523109436,0.512023389339447,460.2989501953125,3995.006103515625,0.6406177282333374,0.4033825099468231,0.4033825099468231,Econ/Fin,1.0 +m4_quarterly/Q/short,Timer,4070192.5,4070192.5,1059.001708984375,2.9413843154907227,0.1890403479337692,0.2066598832607269,117.6553726196289,2017.4718017578125,0.3376892507076263,0.1772582232952118,0.1772582232952118,Econ/Fin,1.0 +m4_monthly/M/short,Timer,3139851.5,3139851.5,773.2167358398438,1.5437763929367063,0.1861013025045395,0.1874160319566726,61.75105667114258,1771.962646484375,0.3682746589183807,0.1607009768486023,0.1607009768486023,Econ/Fin,1.0 +m4_weekly/W/short,Timer,602500.5625,602500.5625,435.4611511230469,3.867011308670044,0.1095710545778274,0.1155303865671157,154.68045043945312,776.2091064453125,0.1414132714271545,0.0793342739343643,0.0793342739343643,Econ/Fin,1.0 +m4_daily/D/short,Timer,409574.21875,409574.21875,216.2593231201172,4.353240966796875,0.0476609021425247,0.0409090295433998,174.129638671875,639.9798583984375,0.0988592058420181,0.0334060862660408,0.0334060862660408,Econ/Fin,1.0 +m4_hourly/H/short,Timer,3704264.25,3704264.25,325.61083984375,3.465131998062134,0.1837441772222519,0.1398412883281707,138.6052703857422,1924.6466064453125,0.262756735086441,0.04445306584239,0.0444530695676803,Econ/Fin,1.0 +electricity/15T/short,Timer,439636.90625,439636.90625,119.53121948242188,2.256010293960572,0.3652159869670868,0.3091564476490021,90.24040985107422,663.0512084960938,1.307004690170288,0.2356195896863937,0.2356196045875549,Energy,1.0 +electricity/H/short,Timer,2290321.25,2290321.25,240.87733459472656,1.3026354312896729,0.3624715209007263,0.1691998392343521,52.10541915893555,1513.3807373046875,0.7144237756729126,0.113711304962635,0.113711304962635,Energy,1.0 +electricity/D/short,Timer,2299531264.0,2299531264.0,5843.25244140625,1.8616673946380613,0.7036624550819397,0.1342314183712005,74.46669006347656,47953.42578125,0.7880340814590454,0.0960240438580513,0.0960240438580513,Energy,1.0 +electricity/W/short,Timer,135113850880.0,135113850880.0,49654.5703125,2.278900384902954,0.2822481393814087,0.132920354604721,91.15601348876952,367578.375,0.8386925458908081,0.1132953464984893,0.1132953464984893,Energy,1.0 +solar/10T/short,Timer,42.372314453125,42.372314453125,3.8650217056274414,1.700700283050537,4.472381591796875,1.6286958456039429,68.02801513671875,6.509401798248291,1.893584370613098,1.124334454536438,1.124334454536438,Energy,1.0 +solar/H/short,Timer,761.1024780273438,761.1024780273438,14.604989051818848,1.12728750705719,5.338653087615967,1.4358646869659424,45.09149932861328,27.588085174560547,1.0169060230255127,0.5383447408676147,0.5383447408676147,Energy,1.0 +solar/D/short,Timer,118298.1796875,118298.1796875,260.9843444824219,1.019014596939087,1.0004303455352783,0.4478209018707275,40.76058578491211,343.94500732421875,0.4968833923339844,0.3770334720611572,0.3770335018634796,Energy,1.0 +solar/W/short,Timer,1952498.25,1952498.25,1030.47265625,1.1244401931762695,0.2236430644989013,0.1972776353359222,44.97760391235352,1397.3182373046875,0.2852275967597961,0.2103452235460281,0.2103452384471893,Energy,1.0 +hospital/M/short,Timer,31698.765625,31698.765625,51.45354080200195,1.4601761102676392,0.2785269916057586,0.3147836923599243,58.40704727172852,178.04147338867188,0.6463319063186646,0.1867882907390594,0.1867883056402206,Healthcare,1.0 +covid_deaths/D/short,Timer,19473190.0,19473190.0,766.7901000976562,82.29279327392578,0.2722060084342956,0.6154463291168213,3291.711669921875,4412.84375,1.6603540182113647,0.2885085046291351,0.2885085046291351,Healthcare,1.0 +us_births/D/short,Timer,1422180.125,1422180.125,1070.29443359375,1.576511263847351,0.1054572388529777,0.1028548330068588,63.06045150756836,1192.552001953125,0.1117920652031898,0.1003314107656478,0.1003314033150672,Healthcare,1.0 +us_births/M/short,Timer,320133216.0,320133216.0,14097.0830078125,1.5948479175567627,0.0438226461410522,0.0438417382538318,63.79391479492188,17892.267578125,0.0555732585489749,0.0437854453921318,0.0437854453921318,Healthcare,1.0 +us_births/W/short,Timer,6209666.5,6209666.5,1802.1334228515625,1.6382945775985718,0.0239783357828855,0.0244099199771881,65.53178405761719,2491.920166015625,0.0338275730609893,0.0244637839496135,0.0244637820869684,Healthcare,1.0 +saugeen/D/short,Timer,1427.042724609375,1427.042724609375,20.21636962890625,4.487884044647217,0.8190188407897949,0.565277636051178,179.51536560058594,37.77621841430664,1.2235807180404663,0.6548130512237549,0.6548130512237549,Nature,1.0 +saugeen/M/short,Timer,398.7861328125,398.7861328125,14.635396003723145,0.8574973940849304,0.5644688606262207,0.4876649081707001,34.299896240234375,19.96963119506836,0.5995482206344604,0.4393984973430633,0.4393985271453857,Nature,1.0 +saugeen/W/short,Timer,1276.0330810546875,1276.0330810546875,19.29665756225586,1.5652990341186523,0.6592676043510437,0.5892956256866455,62.611961364746094,35.72160720825195,1.0782864093780518,0.5824856162071228,0.5824856162071228,Nature,1.0 +temperature_rain/D/short,Timer,184.2439117431641,184.2439117431641,6.942285537719727,1.681937575340271,93.59574127197266,1.425851583480835,67.27750396728516,13.573647499084473,1.5979772806167605,0.8172905445098877,0.8172906041145325,Nature,1.0 +kdd_cup_2018/H/short,Timer,4599.66259765625,4599.66259765625,23.67636489868164,1.0080336332321167,1.0133415460586548,0.5278124213218689,40.32134628295898,67.82080841064453,1.4196124076843262,0.4955891966819763,0.4955892264842987,Nature,1.0 +kdd_cup_2018/D/short,Timer,2857.045654296875,2857.045654296875,21.094287872314453,1.2058345079421997,0.520703136920929,0.473745733499527,48.23337936401367,53.45133972167969,1.197366714477539,0.4725344777107239,0.4725344777107239,Nature,1.0 +car_parts/M/short,Timer,1.33407723903656,1.33407723903656,0.4690718650817871,0.8716527223587036,0.8576087355613708,1.90359365940094,34.86610794067383,1.1550226211547852,2.769625663757324,1.124786138534546,1.124786138534546,Sales,1.0 +restaurant/D/short,Timer,204.9752197265625,204.9752197265625,8.964832305908203,0.8364865183830261,0.8127961754798889,0.4661175012588501,33.4594612121582,14.31695556640625,0.6588994860649109,0.4125823378562927,0.4125823676586151,Sales,1.0 +hierarchical_sales/D/short,Timer,30.41873359680176,30.41873359680176,2.7062768936157227,0.8625249266624451,0.8648467659950256,0.9972294569015504,34.50099563598633,5.515317916870117,1.6920385360717771,0.8302557468414307,0.8302558064460754,Sales,1.0 +hierarchical_sales/W/short,Timer,478.2153625488281,478.2153625488281,10.28359031677246,0.8074617385864258,0.7302916646003723,0.5080252885818481,32.29846954345703,21.868135452270508,1.003976821899414,0.4721246361732483,0.472124695777893,Sales,1.0 +loop_seattle/5T/short,Timer,72.92700958251953,72.92700958251953,5.170968532562256,0.8028478026390076,0.1457836925983429,0.1057910025119781,32.11391067504883,8.5397310256958,0.1464375406503677,0.0886706933379173,0.0886706933379173,Transport,1.0 +loop_seattle/H/short,Timer,88.27337646484375,88.27337646484375,5.952459335327148,1.1994346380233765,0.1736440658569336,0.1311271786689758,47.977386474609375,9.395391464233398,0.1663079857826233,0.1053645834326744,0.1053645834326744,Transport,1.0 +loop_seattle/D/short,Timer,36.45976257324219,36.45976257324219,4.904968738555908,1.5016674995422363,0.0917601883411407,0.0887525826692581,60.06670379638672,6.038191795349121,0.1079105660319328,0.0876583456993103,0.0876583456993103,Transport,1.0 +sz_taxi/15T/short,Timer,18.27078628540039,18.27078628540039,2.948929071426392,0.5914117693901062,1240829919232.0,0.4183946549892425,23.656471252441406,4.2744340896606445,0.3996778726577759,0.2757374942302704,0.2757374942302704,Transport,1.0 +sz_taxi/H/short,Timer,7.8406662940979,7.8406662940979,2.0194859504699707,0.6149353384971619,1.5491769313812256,0.3229924738407135,24.59741401672364,2.800118923187256,0.2608483731746673,0.1881275773048401,0.1881275624036789,Transport,1.0 +m_dense/H/short,Timer,76534.609375,76534.609375,139.14401245117188,1.143413782119751,0.4861980676651001,0.3362470269203186,45.736549377441406,276.6488952636719,0.490706205368042,0.246806800365448,0.246806800365448,Transport,1.0 +m_dense/D/short,Timer,19875.603515625,19875.603515625,87.71620178222656,1.1955851316452026,0.206885740160942,0.175531655550003,47.82340240478516,140.98086547851562,0.2440587282180786,0.1518497169017791,0.1518497318029403,Transport,1.0 +ett1/15T/short,Timer,16.142446517944336,16.142446517944336,2.2090744972229004,1.3419594764709473,0.6826388239860535,0.4235529601573944,53.67837905883789,4.01776647567749,0.7595250606536865,0.4176070392131805,0.4176070094108581,Energy,7.0 +ett1/H/short,Timer,117.29533386230467,117.29533386230467,5.761704444885254,0.9536104202270508,0.5801722407341003,0.3092675805091858,38.14441680908203,10.830296516418455,0.505494236946106,0.2689222991466522,0.2689222991466522,Energy,7.0 +ett1/D/short,Timer,39235.36328125,39235.36328125,131.30612182617188,1.6968214511871338,0.8816156387329102,0.5224992036819458,67.87285614013672,198.0791931152344,0.5202615857124329,0.3448798954486847,0.3448798656463623,Energy,7.0 +ett1/W/short,Timer,2657829.5,2657829.5,1305.752197265625,2.5689876079559326,0.7078898549079895,0.8641673922538757,102.75950622558594,1630.2850341796875,0.6487293243408203,0.5195899605751038,0.5195900201797485,Energy,7.0 +ett2/15T/short,Timer,11.666239738464355,11.666239738464355,2.246156930923462,1.0045720338821411,0.1447132229804992,0.1650290340185165,40.182884216308594,3.415587902069092,0.1621500998735427,0.1066330522298812,0.1066330447793006,Energy,7.0 +ett2/H/short,Timer,142.5553436279297,142.5553436279297,7.492897033691406,0.8407788276672363,0.1294213384389877,0.1227191388607025,33.63115310668945,11.939654350280762,0.1514280438423156,0.0950307846069336,0.0950307846069336,Energy,7.0 +ett2/D/short,Timer,129063.671875,129063.671875,235.2665405273437,1.7539362907409668,0.5099589824676514,0.1784910410642624,70.15745544433594,359.2543334960937,0.2056107223033905,0.134649246931076,0.134649246931076,Energy,7.0 +ett2/W/short,Timer,15428875.0,15428875.0,2803.319091796875,1.975763440132141,0.2883638441562652,0.346852570772171,79.0305404663086,3927.960693359375,0.3294956982135772,0.2351555079221725,0.2351554930210113,Energy,7.0 +jena_weather/10T/short,Timer,1421.3758544921875,1421.3758544921875,10.759843826293944,0.4265126883983612,0.5133702158927917,0.6040313243865967,17.06050682067871,37.70113754272461,0.2341713309288025,0.0668321177363395,0.0668321177363395,Nature,21.0 +jena_weather/H/short,Timer,1203.45263671875,1203.45263671875,10.736539840698242,0.626010537147522,2.478585958480835,0.6297770738601685,25.04042053222656,34.69081497192383,0.2126524597406387,0.0658142939209938,0.0658142939209938,Nature,21.0 +jena_weather/D/short,Timer,513.686279296875,513.686279296875,12.664178848266602,1.7647336721420288,1.22536039352417,0.4777471721172333,70.58934783935547,22.66464805603028,0.1364782303571701,0.0762590542435646,0.0762590542435646,Nature,21.0 +bitbrains_fast_storage/5T/short,Timer,2012037.125,2012037.125,222.65597534179688,1.0162328481674194,3.530824899673462,0.7916674017906189,40.64931488037109,1418.462890625,4.453622817993164,0.6990846991539001,0.6990846991539001,Web/CloudOps,2.0 +bitbrains_fast_storage/H/short,Timer,3378123.0,3378123.0,387.0008544921875,1.4674376249313354,6.137554168701172,0.6591413617134094,58.69750595092773,1837.967041015625,5.238802909851074,1.1030781269073486,1.1030781269073486,Web/CloudOps,2.0 +bitbrains_rnd/5T/short,Timer,1855462.875,1855462.875,173.45669555664062,2.103407382965088,2.341862916946411,0.7406865358352661,84.13629913330078,1362.15380859375,5.573539733886719,0.7097347378730774,0.7097347378730774,Web/CloudOps,2.0 +bitbrains_rnd/H/short,Timer,2164091.0,2164091.0,238.7935943603516,6.159127712249756,4.8008222579956055,0.6970577239990234,246.36509704589844,1471.0849609375,6.438187599182129,1.045077562332153,1.0450776815414429,Web/CloudOps,2.0 +bizitobs_application/10S/short,Timer,5609835.5,5609835.5,1088.42431640625,3.750981569290161,0.0530213117599487,0.053666565567255,150.0392608642578,2368.509033203125,0.0913439318537712,0.0419761762022972,0.0419761724770069,Web/CloudOps,2.0 +bizitobs_service/10S/short,Timer,24416.603515625,24416.603515625,51.10384750366211,2.169052124023437,0.3454442322254181,0.1400158554315567,86.7620849609375,156.2581329345703,0.1157610192894935,0.0378593653440475,0.0378593653440475,Web/CloudOps,2.0 +bizitobs_l2c/5T/short,Timer,27.20807838439941,27.20807838439941,3.4999563694000244,0.3761106431484222,0.1919722855091095,0.2377228885889053,15.044425964355469,5.216136455535889,0.1791432350873947,0.1202026680111885,0.1202026605606079,Web/CloudOps,7.0 +bizitobs_l2c/H/short,Timer,174.7396697998047,174.7396697998047,8.77177906036377,0.8403541445732117,0.9407946467399596,0.7788866758346558,33.614166259765625,13.218914031982422,0.7125332951545715,0.4728213548660278,0.4728213548660278,Web/CloudOps,7.0 +electricity/15T/medium,Timer,2507608.75,2507608.75,187.1243438720703,2.7405874729156494,0.4590089321136474,0.3416807055473327,109.62349700927734,1583.5430908203125,2.728604316711426,0.3224340975284576,0.3224340975284576,Energy,1.0 +electricity/H/medium,Timer,12729977.0,12729977.0,378.74114990234375,1.6499006748199463,0.3859025835990906,0.1930912882089615,65.99602508544922,3567.909423828125,1.3926725387573242,0.1478351503610611,0.1478351503610611,Energy,1.0 +solar/10T/medium,Timer,82.85549926757812,82.85549926757812,6.211141109466553,2.7358651161193848,3.19260573387146,1.6824835538864136,109.43460845947266,9.102499961853027,2.019880771636963,1.3782769441604614,1.3782769441604614,Energy,1.0 +solar/H/medium,Timer,894.169189453125,894.169189453125,16.82102394104004,1.2898763418197632,6.022534370422363,1.455984592437744,51.59505081176758,29.90266227722168,1.0771636962890625,0.6059325337409973,0.6059325933456421,Energy,1.0 +kdd_cup_2018/H/medium,Timer,4972.2255859375,4972.2255859375,25.70556831359864,1.1031593084335327,1.0825453996658323,0.5811611413955688,44.126373291015625,70.51400756835938,1.4759858846664429,0.5380641222000122,0.5380640625953674,Nature,1.0 +loop_seattle/5T/medium,Timer,175.62289428710938,175.62289428710938,8.868509292602539,1.3719229698181152,0.3077051937580108,0.1890646368265152,54.876914978027344,13.252278327941896,0.2357852756977081,0.1577890068292617,0.1577890068292617,Transport,1.0 +loop_seattle/H/medium,Timer,115.57488250732422,115.57488250732422,7.100574016571045,1.4409586191177368,0.1956809163093567,0.1531565189361572,57.63834381103516,10.75057601928711,0.1900152266025543,0.1255018562078476,0.1255018562078476,Transport,1.0 +sz_taxi/15T/medium,Timer,18.135269165039062,18.135269165039062,2.9966444969177246,0.5864758491516113,11960251842560.0,0.4295547008514404,23.459033966064453,4.258552551269531,0.396329402923584,0.2788877785205841,0.2788877785205841,Transport,1.0 +m_dense/H/medium,Timer,98668.859375,98668.859375,162.61505126953125,1.2639591693878174,0.6239790916442871,0.394538938999176,50.55836868286133,314.1159973144531,0.5465221405029297,0.2829296588897705,0.2829296588897705,Transport,1.0 +ett1/15T/medium,Timer,29.358871459960938,29.358871459960938,3.064739227294922,1.7557613849639893,0.910193681716919,0.6591196060180664,70.23045349121094,5.41838264465332,1.032749056816101,0.5841423273086548,0.5841423273086548,Energy,7.0 +ett1/H/medium,Timer,237.18679809570312,237.18679809570312,8.913472175598145,1.553605079650879,7674470072320.0,0.542371928691864,62.14420318603516,15.400870323181152,0.7376651763916016,0.4269341826438904,0.4269341826438904,Energy,7.0 +ett2/15T/medium,Timer,17.68214225769043,17.68214225769043,2.716909646987915,1.1290565729141235,0.1674913018941879,0.1970102041959762,45.16226196289063,4.205013751983643,0.2240927964448928,0.1447890400886535,0.1447890549898147,Energy,7.0 +ett2/H/medium,Timer,274.37237548828125,274.37237548828125,10.453105926513672,1.1246135234832764,0.1859084665775299,0.1857642829418182,44.98453903198242,16.564189910888672,0.2232131063938141,0.1408623307943344,0.1408623307943344,Energy,7.0 +jena_weather/10T/medium,Timer,2489.984375,2489.984375,16.714487075805664,0.7859023213386536,2.6763811111450195,0.6979338526725769,31.436092376708984,49.89974212646485,0.3062338829040527,0.1025765240192413,0.1025765165686607,Nature,21.0 +jena_weather/H/medium,Timer,1809.32958984375,1809.32958984375,13.810235023498535,0.936582088470459,5.170769691467285,0.6923579573631287,37.46328353881836,42.53621292114258,0.2602866291999817,0.0845072790980339,0.0845072865486145,Nature,21.0 +bitbrains_fast_storage/5T/medium,Timer,3149185.25,3149185.25,322.8526916503906,1.3148092031478882,5.947714805603027,0.8331922292709351,52.59236907958984,1774.594482421875,5.391707897186279,0.9809156060218812,0.9809156060218812,Web/CloudOps,2.0 +bitbrains_rnd/5T/medium,Timer,2225999.25,2225999.25,216.0227203369141,5.005271911621094,3.1543328762054443,0.7927706837654114,200.2108764648437,1491.978271484375,6.27751350402832,0.9089177846908568,0.9089177846908568,Web/CloudOps,2.0 +bizitobs_application/10S/medium,Timer,15773824.0,15773824.0,2290.66552734375,7.47352123260498,0.1042062267661094,0.1052982732653617,298.9408569335937,3971.62744140625,0.1559541225433349,0.0899477005004882,0.0899476930499076,Web/CloudOps,2.0 +bizitobs_service/10S/medium,Timer,128696.2578125,128696.2578125,119.21471405029295,4.256953716278076,0.7849547266960144,0.2061486095190048,170.2781524658203,358.7426147460937,0.2699981331825256,0.0897238031029701,0.0897238031029701,Web/CloudOps,2.0 +bizitobs_l2c/5T/medium,Timer,270.9913330078125,270.9913330078125,11.936371803283691,1.1333492994308472,0.6806544661521912,0.9873334765434264,45.3339729309082,16.461814880371094,0.8661335110664368,0.6280286908149719,0.6280286908149719,Web/CloudOps,7.0 +bizitobs_l2c/H/medium,Timer,196.0673217773437,196.0673217773437,10.511180877685549,1.037947177886963,1.453730583190918,0.910856544971466,41.51788711547852,14.00240421295166,0.8478668332099915,0.6364679336547852,0.6364679336547852,Web/CloudOps,7.0 +electricity/15T/long,Timer,4811026.0,4811026.0,222.29676818847656,2.8717730045318604,0.4768320024013519,0.3436427712440491,114.87092590332033,2193.405029296875,3.460365533828736,0.3507004082202911,0.3507004082202911,Energy,1.0 +electricity/H/long,Timer,18253970.0,18253970.0,440.7637023925781,1.9044365882873533,0.4604615569114685,0.2154321521520614,76.1774673461914,4272.46630859375,1.6850595474243164,0.1738370805978775,0.1738370805978775,Energy,1.0 +solar/10T/long,Timer,85.07907104492188,85.07907104492188,6.364139556884766,2.790985107421875,3.2258267402648926,1.6921824216842651,111.639404296875,9.223832130432127,1.9935836791992188,1.3755069971084597,1.37550687789917,Energy,1.0 +solar/H/long,Timer,937.348876953125,937.348876953125,17.726049423217773,1.3549171686172483,8.945707321166992,1.454779028892517,54.19668960571289,30.616153717041016,1.0615178346633911,0.6145944595336914,0.6145943999290466,Energy,1.0 +kdd_cup_2018/H/long,Timer,3933.24560546875,3933.24560546875,24.26762962341309,1.0618218183517456,1.1776909828186035,0.6198624968528748,42.47286987304688,62.71559143066406,1.4715611934661863,0.5694166421890259,0.5694167017936707,Nature,1.0 +loop_seattle/5T/long,Timer,174.40982055664062,174.40982055664062,8.904092788696289,1.3811830282211304,0.312557190656662,0.1870190501213073,55.247318267822266,13.206430435180664,0.2334924042224884,0.1574261784553527,0.1574261933565139,Transport,1.0 +loop_seattle/H/long,Timer,119.13572692871094,119.13572692871094,7.016603946685791,1.4354292154312134,0.2177831083536148,0.1524847447872162,57.41716766357422,10.914931297302246,0.195064440369606,0.1253961026668548,0.1253961026668548,Transport,1.0 +sz_taxi/15T/long,Timer,17.1718692779541,17.1718692779541,2.921497344970703,0.5541780591011047,8475207794688.0,0.4209156036376953,22.16712188720703,4.143895626068115,0.3835515677928924,0.2704086005687713,0.2704086005687713,Transport,1.0 +m_dense/H/long,Timer,132903.703125,132903.703125,189.4039306640625,1.4404672384262085,0.7668570280075073,0.4415754973888397,57.61868667602539,364.5596008300781,0.6311065554618835,0.3278862237930298,0.3278862237930298,Transport,1.0 +ett1/15T/long,Timer,26.45624542236328,26.45624542236328,2.947497844696045,1.726724624633789,0.8878867030143738,0.6636614799499512,69.06898498535156,5.143563270568848,0.9803682565689088,0.5617960095405579,0.5617959499359131,Energy,7.0 +ett1/H/long,Timer,252.66668701171875,252.66668701171875,9.196810722351074,1.6078972816467283,5673646555136.0,0.5512914657592773,64.31588745117188,15.895492553710938,0.7612007260322571,0.4404153525829315,0.4404153823852539,Energy,7.0 +ett2/15T/long,Timer,17.282947540283203,17.282947540283203,2.730286836624145,1.1427444219589231,0.1719768047332763,0.1983316242694854,45.70977783203125,4.157276630401611,0.2215487957000732,0.1455019265413284,0.1455019265413284,Energy,7.0 +ett2/H/long,Timer,318.6417541503906,318.6417541503906,11.12791919708252,1.1976245641708374,0.1980889439582824,0.2000740468502044,47.90498352050781,17.850540161132812,0.2444494068622589,0.1523882895708084,0.1523882895708084,Energy,7.0 +jena_weather/10T/long,Timer,2642.381103515625,2642.381103515625,16.85823631286621,0.738312840461731,1.7886229753494265,0.6904714107513428,29.532514572143555,51.40409851074219,0.3145504891872406,0.1031584367156028,0.1031584441661834,Nature,21.0 +jena_weather/H/long,Timer,1726.2587890625,1726.2587890625,15.776041984558104,1.083956956863403,5.871225833892822,0.6670762300491333,43.3582763671875,41.54827117919922,0.2501666247844696,0.0949892476201057,0.0949892476201057,Nature,21.0 +bitbrains_fast_storage/5T/long,Timer,3541142.5,3541142.5,364.0123901367188,1.2159849405288696,5.811825275421143,0.842163622379303,48.63940048217773,1881.7923583984373,4.973015308380127,0.9619760513305664,0.9619760513305664,Web/CloudOps,2.0 +bitbrains_rnd/5T/long,Timer,2221176.0,2221176.0,217.41079711914065,3.946690082550049,3.3151423931121826,0.7730594873428345,157.8675994873047,1490.3609619140625,5.709237098693848,0.8328518271446228,0.832851767539978,Web/CloudOps,2.0 +bizitobs_application/10S/long,Timer,18921282.0,18921282.0,2421.072021484375,8.513622283935547,0.1152883917093277,0.1179966256022453,340.5448913574219,4349.85986328125,0.1677567064762115,0.093371070921421,0.0933710560202598,Web/CloudOps,2.0 +bizitobs_service/10S/long,Timer,205238.046875,205238.046875,133.7473602294922,4.24716854095459,1.0456585884094238,0.2048772722482681,169.88673400878906,453.0320434570313,0.335620641708374,0.0990843251347541,0.0990843251347541,Web/CloudOps,2.0 +bizitobs_l2c/5T/long,Timer,375.9095153808594,375.9095153808594,14.159292221069336,1.3388009071350098,1.0445107221603394,1.155885934829712,53.55203628540039,19.38838577270508,1.088628888130188,0.7950230836868286,0.7950231432914734,Web/CloudOps,7.0 +bizitobs_l2c/H/long,Timer,194.76385498046875,194.76385498046875,10.17976188659668,1.0375962257385254,1.343863010406494,0.9064521193504332,41.503849029541016,13.955781936645508,0.8524594306945801,0.6218092441558838,0.621809184551239,Web/CloudOps,7.0 diff --git a/results/Timer/config.json b/results/Timer/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6cfff8cd47e0eacd90d3408216df7e12ca548b --- /dev/null +++ b/results/Timer/config.json @@ -0,0 +1,6 @@ +{ + "model": "Timer", + "model_type": "pretrained", + "model_dtype": "float32", + "model_link": "https://drive.google.com/drive/folders/15oaiAl4OO5gFqZMJD2lOtX2fxHbpgcU8" +} \ No newline at end of file diff --git a/results/TimesFM-2.5/all_results.csv b/results/TimesFM-2.5/all_results.csv deleted file mode 100644 index de38d426afd186db61fe5dbe08f54900615119d9..0000000000000000000000000000000000000000 --- a/results/TimesFM-2.5/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,TimesFM-2.5,3968779.822756159,3968779.822756159,953.2987398798642,3.5746010048672225,0.17252063438504306,0.16117622395634967,42.23022041922721,1992.179666284183,0.3194552439262296,0.15286587180709002,0.12711092791527298,Econ/Fin,1 -electricity/W/short,TimesFM-2.5,33954992519.14955,33954992519.14955,25325.864864864863,1.3967853300134194,0.12850979827411782,0.08910185097975712,10.980216234144839,184268.80506246723,0.4204406042476741,0.05778526605883745,0.04606456098528357,Energy,1 -ett1/W/short,TimesFM-2.5,1001010.3571428572,1001010.3571428572,809.7381417410714,1.4441483369953474,0.6339326586042132,0.5109938893999372,8.298726760870942,1000.5050510331555,0.39812483960946055,0.32221413322544384,0.24052818697627457,Energy,7 -ett2/W/short,TimesFM-2.5,3417968.285714286,3417968.285714286,1252.2313058035713,0.8412490672071247,0.13135623931884766,0.14454969338008336,8.118279344541653,1848.7748066528509,0.15508386283292705,0.1050430086809398,0.08693052088780598,Energy,7 -hierarchical_sales/W/short,TimesFM-2.5,469.1105336334746,469.1105336334746,9.0145263671875,0.7191135875727374,0.5282080025007649,0.4589081780385163,6.710618756079831,21.658959661846055,0.9943734322605166,0.4138613148503897,0.34775590836952114,Sales,1 -m4_quarterly/Q/short,TimesFM-2.5,1830040.7706666666,1830040.7706666666,566.2758040364583,1.2049703896311814,0.11188401269912719,0.10345247332255046,9.87322457721087,1352.7899950349524,0.22643322331112486,0.09478459780282542,0.07606786007572805,Econ/Fin,1 -saugeen/W/short,TimesFM-2.5,967.25107421875,967.25107421875,15.383941650390625,1.2478244362544069,0.4173621654510498,0.41068339347839355,12.957193485059104,31.10066035020398,0.9387992936576998,0.4643770708541647,0.37806356455022827,Nature,1 -solar/W/short,TimesFM-2.5,2053658.1605839415,2053658.1605839415,1162.6410812043796,1.2791753235012961,0.2676908395586223,0.2258254837815779,6.8038428993346765,1433.0590220168679,0.2925231746374103,0.23732411213539895,0.17045972695719927,Energy,1 -us_births/W/short,TimesFM-2.5,2417059.4285714286,2417059.4285714286,1195.552455357143,1.0865839544744331,0.016139235879693712,0.01620925111430032,7.075571049667237,1554.6894958709372,0.021104757462262373,0.016229507352263858,0.01275577112614316,Healthcare,1 -car_parts/M/short,TimesFM-2.5,1.339460088862387,1.339460088862387,0.463792010545889,0.8384127237589201,0.778507525928713,1.829889115650894,15.66418845184222,1.1573504606913099,2.7752076832186967,1.1121256653007028,0.9421112990920802,Sales,1 -hospital/M/short,TimesFM-2.5,2819.2294654498046,2819.2294654498046,17.944049462190353,0.7589560550538742,0.19105574215354737,0.17283751725217975,5.046685729393158,53.096416691240144,0.1927523223295072,0.06514106641029434,0.05070281221663658,Healthcare,1 -saugeen/M/short,TimesFM-2.5,392.646484375,392.646484375,12.191250755673362,0.7143579153975297,0.3479558172680083,0.3564650671822684,4.918596428533239,19.8153093434092,0.5949150318604098,0.36601792109503534,0.2979973343270517,Nature,1 -us_births/M/short,TimesFM-2.5,41875856.0,41875856.0,5386.484375,0.6082879638609939,0.016612852613131206,0.01679656406243642,4.3013148773958045,6471.155692764624,0.02009936546194479,0.016730383744163482,0.012589306470978744,Healthcare,1 -m4_weekly/W/short,TimesFM-2.5,243507.58945789587,243507.58945789587,254.57879794300408,2.002006738398413,0.0620477094828048,0.06214771716222279,15.567259885908687,493.46488168652473,0.0899016540506798,0.04638031169125618,0.03672889993922486,Econ/Fin,1 -m4_daily/D/short,TimesFM-2.5,357812.1862854439,357812.1862854439,178.28089804276928,3.2973720846324865,0.03944506479697969,0.030716751373496262,25.289477981396118,598.174043473506,0.09240135890228973,0.027539471873125634,0.02170887240038401,Econ/Fin,1 -m4_monthly/M/short,TimesFM-2.5,1913012.8705555554,1913012.8705555554,568.2958120659722,0.9461520763200302,0.16367346827189128,0.13421268512584544,7.807885830350579,1383.1170849048012,0.28745921332034574,0.1181113796168424,0.09402596823179095,Econ/Fin,1 -covid_deaths/D/short,TimesFM-2.5,488996.0581453634,488996.0581453634,112.37994204260652,36.90652519783753,0.09635046916693711,0.140087063029661,787.7337998798989,699.2825309882719,0.2631084553844905,0.042283500097222745,0.03520733321768402,Healthcare,1 -electricity/D/short,TimesFM-2.5,1280911090.9186306,1280911090.9186306,4179.362306306306,1.376394006228205,0.4039522552711184,0.09069691375583717,11.162884282948307,35789.818257692095,0.5881455722977507,0.06868080239423598,0.05412539110984307,Energy,1 -ett1/D/short,TimesFM-2.5,52789.36507936508,52789.36507936508,150.3059523809524,1.7128243257015157,1.458841765873016,0.4725104437934028,10.795936587971358,229.75936342043838,0.6034705892197278,0.3947835696279482,0.2995758069334773,Energy,7 -ett2/D/short,TimesFM-2.5,101776.23492063492,101776.23492063492,190.6203621031746,1.2742572379259611,0.4289772639198909,0.1325021531846788,12.658773761919827,319.0238782922603,0.1825857827077706,0.10909706258025628,0.09198037762452532,Energy,7 -hierarchical_sales/D/short,TimesFM-2.5,28.97277025630426,28.97277025630426,2.310100700186027,0.7450299152621455,0.6223225357596032,1.0400629635148535,7.220078385362829,5.382635995151841,1.6513331142148036,0.7087133119950793,0.5736083169156433,Sales,1 -jena_weather/D/short,TimesFM-2.5,375.2297123015873,375.2297123015873,9.433528645833333,0.9881361987704972,0.6427340693078468,0.44242650592137894,6.275877127658018,19.37084696913347,0.116644158935524,0.05680526083556301,0.04450014058691751,Nature,21 -kdd_cup_2018/D/short,TimesFM-2.5,2890.372764146552,2890.372764146552,21.100617561008047,1.1915478125321073,0.5224141515912794,0.4641057599200412,9.315328509011994,53.76218712205217,1.2043300408538107,0.47267622412009197,0.3752128249284657,Nature,1 -loop_seattle/D/short,TimesFM-2.5,16.954627837977295,16.954627837977295,2.8356732536764704,0.8626503798094816,0.0531549569869066,0.05225526162099297,6.654660729601823,4.117599766608855,0.07358701485670618,0.050677273575549284,0.040881989395164374,Transport,1 -m_dense/D/short,TimesFM-2.5,7833.57037037037,7833.57037037037,43.863246527777775,0.6614871883280495,0.12206315251966156,0.09669913963035301,7.499356335144781,88.50745940524092,0.15321950419869124,0.07593376796366513,0.06262679111637355,Transport,1 -restaurant/D/short,TimesFM-2.5,140.26725506677874,140.26725506677874,7.113218863827402,0.6817651803581497,0.672049024586719,0.39261853007962083,4.655271897681685,11.843447769411522,0.5450629073410869,0.3273668132758502,0.2565109173699536,Sales,1 -saugeen/D/short,TimesFM-2.5,1089.804375,1089.804375,12.220719401041666,2.7150354225140947,0.27541824340820314,0.3160841369628906,29.34069807619617,33.01218525029811,1.0692725459192864,0.39583201317456074,0.3334447865947913,Nature,1 -solar/D/short,TimesFM-2.5,132874.1216545012,132874.1216545012,252.92551703163016,0.9852001248980804,1.096399549498175,0.4259804533055809,6.236162946245026,364.5190278359982,0.5266058384471678,0.36539122457291817,0.282132009142779,Energy,1 -temperature_rain/D/short,TimesFM-2.5,186.78465516533322,186.78465516533322,5.805680490917364,1.3471367892505408,21.423586487204133,1.4632016736168019,21.248662959386536,13.66691827609038,1.608957605174183,0.6834820835516865,0.5531139937752197,Nature,1 -us_births/D/short,TimesFM-2.5,157681.28,157681.28,232.9815625,0.3429802957289049,0.022539599736531576,0.022174895604451496,3.6081882000067926,397.09102231100616,0.03722406113386493,0.02184013094803981,0.018176382837871884,Healthcare,1 -bitbrains_fast_storage/5T/short,TimesFM-2.5,1551046.7231364634,1551046.7231364634,156.93641081467695,0.7111346915871751,2.084529976047417,0.6547266354579385,13.67728689253313,1245.410262980221,3.910280264568912,0.4927415232089318,0.3972588797180351,Web/CloudOps,2 -bitbrains_fast_storage/H/short,TimesFM-2.5,3913275.4918301073,3913275.4918301073,371.2564840757782,1.0595267693966375,5.106235520697134,0.48342517907401633,20.054053739570602,1978.2000636513253,5.638512621530631,1.0582015488476546,0.7823841680489255,Web/CloudOps,2 -bitbrains_rnd/5T/short,TimesFM-2.5,1513638.7558408459,1513638.7558408459,117.6014834052131,1.657069216715042,1.1087619891569254,0.5783237763087733,53.280834266860396,1230.3002706009804,5.034033519329187,0.4811913185189265,0.407886182497497,Web/CloudOps,2 -bitbrains_rnd/H/short,TimesFM-2.5,1813002.1042581305,1813002.1042581305,150.79917203017774,5.851180132676808,1.460220402267021,0.5174088677303353,207.67049567801752,1346.4776657108466,5.892845311867718,0.6599709869397857,0.6099774852778457,Web/CloudOps,2 -bizitobs_l2c/5T/short,TimesFM-2.5,20.796484375,20.796484375,2.631097121465774,0.2741012438522254,0.14270905494985106,0.16538955000808847,2.013703845534267,4.5603162582215715,0.15661971907450054,0.09036252502856015,0.07211338271113545,Web/CloudOps,7 -bizitobs_l2c/H/short,TimesFM-2.5,47.761645120287696,47.761645120287696,4.257519918774801,0.42609815531255246,0.3995344495887665,0.5085681733630952,2.7225835220301575,6.910980040507113,0.3725195235091099,0.22949122731891952,0.17948148016354798,Web/CloudOps,7 -electricity/15T/short,TimesFM-2.5,163057.3105518018,163057.3105518018,58.07214773367117,1.0813719758872353,0.18559918853932805,0.16279361898251307,12.475559481195296,403.80355440709263,0.7959764011378097,0.11447165002611698,0.09543840899225484,Energy,1 -electricity/H/short,TimesFM-2.5,1568839.215855856,1568839.215855856,166.87064893018018,0.9074777822980884,0.1794840493630383,0.11426110071487865,9.940571663819872,1252.533119664249,0.5912850807301807,0.07877486317537694,0.06457445714079207,Energy,1 -ett1/15T/short,TimesFM-2.5,4.448926943824405,4.448926943824405,1.0595989408947173,0.6939013682200964,0.47515989102516376,0.24041915166945685,4.839423711166975,2.1092479569326135,0.39873566163693147,0.20030830580102588,0.15687816257146692,Energy,7 -ett1/H/short,TimesFM-2.5,105.33231956845238,105.33231956845238,5.079697381882441,0.8386783804405953,0.45821060907273065,0.26602768670944943,6.054378576866275,10.263153490445925,0.4790233616991357,0.23709025871519926,0.1879599836121311,Energy,7 -ett2/15T/short,TimesFM-2.5,7.4982526506696425,7.4982526506696425,1.6455150785900297,0.7236933910820915,0.10664228708522598,0.12580321175711495,5.341338494253383,2.738293748060942,0.12999654557966517,0.07811845462797092,0.06281700151477447,Energy,7 -ett2/H/short,TimesFM-2.5,104.99975818452381,104.99975818452381,6.35654529389881,0.7392435317380189,0.12280855677774258,0.11098523821149553,5.328225508474343,10.246938966565763,0.1299596972133326,0.08061868079953902,0.06446220733378248,Energy,7 -jena_weather/10T/short,TimesFM-2.5,798.3594246031746,798.3594246031746,5.606163969494047,0.2759814563236545,0.3652289608908554,0.42951180463387495,2.275895126693949,28.255254813984152,0.17550054531012108,0.03482130457578503,0.028310176884314968,Nature,21 -jena_weather/H/short,TimesFM-2.5,1126.1365914786968,1126.1365914786968,8.669866397765247,0.5256772076315999,1.4817648312325153,0.5492415656817933,4.73762418100369,33.55795869058034,0.2057081264190382,0.053145722879987915,0.04341925652526499,Nature,21 -kdd_cup_2018/H/short,TimesFM-2.5,4248.663933293256,4248.663933293256,22.17124477002365,0.9326107374703951,0.8757481970428548,0.49470745038344477,7.680681551277781,65.18177608268478,1.3643726270227903,0.46408430836357945,0.3738848939157661,Nature,1 -loop_seattle/5T/short,TimesFM-2.5,43.08166415723362,43.08166415723362,3.586675238043408,0.5676442597911147,0.09546041478805621,0.07680766614474994,4.533866477064781,6.563662404270471,0.11255232274693105,0.06150356372960898,0.04905795669491059,Transport,1 -loop_seattle/H/short,TimesFM-2.5,39.994447527972405,39.994447527972405,3.514618150324534,0.7249853379569574,0.10246250665813958,0.08120649798884669,6.584976470863925,6.324116343646154,0.11194329036140971,0.06221231532315934,0.050835101142424816,Transport,1 -m4_hourly/H/short,TimesFM-2.5,816674.7826086957,816674.7826086957,208.1838516505636,0.7336225635960946,0.09490318114054952,0.08605552218770443,6.350530501913226,903.7006045193816,0.1233750796448087,0.028421690933593334,0.023603687196802628,Econ/Fin,1 -m_dense/H/short,TimesFM-2.5,43336.59111111111,43336.59111111111,86.36722222222222,0.7687776769780533,0.28691935354708525,0.20786165198414766,9.04084545954037,208.1744247286662,0.36924955572688317,0.1531939308898643,0.1277156153276405,Transport,1 -solar/10T/short,TimesFM-2.5,28.025799308090026,28.025799308090026,2.4525100151117702,1.0856914677487488,6.170121805558379,1.3197214159753101,7.284872870463529,5.293939866308459,1.5400065338110493,0.7134348978056745,0.562410294460541,Energy,1 -solar/H/short,TimesFM-2.5,646.6030861826098,646.6030861826098,11.653784395409144,0.8975871598886551,5.841614099542492,1.2377798262632285,5.693488431202964,25.42839134083416,0.9372989998079994,0.42956238605051833,0.33684762602429275,Energy,1 -sz_taxi/15T/short,TimesFM-2.5,16.662802001058836,16.662802001058836,2.7269209977034685,0.5443528714934225,1026926057729.6897,0.3942703507904314,3.9106452271059777,4.082009554258642,0.38168535135495124,0.25497877584332923,0.20064804516122467,Transport,1 -sz_taxi/H/short,TimesFM-2.5,7.158167651575854,7.158167651575854,1.8492835933326657,0.5632662593759367,1.1100459954677484,0.2958396194327591,3.9913149724075403,2.6754752197648655,0.24923702677844456,0.17227217844047116,0.1356919309691824,Transport,1 -bizitobs_application/10S/short,TimesFM-2.5,330682.1688888889,330682.1688888889,312.1507986111111,1.0240725901771486,0.014836207495795357,0.014869072172376844,6.58489661806276,575.0497099285321,0.02217736877256235,0.012038408600030099,0.00946548739519973,Web/CloudOps,2 -bizitobs_service/10S/short,TimesFM-2.5,1955.9106878306877,1955.9106878306877,17.747296626984127,0.7260925792291387,0.05928833653687169,0.04773929010623346,5.30990240831582,44.225679054489234,0.03276379690298709,0.013147764708082023,0.010208833214811,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,TimesFM-2.5,3601301.9677239405,3601301.9677239405,333.99372576574876,0.9944598806111055,5.372166098480442,0.7593947512167613,21.96364394552426,1897.709663706211,5.765766417782852,1.0147652428609104,0.7661455370329273,Web/CloudOps,2 -bitbrains_rnd/5T/medium,TimesFM-2.5,3665120.6096301274,3665120.6096301274,234.1662596679912,4.393525889829144,3.6995257800091004,0.6856240051857986,160.5133725802906,1914.4504719710374,8.05506799667902,0.9852566946857011,0.8462471494476078,Web/CloudOps,2 -bizitobs_l2c/5T/medium,TimesFM-2.5,99.9295068027211,99.9295068027211,5.961732700892857,0.570373055989672,0.3703254562848163,0.6968634861840967,3.301286113895149,9.996474718755662,0.5259615747336067,0.3136748111430989,0.24085762425754448,Web/CloudOps,7 -bizitobs_l2c/H/medium,TimesFM-2.5,70.47013578869047,70.47013578869047,4.955648949032738,0.5055644180297029,0.4936398311792312,0.7154494065504807,3.905736395801375,8.394649235595878,0.5083087619553032,0.3000720710624336,0.237198842814338,Web/CloudOps,7 -electricity/15T/medium,TimesFM-2.5,245526.12763963963,245526.12763963963,54.77988457207207,0.8656848904607868,0.131655882851089,0.12592427873321507,7.486144267713972,495.5059309833129,0.8538066156813237,0.09439125735806687,0.07663894354344278,Energy,1 -electricity/H/medium,TimesFM-2.5,4886055.971171171,4886055.971171171,234.51938626126127,1.0554992252868496,0.2379627267788109,0.11974627945134979,8.665759305545906,2210.4424831176157,0.8628085446874673,0.09154064487382105,0.07332413687793134,Energy,1 -ett1/15T/medium,TimesFM-2.5,10.993867807539683,10.993867807539683,1.668860367063492,1.0375436922235097,0.7040071466381042,0.404935050843254,7.266799181374099,3.315700198681974,0.631975736242916,0.31808643603568854,0.25117656002602395,Energy,7 -ett1/H/medium,TimesFM-2.5,171.96450892857143,171.96450892857143,7.689971051897321,1.34756523642872,3293537138066.087,0.459912109375,9.154208098367983,13.11352389438367,0.6281067090422288,0.3683313843737993,0.2867362399532431,Energy,7 -ett2/15T/medium,TimesFM-2.5,12.404727182539682,12.404727182539682,2.214093501984127,0.9149022185515798,0.1379066592906699,0.1654053703187004,6.748113218034075,3.522034523189641,0.18769559508560998,0.11799305052062278,0.09401224843254971,Energy,7 -ett2/H/medium,TimesFM-2.5,224.64012276785715,224.64012276785715,9.31343238467262,1.008217465819344,0.1738937768769248,0.16576941353934152,7.381878701158718,14.987999291695244,0.20197292887726445,0.1255044906277103,0.10058578998484317,Energy,7 -jena_weather/10T/medium,TimesFM-2.5,1479.3066378066378,1479.3066378066378,9.875244633838383,0.6270768476108538,0.7750044145782583,0.5936458308871877,5.373854923414963,38.46175552164303,0.23603913612001756,0.060604207497325306,0.04946904679343144,Nature,21 -jena_weather/H/medium,TimesFM-2.5,1235.7421626984126,1235.7421626984126,10.19021499875992,0.7294967251334445,2.3896717268912635,0.6012966534781367,6.354879678644171,35.15312450833372,0.21510820338191022,0.06235573284357655,0.050672936073476144,Nature,21 -kdd_cup_2018/H/medium,TimesFM-2.5,5162.1255940287765,5162.1255940287765,25.196162234030883,1.0357826893571056,0.9797903674234278,0.5530126460115053,8.65504762565764,71.84793381878686,1.5039073774973635,0.527401307933942,0.4190070378888065,Nature,1 -loop_seattle/5T/medium,TimesFM-2.5,71.91240970072239,71.91240970072239,4.92636638125645,0.78725902027526,0.16166315651530452,0.1126028883297254,6.407538652932233,8.480118495676956,0.15087872930691842,0.08765017848314394,0.0708601477568756,Transport,1 -loop_seattle/H/medium,TimesFM-2.5,45.79930340557276,45.79930340557276,3.83461566369969,0.7993525494107476,0.11227852237728973,0.08791354625217686,6.975295469379894,6.767518260453588,0.11961511420510942,0.0677763949639343,0.055399418785493454,Transport,1 -m_dense/H/medium,TimesFM-2.5,34187.49555555556,34187.49555555556,77.32255208333333,0.6636513116342978,0.2717813041665215,0.18925484551323785,7.38390700565494,184.89860885240742,0.3217002199459681,0.1345314719584924,0.1125194108107084,Transport,1 -solar/10T/medium,TimesFM-2.5,20.173338310108385,20.173338310108385,1.9253210279114135,0.8419677192257834,3.852799789556019,1.3532090771525966,5.494851046581868,4.491473957411796,0.9966758700882449,0.4272363658095386,0.3403948288162354,Energy,1 -solar/H/medium,TimesFM-2.5,603.5896593673966,603.5896593673966,11.535751216545012,0.8790033209032653,5.224685845940163,1.175929948247294,4.404438154662543,24.568061774739103,0.8849989272629146,0.4155446833950584,0.3144896878693224,Energy,1 -sz_taxi/15T/medium,TimesFM-2.5,16.498450854700856,16.498450854700856,2.767457306690705,0.5392917458753017,9618698564039.111,0.40737302079159987,3.848443609666335,4.061828511237378,0.3780209409954844,0.25755809541091446,0.20323306040383926,Transport,1 -bizitobs_application/10S/medium,TimesFM-2.5,3158747.0933333333,3158747.0933333333,990.3447916666667,2.1551507567479864,0.030904585520426433,0.03146876653035482,17.109516540347528,1777.2864409918097,0.06978880767120475,0.03888792520986933,0.03257143498433615,Web/CloudOps,2 -bizitobs_service/10S/medium,TimesFM-2.5,5948.295238095238,5948.295238095238,28.9795064484127,0.9590529440271665,0.06441918751550099,0.05834050738622272,8.456956522329744,77.12519198092954,0.05804623694845981,0.021810659458324874,0.01769820600229914,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,TimesFM-2.5,4890046.868961695,4890046.868961695,426.2569190373294,0.9039245531589026,5.605767855449335,0.7718292500663085,16.66245616890249,2211.345036162764,5.843924714622483,1.1264697743702696,0.8314849123811503,Web/CloudOps,2 -bitbrains_rnd/5T/long,TimesFM-2.5,3044285.974824346,3044285.974824346,207.04964470302644,3.3142125022990747,2.7015518762848894,0.6555396986637931,116.67458790936918,1744.7882320855865,6.683890913191733,0.7931605758000385,0.7434909857278398,Web/CloudOps,2 -bizitobs_l2c/5T/long,TimesFM-2.5,111.76350198412699,111.76350198412699,6.276284722222222,0.6111149916688,0.4408991726368691,0.7898923797979285,3.491541410749553,10.571825858579349,0.5935922170511804,0.3524040040902626,0.27905802661633616,Web/CloudOps,7 -bizitobs_l2c/H/long,TimesFM-2.5,85.50641121031747,85.50641121031747,5.561752077132937,0.5939030401954193,0.6468779454303172,0.7742091796875,4.914139521012379,9.246967676504415,0.5648314740731051,0.33972786907628705,0.27330386842038445,Web/CloudOps,7 -electricity/15T/long,TimesFM-2.5,329463.5568168168,329463.5568168168,60.65731390765766,0.9183277522976082,0.13831809860818936,0.129371021347215,7.711218268087252,573.9891608879185,0.9055382462309927,0.09569434651353753,0.07735959269693138,Energy,1 -electricity/H/long,TimesFM-2.5,6575584.818738739,6575584.818738739,260.5505885885886,1.144049401490204,0.29030430335495183,0.12943613493745162,9.2904288226515,2564.2903148315204,1.0113553417198615,0.10276107507532205,0.0816564838649398,Energy,1 -ett1/15T/long,TimesFM-2.5,10.996666666666666,10.996666666666666,1.6941745721726191,1.0596462769928112,0.7206295415912841,0.4119231693328373,7.389508660923909,3.3161222333723868,0.6320561764721924,0.32291134855872183,0.2554749042500906,Energy,7 -ett1/H/long,TimesFM-2.5,162.15492724867724,162.15492724867724,7.893733982308201,1.446573840888557,2715446555209.994,0.5015758050181879,9.304267357922857,12.734006724070678,0.609804002071968,0.37801382377189935,0.2877906458769053,Energy,7 -ett2/15T/long,TimesFM-2.5,12.42687003968254,12.42687003968254,2.2516981336805557,0.9372296383448865,0.14191541892974466,0.16810465494791665,7.145066945792213,3.5251765969497955,0.1878630418838546,0.1199970694130453,0.09667094654545266,Energy,7 -ett2/H/long,TimesFM-2.5,220.8450892857143,220.8450892857143,9.100840153769841,1.0069646760450506,0.1784442967131424,0.16981596770109952,7.714988377815448,14.860857622819562,0.20350800738377303,0.12462900138198967,0.10067781931216822,Energy,7 -jena_weather/10T/long,TimesFM-2.5,1435.1953703703705,1435.1953703703705,9.894154059193122,0.6749863328104863,0.8680234804586678,0.599796932756251,5.738901634587184,37.88397247346654,0.2318185235854962,0.060544025253318826,0.04992824523497636,Nature,21 -jena_weather/H/long,TimesFM-2.5,1341.0650793650793,1341.0650793650793,11.353404017857143,0.8486795076978108,2.015469348056003,0.5885197621247712,6.354005177419509,36.62055542130784,0.22049631648973175,0.06836007091527112,0.0551137663589809,Nature,21 -kdd_cup_2018/H/long,TimesFM-2.5,4074.3718164294796,4074.3718164294796,23.739632706628154,1.0077571619740389,1.026435179759149,0.6001084442615171,7.647535802284041,63.83080617091938,1.4977286463162567,0.5570277126460926,0.43987586865756206,Nature,1 -loop_seattle/5T/long,TimesFM-2.5,84.09417727324848,84.09417727324848,5.307352224515538,0.8448467761485187,0.17022654995843367,0.11971050705552688,6.808573822813049,9.170287742118482,0.1621325627087778,0.09383507274331708,0.07588087720699609,Transport,1 -loop_seattle/H/long,TimesFM-2.5,48.53299363605091,48.53299363605091,3.8958870829033367,0.8221961743610641,0.12143411915204678,0.09067764938349243,7.442797918638279,6.966562540884199,0.12450179950672687,0.06962471802268728,0.056986323021417803,Transport,1 -m_dense/H/long,TimesFM-2.5,33158.79901234568,33158.79901234568,77.57057098765432,0.6618874921204684,0.27207014923062633,0.18976261091820987,7.102992215276454,182.0955765864335,0.31523436169215885,0.13428612539515852,0.11246622901628173,Transport,1 -solar/10T/long,TimesFM-2.5,19.27295222843167,19.27295222843167,1.9489042184648722,0.8528475638795701,3.4750867936753687,1.3573541443547488,5.328555160230005,4.390097063668601,0.9488491510429032,0.42122442541373395,0.33093699944755023,Energy,1 -solar/H/long,TimesFM-2.5,652.8540145985402,652.8540145985402,12.415590784671533,0.9437132847905029,6.388343372750389,1.1941705061020644,4.811222981049376,25.55100809358684,0.8859000159936807,0.4304711592757934,0.32449917710721776,Energy,1 -sz_taxi/15T/long,TimesFM-2.5,15.859452902421653,15.859452902421653,2.718789507656695,0.513075909719218,7220194879423.817,0.39991559411725425,3.811237449056315,3.9823928613864368,0.36860318934502995,0.25164631380219227,0.19981733804663077,Transport,1 -bizitobs_application/10S/long,TimesFM-2.5,6339259.164444445,6339259.164444445,1536.017361111111,3.1622200569089265,0.04588777754041884,0.04694717831081814,46.64809741593758,2517.7885464121973,0.09710104034667998,0.05923804998119693,0.053380379324798874,Web/CloudOps,2 -bizitobs_service/10S/long,TimesFM-2.5,127076.42243386243,127076.42243386243,75.45964947089946,1.3103560944912158,0.08195709067046958,0.07877580915178571,16.251980995668283,356.4778007588445,0.26409015111099166,0.055902920712399704,0.050475274198989964,Web/CloudOps,2 diff --git a/results/TimesFM-2.5/config.json b/results/TimesFM-2.5/config.json deleted file mode 100644 index c76d7aff88aa4a8ed45ced6b7e9db680bdb3f45e..0000000000000000000000000000000000000000 --- a/results/TimesFM-2.5/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "TimesFM-2.5", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/google/timesfm-2.5-200m-pytorch", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/timesfm2p5.ipynb", - "org": "Google Research", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/Toto_Open_Base_1.0/all_results.csv b/results/Toto_Open_Base_1.0/all_results.csv deleted file mode 100644 index 7b0dd4434c49e205691b2a1be056781f27d5a000..0000000000000000000000000000000000000000 --- a/results/Toto_Open_Base_1.0/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/MAAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bizitobs_l2c/H/medium,Toto_Open_Base_1.0,37398.41667,157.5728981,7.473105004,0.7566620283,0.5795229645,0.6969685872,0.9660157703,6.472265453,193.3867024,11.70985857,0.452507859,0.3562059739,Web/CloudOps,7 -m_dense/D/short,Toto_Open_Base_1.0,20314.09185,11139,51.54292824,0.7633603786,0.1379826619,0.111935843,0.1106136294,8.515403944,142.5275126,0.2467362069,0.08922843298,0.0745206581,Transport,1 -ett2/H/long,Toto_Open_Base_1.0,21834922.12,250.4522817,9.895243262,1.077451519,0.1819867424,0.1717069757,0.1776895674,8.036886582,4672.785264,63.99019775,0.1355077405,0.1082210883,Energy,7 -bizitobs_application/10S/medium,Toto_Open_Base_1.0,135623775.6,3939721.813,1075.867917,2.303904467,0.03309280396,0.03302663167,0.03399109523,17.39248079,11645.76213,0.4572948033,0.04224616662,0.03422610427,Web/CloudOps,2 -ett2/W/short,Toto_Open_Base_1.0,99790116.57,5574654.857,1591.609794,0.9866367346,0.1466089998,0.143275312,0.1567967449,11.79440226,9989.500316,0.8379657713,0.1335116358,0.1058714494,Energy,7 -m4_weekly/W/short,Toto_Open_Base_1.0,1183982.392,477587.8294,339.1406685,2.399577752,0.08559447641,0.08309790376,0.09231963647,20.03856176,1088.10955,0.1982366778,0.061786168,0.049082368,Econ/Fin,1 -jena_weather/10T/short,Toto_Open_Base_1.0,3224.957937,766.3789187,5.19020066,0.2656077383,0.2951250528,0.4302952358,0.5413362049,2.447266509,56.78871311,0.3527291101,0.03223765357,0.02662086378,Nature,21 -bitbrains_rnd/H/short,Toto_Open_Base_1.0,12556663.97,1934760.008,139.8294697,5.637954952,0.748071646,0.3988558514,0.5126118129,194.5472316,3543.538341,15.50825784,0.6119621858,0.5927710423,Web/CloudOps,2 -ett1/W/short,Toto_Open_Base_1.0,4061652.857,1412445.286,901.6033761,1.578953942,0.7711945942,0.4131914548,0.5430430004,8.184546837,2015.354276,0.8019575683,0.3587695026,0.2629936609,Energy,7 -solar/H/long,Toto_Open_Base_1.0,229425639.3,826.3135645,12.58941638,0.9574328268,3.103650801,1.13842416,1.450042967,5.574609656,15146.80294,525.1673257,0.436498062,0.3313992776,Energy,1 -sz_taxi/15T/medium,Toto_Open_Base_1.0,40822.83077,16.65023872,2.799652152,0.5453405167,9227365555594,0.3907462878,0.4130393525,4.340224252,202.0466054,18.8038091,0.2605543631,0.2054407513,Transport,1 -loop_seattle/H/medium,Toto_Open_Base_1.0,320976.3534,72.4425882,4.540610488,0.9291967897,0.1228339669,0.09776583928,0.1037258278,7.571202991,566.5477503,10.01366808,0.08025478219,0.06448085654,Transport,1 -saugeen/M/short,Toto_Open_Base_1.0,762.5173456,428.9092262,12.91658093,0.7569064687,0.3590344247,0.3242794446,0.3679617927,5.364999916,27.61371662,0.8290466133,0.3877945088,0.2987905772,Nature,1 -ett2/15T/short,Toto_Open_Base_1.0,9.490230887,8.237386068,1.822001721,0.7858460986,0.112535502,0.1269984836,0.1317325047,5.123620462,3.080621835,0.1462480777,0.08649690337,0.06808393659,Energy,7 -ett1/D/short,Toto_Open_Base_1.0,58832.92698,44959.96825,137.6339658,1.658963218,1.487493703,0.3903144231,0.4549627879,10.99019236,242.5549979,0.6370787475,0.3615001765,0.2838008056,Energy,7 -bizitobs_application/10S/short,Toto_Open_Base_1.0,11946828.23,562513.0667,372.4225347,1.24709962,0.01725755056,0.01722629865,0.01764700784,10.70973968,3456.418411,0.1333002424,0.01436284855,0.01159086669,Web/CloudOps,2 -kdd_cup_2018/D/short,Toto_Open_Base_1.0,101953.7318,2979.821774,21.49683797,1.211969669,0.5038438613,0.3928861529,0.4758361871,9.826820957,319.3019446,7.152702717,0.4815520042,0.3873948742,Nature,1 -restaurant/D/short,Toto_Open_Base_1.0,550.5961258,181.4894111,8.217001659,0.7834700744,0.6456841315,0.3971230152,0.467267707,5.308714575,23.4647848,1.079903764,0.378165455,0.2973646443,Sales,1 -hospital/M/short,Toto_Open_Base_1.0,5870.95904,3106.503857,18.43248095,0.7832408926,0.194963272,0.1766570687,0.1788603291,5.280802512,76.62218373,0.2781563197,0.06691418614,0.05238767787,Healthcare,1 -sz_taxi/H/short,Toto_Open_Base_1.0,25.21437947,7.372503297,1.861796518,0.5675622397,1.311333885,0.2796560923,0.2980483862,4.380735836,5.021392184,0.4677736798,0.1734378388,0.1373674751,Transport,1 -bizitobs_service/10S/long,Toto_Open_Base_1.0,7877186.208,130185.9285,75.13797619,1.302467646,0.07940020849,0.06686133329,0.07720037228,20.63749334,2806.632539,2.079243111,0.05566461486,0.05102757171,Web/CloudOps,2 -us_births/W/short,Toto_Open_Base_1.0,3784342.286,3362242.286,1358.152344,1.234635428,0.01820384392,0.01819532684,0.01831153248,9.091229699,1945.338604,0.02640778081,0.01843678489,0.01447765155,Healthcare,1 -ett1/H/long,Toto_Open_Base_1.0,64097.39259,160.3568287,7.115493242,1.369602803,4903367495009,0.3615915773,0.481282649,9.37411769,253.1746287,12.12398463,0.3407455603,0.2668288663,Energy,7 -ett1/15T/short,Toto_Open_Base_1.0,9.704888044,4.876736305,1.09491272,0.6927710736,0.5079014172,0.2077161335,0.2401852562,4.874973738,3.11526693,0.5889150585,0.2069840572,0.1616278199,Energy,7 -hierarchical_sales/W/short,Toto_Open_Base_1.0,2810.427436,488.3472425,9.143537101,0.7442683038,0.5216713315,0.4083515587,0.4689143229,6.695418071,53.01346467,2.433874094,0.4197842607,0.3558391793,Sales,1 -hierarchical_sales/D/short,Toto_Open_Base_1.0,123.3105751,29.73635348,2.291699299,0.7354232305,0.5850923014,0.7704961101,1.053829294,6.894183285,11.10452949,3.406746671,0.703067966,0.5704088874,Sales,1 -m4_hourly/H/short,Toto_Open_Base_1.0,3400969.634,3374528.178,325.7894297,0.8608470877,0.1000763959,0.08706018813,0.09206759258,6.491836545,1844.171802,0.2517701712,0.04447745075,0.03460384789,Econ/Fin,1 -covid_deaths/D/short,Toto_Open_Base_1.0,9959467.475,321507.2149,89.58891602,32.61906192,0.08334196633,0.2733040927,0.3592514707,580.5163294,3155.862398,1.187408529,0.03370826405,0.02727374002,Healthcare,1 -bizitobs_l2c/5T/short,Toto_Open_Base_1.0,45.78455636,19.32186454,2.504416039,0.2585959492,0.1496430341,0.1714225042,0.2033655439,2.81569694,6.766428627,0.2323865475,0.08601178389,0.06947410461,Web/CloudOps,7 -solar/D/short,Toto_Open_Base_1.0,444993.8453,154174.6287,259.1266138,1.011716133,1.185910688,0.4337246415,0.4276860555,5.586282249,667.07859,0.9637013844,0.3743497097,0.2897205121,Energy,1 -solar/W/short,Toto_Open_Base_1.0,4998447.65,2598564.438,1285.94594,1.426511618,0.2974762186,0.2783865546,0.2451364002,7.223323168,2235.720834,0.4563665181,0.2624937165,0.1860354557,Energy,1 -ett2/15T/medium,Toto_Open_Base_1.0,67342.93841,12.99678819,2.234408172,0.9225758172,0.1396207755,0.1561704218,0.165291283,6.545983746,259.5051799,13.82949986,0.1190756482,0.09284270687,Energy,7 -saugeen/D/short,Toto_Open_Base_1.0,2401.326667,1292.026354,13.34403158,2.964942115,0.2790883382,0.2586500549,0.3283487956,26.48296733,49.00333322,1.587229639,0.4322163622,0.3532752348,Nature,1 -m4_daily/D/short,Toto_Open_Base_1.0,792623.8843,387850.5572,174.9001043,3.312199429,0.03974167746,0.03059911665,0.03097884076,30.36738222,890.2942684,0.1375258593,0.02701723239,0.02186022254,Econ/Fin,1 -ett2/15T/long,Toto_Open_Base_1.0,17880.44825,10.6662748,2.076824932,0.8713644491,0.1326568412,0.1502894035,0.1596898251,7.092517536,133.7177933,7.126062783,0.1106777526,0.08833617314,Energy,7 -ett2/H/short,Toto_Open_Base_1.0,153.0302176,103.0952195,6.32607945,0.7349521832,0.1200016492,0.1090898786,0.1105930964,6.540849498,12.37053829,0.1568928454,0.08023228913,0.06450730164,Energy,7 -sz_taxi/15T/long,Toto_Open_Base_1.0,1767026.507,16.06504184,2.745417112,0.5175745084,8452365932963,0.3840168227,0.4043976613,4.809020317,1329.295493,123.0372229,0.2541109174,0.2018226322,Transport,1 -jena_weather/H/short,Toto_Open_Base_1.0,2050.049916,1141.295948,8.399357769,0.5438768279,1.19870911,0.494068474,0.6225096432,4.050156961,45.27747692,0.2775480409,0.05148752652,0.04224081567,Nature,21 -bizitobs_l2c/H/short,Toto_Open_Base_1.0,159.4930866,60.77701048,4.745320638,0.4696071924,0.4324744428,0.5125146291,0.6477213057,3.146160506,12.62905723,0.6807385995,0.2557849621,0.1989570271,Web/CloudOps,7 -bitbrains_rnd/5T/medium,Toto_Open_Base_1.0,66664251431,2268161.072,152.7860747,4.416975059,0.7249773977,0.543368251,0.7244324769,172.9594116,258194.2126,1086.3548,0.6428489778,0.6281348146,Web/CloudOps,2 -jena_weather/D/short,Toto_Open_Base_1.0,1598.264187,400.0442708,10.0961325,1.195979531,0.5814537397,0.3924275716,0.4858788869,8.356709248,39.97829644,0.2407346861,0.06079521903,0.05050018266,Nature,21 -bizitobs_service/10S/short,Toto_Open_Base_1.0,17440.30815,2418.000741,19.37142733,0.7885676144,0.05277840751,0.04206951,0.0429771496,6.834426153,132.0617588,0.09783557861,0.0143509735,0.01143907001,Web/CloudOps,2 -bizitobs_l2c/H/long,Toto_Open_Base_1.0,43524.07619,138.8084077,7.564872427,0.7971592662,0.751385241,0.6967385428,0.9400415814,8.978071088,208.6242464,12.7433711,0.4620842414,0.3685722769,Web/CloudOps,7 -m_dense/H/short,Toto_Open_Base_1.0,76576.03111,56413.84,101.2946419,0.8785644308,0.3165610233,0.2169596693,0.227740682,10.54627774,276.7237451,0.4908389989,0.1796714648,0.1483854659,Transport,1 -sz_taxi/15T/short,Toto_Open_Base_1.0,35.3490883,17.17536928,2.758330405,0.5498899178,918322775258,0.3595493862,0.3941651881,3.879414269,5.945509928,0.5559306294,0.2579157005,0.2026779445,Transport,1 -saugeen/W/short,Toto_Open_Base_1.0,1621.368164,1042.626758,16.11312256,1.307169824,0.433044672,0.3785656452,0.4306035042,10.04483078,40.26621616,1.215469217,0.4863880029,0.3900837548,Nature,1 -bizitobs_l2c/5T/long,Toto_Open_Base_1.0,1942267.124,304.8233929,12.6343192,1.177213459,0.8224734879,0.7236397104,1.065886579,6.787914198,1393.65244,78.251501,0.7093981345,0.5326296421,Web/CloudOps,7 -bitbrains_fast_storage/H/short,Toto_Open_Base_1.0,24009418.08,3322312.737,278.0267631,0.9447498096,2.774899699,0.444406509,0.5689816659,19.45060895,4899.94062,13.96642267,0.7924666008,0.6227602305,Web/CloudOps,2 -kdd_cup_2018/H/short,Toto_Open_Base_1.0,9510.549401,5040.024176,24.1799674,0.9901861829,0.812027319,0.440681097,0.5372339059,7.707072322,97.52204572,2.041313042,0.5061305107,0.4029938215,Nature,1 -electricity/W/short,Toto_Open_Base_1.0,132357951466,70021748932,36191.03851,1.789913428,0.2654867862,0.1058043033,0.1129543184,12.99505113,363810.3235,0.8300951004,0.08257600681,0.06366477644,Energy,1 -jena_weather/10T/medium,Toto_Open_Base_1.0,2884813.096,1499.197547,9.720549242,0.5977261209,0.6488183008,0.5138719787,0.6587338226,6.209115747,1698.473755,10.42350324,0.05965483792,0.04890392733,Nature,21 -solar/H/medium,Toto_Open_Base_1.0,8428203.529,814.2207953,11.34814768,0.8654755987,3.420207596,1.133391799,1.428819249,6.837183454,2903.136843,104.5777606,0.4087867486,0.3312314098,Energy,1 -jena_weather/H/long,Toto_Open_Base_1.0,690451.9111,1390.475926,11.6833664,1.013920841,1.949344184,0.5521645198,0.6782610212,9.512091035,830.9343603,5.003145961,0.07034681705,0.05715514827,Nature,21 -jena_weather/10T/long,Toto_Open_Base_1.0,16709049.02,1481.672751,9.802677538,0.6347010837,0.6718184922,0.5140006575,0.6538680142,7.150488703,4087.670366,25.01315589,0.05998426473,0.04982757937,Nature,21 -m4_quarterly/Q/short,Toto_Open_Base_1.0,8191969.069,1884532.298,576.4840781,1.226756767,0.1140628526,0.09880934656,0.1050152807,10.04283167,2862.161608,0.4790754509,0.09649328287,0.07759517241,Econ/Fin,1 -jena_weather/H/medium,Toto_Open_Base_1.0,22609.89048,1427.170437,10.30184539,0.7525596047,1.445821565,0.5293127151,0.6805553134,6.384433833,150.3658554,0.9201152673,0.06303881424,0.05268077696,Nature,21 -m4_yearly/A/short,Toto_Open_Base_1.0,14551917.71,3767044.312,918.9365125,3.39694196,0.16269661,0.1370154016,0.1588259838,34.50765278,3814.697591,0.6117044402,0.1473557291,0.1216313197,Econ/Fin,1 -bizitobs_application/10S/long,Toto_Open_Base_1.0,102915131.2,6357697.422,1513.660972,3.274733168,0.04762808482,0.04752889845,0.04876194424,51.24138813,10144.70952,0.3912408965,0.05837585342,0.05250523855,Web/CloudOps,2 -solar/H/short,Toto_Open_Base_1.0,892.3889222,729.6633282,10.73672368,0.8282911464,3.436526479,1.129916488,1.41346713,9.66753028,29.87287938,1.1011243,0.3957592166,0.3281712475,Energy,1 -ett2/H/medium,Toto_Open_Base_1.0,301437.2,229.789881,9.397626023,1.017258383,0.1697165203,0.1604266212,0.1658503941,7.095124107,549.032968,7.398571851,0.1266390461,0.1017818934,Energy,7 -solar/10T/short,Toto_Open_Base_1.0,38.99912798,28.8004132,2.328592015,1.032864175,4.504511879,1.233119881,1.578546369,7.924051156,6.244928181,1.816648932,0.6773871651,0.5406264183,Energy,1 -us_births/D/short,Toto_Open_Base_1.0,298692.0533,298147.7333,338.3152604,0.4979574637,0.03238560359,0.03221077601,0.0322724088,4.706409047,546.5272668,0.05123249645,0.03171431039,0.02593239217,Healthcare,1 -bizitobs_service/10S/medium,Toto_Open_Base_1.0,151640546.4,63206.46095,50.15047619,1.083177612,0.07057723757,0.06131290496,0.0660597011,10.93458683,12314.24161,9.267987315,0.03774442567,0.02729092177,Web/CloudOps,2 -loop_seattle/H/long,Toto_Open_Base_1.0,122602991.5,70.56007267,4.516094838,0.9429943618,0.1409278158,0.1038558997,0.1031635044,7.743303101,11072.62351,197.8825949,0.0807086563,0.06461756368,Transport,1 -loop_seattle/H/short,Toto_Open_Base_1.0,117.8778003,70.50443307,4.440241586,0.89936996,0.1235948057,0.09710036909,0.1015866682,8.034821411,10.85715434,0.1921826755,0.0785967926,0.06349726817,Transport,1 -ett1/H/medium,Toto_Open_Base_1.0,249021.1429,142.4777995,6.782524182,1.271840857,5101261675861,0.3463809059,0.4439577375,8.757355818,499.0201828,23.90188383,0.3248668304,0.2538733743,Energy,7 -ett1/H/short,Toto_Open_Base_1.0,151.5017857,123.0018322,5.310755557,0.8649675899,0.5511348906,0.2243749709,0.2674878438,6.242362432,12.30860616,0.5744930063,0.2478746891,0.1937806943,Energy,7 -solar/10T/long,Toto_Open_Base_1.0,248394281.7,24.29987644,2.012680234,0.8802104975,2.653774763,1.173673626,1.502075159,10.68856756,15760.52923,3406.386319,0.4350086417,0.352392243,Energy,1 -bitbrains_rnd/5T/long,Toto_Open_Base_1.0,185133487404,2150290.438,167.7686039,3.336747658,1.638555064,0.5464242455,0.7114870421,129.292375,430271.4113,1648.272946,0.642683766,0.5889880229,Web/CloudOps,2 -m_dense/H/long,Toto_Open_Base_1.0,999569700.3,42277.33728,90.20064043,0.7801472265,0.2769145027,0.2013021135,0.2136976906,7.749645307,31615.97223,54.73192163,0.1561506427,0.1284824893,Transport,1 -ett1/15T/long,Toto_Open_Base_1.0,5819.542857,10.78491195,1.689712767,1.069131671,0.8389982592,0.315474776,0.4087703063,8.044604274,76.2859283,14.54017336,0.322060924,0.2511173411,Energy,7 -electricity/15T/short,Toto_Open_Base_1.0,264941.1271,164515.0974,60.84776745,1.144713264,0.1923987992,0.1668248233,0.1841301134,11.95686549,514.7243215,1.014623107,0.1199429448,0.09944710545,Energy,1 -loop_seattle/5T/short,Toto_Open_Base_1.0,106.8326729,45.32785115,3.543208074,0.5618224956,0.09517780755,0.0738451314,0.07534042072,4.340436915,10.33598921,0.177239396,0.06075819607,0.04834597093,Transport,1 -kdd_cup_2018/H/long,Toto_Open_Base_1.0,724480550.1,4217.507247,24.57593103,1.041030982,0.8698310088,0.4958640978,0.6338830421,10.62248093,26916.17636,631.5622494,0.5766506384,0.4566059798,Nature,1 -car_parts/M/short,Toto_Open_Base_1.0,13.16870377,1.412413313,0.4252767785,0.810434487,0.9362636455,1.386385595,1.963242548,14.08657555,3.628870867,8.701660087,1.01977009,0.8989729648,Sales,1 -electricity/H/medium,Toto_Open_Base_1.0,7554248694,5027753.095,238.8937961,1.102564665,0.2017909919,0.1235894356,0.1342696326,9.839924909,86915.18104,33.9258586,0.09324811904,0.07507115984,Energy,1 -bitbrains_fast_storage/5T/medium,Toto_Open_Base_1.0,52323102590,2994436.864,279.761322,0.9847437587,4.050228287,0.6095507418,0.8091713329,24.59009573,228742.4372,694.9827225,0.8499921903,0.629301822,Web/CloudOps,2 -bizitobs_l2c/5T/medium,Toto_Open_Base_1.0,221127.0531,134.4320047,7.904009885,0.7541455962,0.4971016052,0.5809160505,0.8020871532,5.693576376,470.2414838,24.74161725,0.4158671535,0.3155920059,Web/CloudOps,7 -m4_monthly/M/short,Toto_Open_Base_1.0,14562640.19,1990627.53,581.9357991,0.9827794763,0.1631085373,0.1355008769,0.1373329189,8.427619296,3816.102749,0.7931171544,0.1209462363,0.09687896759,Econ/Fin,1 -electricity/H/long,Toto_Open_Base_1.0,5922055962,6609210.048,261.1795038,1.235409238,0.2984009959,0.1365030847,0.1484970644,11.27322499,76954.89564,30.35098714,0.1030091159,0.08349723518,Energy,1 -solar/10T/medium,Toto_Open_Base_1.0,3413751.302,24.3248469,2.008026238,0.8811358769,3.107376946,1.176572155,1.494887214,7.98364065,1847.633974,409.997344,0.4455890266,0.3530037168,Energy,1 -electricity/H/short,Toto_Open_Base_1.0,8549918.024,1703296.617,175.6078327,0.9756653182,0.1660374339,0.1175242395,0.1308332802,11.92837172,2924.024286,1.380348293,0.08289943877,0.0689230012,Energy,1 -bitbrains_fast_storage/5T/long,Toto_Open_Base_1.0,60371635062396,3498794.655,356.8014712,0.8969665165,4.132958995,0.625213384,0.8308686532,20.3471869,7769918.601,20533.57434,0.9429197281,0.6688295234,Web/CloudOps,2 -ett1/15T/medium,Toto_Open_Base_1.0,6064.968889,11.92386781,1.753412078,1.080607726,0.821602986,0.3157942321,0.409499473,7.527531484,77.87791015,14.84360492,0.3342020363,0.2604685887,Energy,7 -temperature_rain/D/short,Toto_Open_Base_1.0,2721.590325,190.2997162,5.890219042,1.365017681,23.98484399,1.084659305,1.546957797,18.23690176,52.16886356,6.141654542,0.6934345137,0.559701936,Nature,1 -us_births/M/short,Toto_Open_Base_1.0,43483594.67,45510197.33,5144.822917,0.5814147368,0.01584235082,0.01583728567,0.01602387925,4.620249499,6594.209177,0.02048156874,0.0159797849,0.01269073845,Healthcare,1 -bitbrains_rnd/5T/short,Toto_Open_Base_1.0,5755247.409,1885353.991,121.8783778,1.649742225,0.9224266805,0.4824698187,0.6465897673,53.0055946,2399.009673,9.816055112,0.4986911419,0.3985907533,Web/CloudOps,2 -kdd_cup_2018/H/medium,Toto_Open_Base_1.0,688932790.1,5444.038506,26.52247464,1.077645508,0.8338213196,0.4752643862,0.6078889982,10.70520091,26247.52922,549.4082931,0.5551633983,0.4414215174,Nature,1 -m_dense/H/medium,Toto_Open_Base_1.0,20470620.16,37971.29778,84.76822049,0.7282179456,0.2750284714,0.1948773024,0.20424096,7.242183538,4524.446945,7.871966082,0.1474859943,0.121148238,Transport,1 -loop_seattle/5T/medium,Toto_Open_Base_1.0,7447219.553,81.10869925,5.071234732,0.8042105073,0.1584395226,0.1121562314,0.115953968,6.636613702,2728.959427,48.55379626,0.09022768735,0.07234440444,Transport,1 -loop_seattle/5T/long,Toto_Open_Base_1.0,86819307.2,94.12713751,5.375400578,0.8478243958,0.1684977292,0.1162421453,0.1224421713,7.401495938,9317.687868,164.7386371,0.09503818732,0.07661619596,Transport,1 -loop_seattle/D/short,Toto_Open_Base_1.0,28.73048971,19.93816394,3.069733063,0.9245607514,0.05696543328,0.05646385295,0.05660744698,7.53428119,5.360082995,0.09579184361,0.05486023066,0.04439360883,Transport,1 -electricity/15T/medium,Toto_Open_Base_1.0,474772741.9,286474.7632,62.27742545,0.9876350728,0.171121329,0.1424783133,0.1554430993,8.213132241,21789.28044,37.54512416,0.1073102748,0.08595130674,Energy,1 -electricity/15T/long,Toto_Open_Base_1.0,1205277865547,372753.3002,67.9122961,1.043512109,0.1755482863,0.1452407467,0.1576501538,8.931468734,1097851.477,1731.995287,0.1071399723,0.08562482807,Energy,1 -ett2/D/short,Toto_Open_Base_1.0,629423.0857,148927.2,240.0302827,1.614564587,0.516818092,0.1579685514,0.1562183683,14.4767306,793.3618882,0.4540618892,0.1373756482,0.1114829481,Energy,7 -bitbrains_fast_storage/5T/short,Toto_Open_Base_1.0,7262583.103,2039738.605,152.8956294,0.6717791317,1.543328694,0.5455201532,0.726179967,13.40476802,2694.918014,8.461376119,0.4800544656,0.3706565882,Web/CloudOps,2 -electricity/D/short,Toto_Open_Base_1.0,2239868034,1483995494,4526.021261,1.484873289,0.5564405235,0.09443621331,0.1015192079,12.33176477,47327.2441,0.7777437967,0.07437756046,0.05886452498,Energy,1 \ No newline at end of file diff --git a/results/Toto_Open_Base_1.0/config.json b/results/Toto_Open_Base_1.0/config.json deleted file mode 100644 index 02ba3639e9595f967fae262166d3cd3bff03dc20..0000000000000000000000000000000000000000 --- a/results/Toto_Open_Base_1.0/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "Toto_Open_Base_1.0", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/Datadog/Toto-Open-Base-1.0", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/toto.ipynb", - "org": "Datadog", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/Xihe-max/all_results.csv b/results/Xihe-max/all_results.csv deleted file mode 100644 index 9e34fb0cdd0620b53da214650d4ea5809ac4ab6a..0000000000000000000000000000000000000000 --- a/results/Xihe-max/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -loop_seattle/5T/short,Xihe-max,40.43888994970804,41.483489122577744,3.532342967849275,0.55891249,0.09789339,0.075017267,5.073838869196485,6.3591579591725855,0.10904551950896031,0.060571883335174284,0.04814697,Transport,1 -loop_seattle/5T/medium,Xihe-max,73.00742161297084,76.07616420560824,5.063262503577706,0.8016960572185369,0.16493337001347813,0.11531932677194438,7.50853032,8.544438051327358,0.1520231104267297,0.090085844,0.0723004,Transport,1 -loop_seattle/5T/long,Xihe-max,87.23063086766139,90.94664089822697,5.467993627652541,0.8643729914057043,0.17323739895920676,0.12340939648107248,8.499727219,9.339733982703223,0.16512841767131625,0.096675252,0.078044132,Transport,1 -loop_seattle/D/short,Xihe-max,16.240375649793805,16.37617504825914,2.7048253600420455,0.8177830268838611,0.050378752,0.04997968,10.020992757686034,4.0299349436180485,0.072020321,0.048338842,0.040056892,Transport,1 -loop_seattle/H/short,Xihe-max,53.18773947289918,53.86646760274726,4.018859533424205,0.8226476562270052,0.10943146986327085,0.092176131,9.158625247,7.292992490939448,0.12909338397709974,0.071137901,0.057879168,Transport,1 -loop_seattle/H/medium,Xihe-max,59.42624660369905,59.957970448150476,4.367926264744179,0.9078157749020617,0.11562808677015905,0.09885442,8.781565361873218,7.708842105251543,0.1362529224173427,0.077202608,0.061630729,Transport,1 -loop_seattle/H/long,Xihe-max,56.54217654852511,57.23768306767608,4.254855024392227,0.8994190436126114,0.12534481398953495,0.098256265,8.785541547476168,7.519453208081364,0.1343826740004695,0.076039943,0.061238438,Transport,1 -m_dense/D/short,Xihe-max,8574.526534285324,8594.589633393813,47.25591577045392,0.7148130394027843,0.13003639221660096,0.10533245917940369,8.388933187636884,92.59873937740905,0.16030210858633187,0.081806977,0.066869633,Transport,1 -m_dense/H/short,Xihe-max,45773.40185763889,45237.41684027778,88.21960625542535,0.7847471331801037,0.2906127354064818,0.20796444968044028,12.53978650440644,213.94719408685614,0.37948904879657436,0.15647961454212142,0.13355578212703959,Transport,1 -m_dense/H/medium,Xihe-max,38522.68944444445,38068.765,81.86022461,0.7070103452613254,0.2622218402817095,0.19388590604890618,9.795823139766913,196.27197824560807,0.3414884638689207,0.1424264564085618,0.11893150841853956,Transport,1 -m_dense/H/long,Xihe-max,36433.85283950617,35871.27382716049,82.32956114969136,0.7202059854599885,0.2720325594808469,0.19675861974615394,8.826620727,190.87653821123794,0.3304355219821916,0.14252464848747629,0.11822536578982724,Transport,1 -sz_taxi/15T/short,Xihe-max,16.624965674141233,16.869075050560657,2.745602142070755,0.5481744712554384,1053329474548.7897,0.3994232773728807,4.974008623560617,4.077372398265975,0.38125177536994304,0.25672555481298753,0.2042166140725311,Transport,1 -sz_taxi/15T/medium,Xihe-max,16.32156558516399,16.441373657350617,2.753261575566758,0.5369029547625049,9.34725E+12,0.4091822104176419,4.334017424080532,4.039995740735873,0.3759890263260128,0.2562369382175664,0.20365485778608358,Transport,1 -sz_taxi/15T/long,Xihe-max,15.938422975265969,15.785610608010298,2.7098437749721445,0.5115509994352472,5909669311534.772,0.40451108713902295,3.8244341838232527,3.992295451900569,0.36951974120965486,0.25081830303155633,0.19862658462624133,Transport,1 -sz_taxi/H/short,Xihe-max,7.208081405092837,7.240879333405586,1.848838273495326,0.5632019484004515,1.0071021611465893,0.2991861754769035,4.931010868070151,2.684787031608436,0.2501044795138191,0.17223069414960424,0.13778471061852646,Transport,1 -bitbrains_fast_storage/5T/short,Xihe-max,1686273.4450776412,1804684.443830307,170.80934287803214,0.721762658,2.0146869914927734,0.5241492862205664,14.35932088905056,1298.5659186493542,4.077175814518558,0.5362990909236787,0.41807512414907394,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Xihe-max,3079200.5768346265,3179359.9298359514,275.15055901622594,0.9944859787964451,3.8290502802143296,0.6096423701641663,21.788305704814665,1754.7651058858637,5.331460972970607,0.8359833815737332,0.6466734343231637,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Xihe-max,4315114.182411402,4798245.244979662,395.6892488482168,0.9020835917447682,4.269994214271769,0.641081769,16.608554479770408,2077.285291531089,5.489644789077534,1.0456885396962372,0.753756082,Web/CloudOps,2 -bitbrains_fast_storage/H/short,Xihe-max,3338567.7183994856,2990586.6777310176,344.7589411912258,1.0836269229307502,3.5807866625975886,0.4556640307299031,26.000839539159116,1827.1747914196621,5.208041620507467,0.9826749598328927,0.711651156,Web/CloudOps,2 -bitbrains_rnd/5T/short,Xihe-max,1519467.9174962086,1755427.6889855175,129.4388743118469,1.685214824096653,0.9420500314451983,0.476785465,54.64327586176979,1232.6669937563058,5.043717516443827,0.5296265098215975,0.436287498,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Xihe-max,2227774.2230110224,2312662.8937795945,154.37789718374887,4.405160706227904,0.6873007466652346,0.5455239107870671,161.52038543366845,1492.5730209979754,6.280015466541073,0.6495465001489837,0.6230363221732866,Web/CloudOps,2 -bitbrains_rnd/5T/long,Xihe-max,2252478.4572726097,2383069.9187000855,167.18379346208692,3.3449739213485845,0.9446942625215605,0.5178461158771118,117.02658345092676,1500.8259250401459,5.749326004,0.6404434486466932,0.5877684725365372,Web/CloudOps,2 -bitbrains_rnd/H/short,Xihe-max,1752419.1156340742,1757964.507432716,146.28859898527725,5.846193790597633,1.2675260164823445,0.45932663540578794,214.83024808631765,1323.789679531486,5.793551442,0.6402304963834679,0.6020814628365917,Web/CloudOps,2 -bizitobs_application/10S/short,Xihe-max,277133.17333333334,285303.43111111113,287.13652777777776,1.0127702805614824,0.014074728224012586,0.014199012402425187,12.343137144546272,526.4343960393672,0.020302470436820423,0.011073708158886447,0.00930797,Web/CloudOps,2 -bizitobs_application/10S/medium,Xihe-max,624907.04,1906994.1333333333,729.0697916666667,1.6803280914073229,0.023520924,0.024077376,17.16130210503918,790.5106198,0.031041025427363526,0.028628424938141297,0.019247550331672338,Web/CloudOps,2 -bizitobs_application/10S/long,Xihe-max,6319850.382222222,6435751.253333333,1532.693888888889,3.2666337653169677,0.047292743,0.048358072,90.74420870624368,2513.9312604409497,0.09695228,0.059109876941883945,0.055173652959005275,Web/CloudOps,2 -bizitobs_l2c/5T/short,Xihe-max,21.041132027762277,20.928046162923177,2.686663055419922,0.2760794537119198,0.14519876948232363,0.18783679307128737,3.315343170927217,4.5870613717021795,0.15753824823736404,0.09227088,0.075943918,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Xihe-max,155.61200042517007,161.55875850340135,8.443089657738096,0.8165475147896163,0.5902326631334006,0.8976846511479782,9.658625447784592,12.474453912904167,0.6563397435493132,0.44423069253486547,0.3652128522545015,Web/CloudOps,7 -bizitobs_l2c/5T/long,Xihe-max,253.85899801587303,257.2323214285714,11.377468687996032,1.077452371416156,0.8755407568037507,1.222084454847497,14.285673179126773,15.932953210747625,0.894611503,0.6388278574847336,0.543603364,Web/CloudOps,7 -bizitobs_l2c/H/short,Xihe-max,72.06878874156828,79.93214022004918,5.319949535232453,0.5328521764648978,0.4641356657980563,0.6090653871394596,3.6604720470189354,8.489333822012672,0.45759685846252546,0.28675891955022587,0.22286056808559973,Web/CloudOps,7 -bizitobs_l2c/H/medium,Xihe-max,76.14531655449511,80.55712172052691,5.192008082701517,0.5266006725632296,0.5220757097051393,0.731828762,4.664760718131172,8.726128382879494,0.5283803278112379,0.314383976,0.24960121492073345,Web/CloudOps,7 -bizitobs_l2c/H/long,Xihe-max,82.53830601948776,91.12891830664026,5.715103302632152,0.6080232944354178,0.6264686578853028,0.7846838078544696,6.339762608822812,9.085059494548606,0.5549416550234545,0.34909500452867165,0.279781724,Web/CloudOps,7 -bizitobs_service/10S/short,Xihe-max,2570.819834242725,2666.7467494419643,19.63391042880911,0.7971252845881031,0.06385074,0.054698563453072675,9.475585973561303,50.70325269884295,0.037562591,0.014545428778223048,0.011499442206701717,Web/CloudOps,2 -bizitobs_service/10S/medium,Xihe-max,10837.810357142856,14265.451785714286,32.80849795386905,1.0196288044851052,0.071887067,0.064838037,11.007491150365594,104.10480467847225,0.078351727,0.024692448,0.018631527,Web/CloudOps,2 -bizitobs_service/10S/long,Xihe-max,134972.1766137566,135134.55285714284,78.65493551587302,1.464440826,0.093576408,0.090316884,29.294566045520178,367.3855966335052,0.2721710110110273,0.058270094,0.054416546932152546,Web/CloudOps,2 -car_parts/M/short,Xihe-max,1.2950580818241584,1.3106965380842424,0.47076055615699774,0.8567573709296786,0.7848790167209564,1.8321900291063964,18.15275666178081,1.1380061870763964,2.7288220994342263,1.1288355228396694,0.965325598,Sales,1 -covid_deaths/D/short,Xihe-max,369962.24217722326,306725.0904659206,101.86014916973087,35.65174079300399,0.094724611,0.12901171357269897,835.4092325473938,608.2452154988342,0.2288552116026008,0.038325375026572524,0.032067086,Healthcare,1 -electricity/15T/short,Xihe-max,141279.76029701577,143498.62141644108,55.28438256375424,1.0601449311684314,0.1643888935813446,0.16279208826736385,15.377549594244886,375.8719998842901,0.7409178123499444,0.10897641696871235,0.092251473,Energy,1 -electricity/15T/medium,Xihe-max,264156.09554458974,259977.8466385619,55.30601038367469,0.8610060398250966,0.14337324270636323,0.12597670451853854,8.555981065017582,513.9611809704987,0.8856068805414717,0.095297826,0.077271295,Energy,1 -electricity/15T/long,Xihe-max,330229.5804085785,321604.7113930943,59.31451138589762,0.8983954315844545,0.1436085689166225,0.12797291320754545,8.565390571490592,574.6560540084638,0.9065903880948144,0.093575915,0.075828266,Energy,1 -electricity/D/short,Xihe-max,1049007868.2346092,1011889762.8182234,4058.210878211449,1.411082304708791,0.2995668482155593,0.091769886,14.592334234697502,32388.390948526747,0.5322488391469123,0.066689884,0.053638383,Energy,1 -electricity/H/short,Xihe-max,864274.1619307427,850001.9212516496,103.05088228517825,0.5266777041050522,0.12747457594792713,0.076595659,5.095731011900033,929.6634670302705,0.4388675471508702,0.048647376,0.040923294130087565,Energy,1 -electricity/H/medium,Xihe-max,2048606.9022833966,1204818.8340964597,112.14038305884009,0.5035323510976801,0.15249524095773584,0.067965688,5.045506845956524,1431.2955328245096,0.558681808,0.043772086593359426,0.039023996,Energy,1 -electricity/H/long,Xihe-max,2619405.209735736,1235045.476,118.69111577984235,0.5395718823299412,0.20074142009180723,0.073588002,5.4892995669367215,1618.4576638688254,0.6383192238381162,0.046811741,0.042692657,Energy,1 -electricity/W/short,Xihe-max,35097124115.824234,34565362805.40917,22846.82000839814,1.235510299252794,0.22048633897336387,0.079233755,10.742859933041991,187342.2646276708,0.4274532271467912,0.052128904078536556,0.041332342895364776,Energy,1 -ett1/15T/short,Xihe-max,4.683601633707682,4.781754712,1.092014367239816,0.6921990942942566,0.46022970780082373,0.2418659214262653,5.882522751550575,2.164163033070217,0.40911689125598355,0.20643616784185598,0.16158814025473567,Energy,7 -ett1/15T/medium,Xihe-max,9.432516198536707,9.786245427,1.6470763385106646,1.040924167798281,0.7349869144818671,0.4059869877792498,9.168479425,3.071240172721226,0.5853813789965113,0.3139343601043129,0.24712975265863107,Energy,7 -ett1/15T/long,Xihe-max,9.055559508308532,9.358708844866072,1.6388253541976687,1.052520715561906,0.7997495229989506,0.406379035,9.346769654497372,3.0092456709794453,0.5735651663661423,0.3123617144290837,0.24524764852559736,Energy,7 -ett1/D/short,Xihe-max,42117.94022,41065.48566443939,139.09630686555175,1.766480509851547,1.1146336400647125,0.4962759509027479,13.361876014758181,205.22655827085708,0.539034363,0.3653410638540079,0.2850215832859098,Energy,7 -ett1/H/short,Xihe-max,102.75891345796131,105.52947242373511,5.006617010207403,0.826200412,0.49055025577545164,0.26038069130852565,6.593050895535586,10.137007125279203,0.4731355947506493,0.23367929878494126,0.1820624715996708,Energy,7 -ett1/H/medium,Xihe-max,137.95930989583334,144.55598958333334,6.701797340029762,1.2437675016941583,2958229594157.718,0.42909354591908244,10.742730548237745,11.74560811094229,0.5625867856483142,0.321000206,0.25320319821738657,Energy,7 -ett1/H/long,Xihe-max,147.24996693121693,158.49685846560845,7.101802248677249,1.3763136987074605,2691565298466.4575,0.4715527764014924,10.671366791147076,12.134659736936053,0.5811025729535424,0.34008992825336687,0.26585533280365375,Energy,7 -ett1/W/short,Xihe-max,1082630.1634454986,1099290.0436052172,831.2453031771456,1.4671958716619724,0.7350579233461362,0.4953991750206089,9.137850731076716,1040.495153013938,0.41403785566132795,0.3307723460883873,0.250431905,Energy,7 -ett2/15T/short,Xihe-max,8.479928734188988,8.522592381,1.8113107227143788,0.8047592651818412,0.12011417692619296,0.13820746604082343,8.072898992,2.912031719,0.1382445108618319,0.085989367,0.069044073,Energy,7 -ett2/15T/medium,Xihe-max,12.407130766369047,12.44155536954365,2.2174245101686507,0.9338748634823686,0.1453532325007967,0.17091069911638526,8.017184318229024,3.522375727597646,0.1877137691821878,0.11817056011926685,0.093305455,Energy,7 -ett2/15T/long,Xihe-max,12.964478546626983,12.868114769345238,2.297282211061508,0.9678366807698284,0.15054429966304175,0.1753189455353978,8.541695953865863,3.600621966636734,0.1918836495196575,0.12242631931916545,0.096677287,Energy,7 -ett2/D/short,Xihe-max,111518.1309145851,110034.0390692488,203.35709583949168,1.356286607,0.41431042316951705,0.14128539581126395,10.736582468118788,333.9433049,0.19112456421435373,0.11638663134493499,0.093705476,Energy,7 -ett2/H/short,Xihe-max,103.6254424,104.72493663969495,6.349913533528646,0.733628095,0.12150211194922982,0.11013003111014098,6.334682576557165,10.179658263397696,0.1291063849842115,0.080534568,0.06402786,Energy,7 -ett2/H/medium,Xihe-max,269.56839657738095,254.60595238095237,10.112509300595239,1.069215162168459,0.18085669024168105,0.17186806374606908,8.882452987314263,16.418538198554124,0.22125036059233416,0.1362725659369031,0.10863254137409324,Energy,7 -ett2/H/long,Xihe-max,272.0387896825397,262.48017526455027,10.237462797619047,1.1241418169681254,0.19676106009406535,0.18439072697453876,8.895539484445877,16.49359844553455,0.22586713630071006,0.14019417368011045,0.11113063866015922,Energy,7 -ett2/W/short,Xihe-max,5091021.085950224,5068739.698433199,1464.5697893979975,0.9710979370150499,0.15068958369269844,0.1624888231556683,6.871278590253486,2256.3291173829725,0.18927142131478952,0.12285495210715347,0.096495763,Energy,7 -hierarchical_sales/D/short,Xihe-max,28.0263432,29.23819309536514,2.3097752139243055,0.7463516292324174,0.6202140249506622,1.0553442438712648,9.022377929501571,5.293991235192584,1.6241378835408007,0.7086134564145258,0.5831236837436201,Sales,1 -hierarchical_sales/W/short,Xihe-max,454.78526476444836,466.18196994107353,8.954045607587014,0.720057644,0.5228763031039784,0.4579959825512172,8.624740035186845,21.325694942122013,0.9790730859660866,0.41108461359379145,0.34880283933145034,Sales,1 -hospital/M/short,Xihe-max,3625.7991953751225,3679.2723875942906,18.955264318864028,0.780203251,0.19797000087800695,0.17937506302585765,6.008306866588183,60.21460948453558,0.2185930151143975,0.068812011,0.054635547,Healthcare,1 -jena_weather/10T/short,Xihe-max,818.9422784933968,886.8362285273415,5.616072987753247,0.28751553599284596,0.3237754461593019,0.38452527817733495,2.8714063331566586,28.617167548403472,0.1777484794974158,0.034882852,0.029574733703625084,Nature,21 -jena_weather/10T/medium,Xihe-max,1565.760460104674,1611.9827397157965,9.924880149147727,0.6203327433754269,0.7260717457164849,0.49311269103655897,7.624565437041168,39.56969118030458,0.24283851569157974,0.060908819147809865,0.051739331128416916,Nature,21 -jena_weather/10T/long,Xihe-max,1448.080577773644,1543.9457315434854,9.609944693752066,0.6275846801682163,0.691323929,0.5034306739566203,7.841274962060287,38.05365393459141,0.23285683280540767,0.058804899226460226,0.049827689073264576,Nature,21 -jena_weather/D/short,Xihe-max,326.54342055796184,343.86417625628974,9.369091610909631,1.0149663454998223,0.705425677,0.43529661168996303,8.681070513696572,18.070512459749498,0.10881401989082348,0.056417245,0.044515592,Nature,21 -jena_weather/H/short,Xihe-max,1167.026200456229,1221.3614779466775,8.596583437302158,0.5171584578783028,1.3205445147482329,0.5097919941838875,5.659336185123844,34.16176518355322,0.20940944049520635,0.052696507867880625,0.043661286,Nature,21 -jena_weather/H/medium,Xihe-max,1292.7009424603175,1444.3326388888888,10.570181129092262,0.8089546807730316,2.1931493577552374,0.5972118381836673,7.105759041760581,35.95415055957125,0.22000982802761418,0.064680814,0.052214892,Nature,21 -jena_weather/H/long,Xihe-max,1476.605753968254,1687.5132936507937,11.981829737103174,0.9309834216893209,2.3060649174053185,0.5726754284159489,9.792603258878211,38.42662818890377,0.23137088994142524,0.072143894,0.058060309629235723,Nature,21 -kdd_cup_2018/D/short,Xihe-max,2798.321410722181,3003.2325670488976,21.66298484373411,1.2239837442467476,0.5047500690035901,0.48156760150434175,12.46302953591937,52.89916266560541,1.1849974323969104,0.48527387059320887,0.3897651870136743,Nature,1 -kdd_cup_2018/H/short,Xihe-max,4303.427507431873,4436.856930484852,22.475523223356706,0.945605209,0.8471968441082933,0.5021728205071521,9.390171496226875,65.60051454,1.3731375757110547,0.4704534055851605,0.38084126720785316,Nature,1 -kdd_cup_2018/H/medium,Xihe-max,4917.395795395208,5223.069044463849,25.86723231,1.0743486012716004,0.9277901361414451,0.5845969295515034,10.125915533731941,70.12414559476078,1.4678253806818382,0.5414480245637849,0.4341800600126724,Nature,1 -kdd_cup_2018/H/long,Xihe-max,3881.247314,4196.871132404895,24.67976654914724,1.048325225768913,0.9545433390706034,0.634880493,9.245315082984941,62.29965741827582,1.4618017125088103,0.579087053,0.46059911469724985,Nature,1 -m4_daily/D/short,Xihe-max,335489.8108412311,329900.6559382071,175.06100553852932,3.2812386485657647,0.035150766433400066,0.030779401303794626,30.727895040721254,579.2148227050401,0.089472682,0.027042087206341176,0.021356296670976868,Econ/Fin,1 -m4_hourly/H/short,Xihe-max,1110970.7971219935,1019983.358981468,194.72640797905615,0.7744276539957019,0.083597317,0.076690669,7.511595681,1054.0259945191074,0.1438978286730079,0.026584455638842885,0.021298994161390097,Econ/Fin,1 -m4_monthly/M/short,Xihe-max,1866884.2690082698,1889613.0199496422,554.9756314273126,0.9298277658609315,0.15509177207151656,0.13099568052272262,8.871162411059604,1366.3397341101772,0.2839723028174946,0.11534298837224542,0.093234627,Econ/Fin,1 -m4_quarterly/Q/short,Xihe-max,1781430.7627246699,1813919.0938162762,567.4218171829617,1.1941408917143235,0.11342826915133374,0.10458340805098662,10.513122871146383,1334.702499707208,0.2234056949089419,0.09497642,0.075930545,Econ/Fin,1 -m4_weekly/W/short,Xihe-max,288274.9003286394,297349.9546241077,271.71084702339243,2.150585273240526,0.063924511,0.064546089,16.056025699229043,536.9123767698408,0.097817109,0.049501503,0.038812459,Econ/Fin,1 -m4_yearly/A/short,Xihe-max,3455953.480705686,3522585.207593312,887.4889353,3.3284582612914773,0.1674226734447925,0.1514200831941846,37.12672520723157,1859.0194944393902,0.29810239162140384,0.14231296387006703,0.11613813839200383,Econ/Fin,1 -restaurant/D/short,Xihe-max,137.98522729416814,140.57899426598973,7.084838090506468,0.6800204241893651,0.6388644431147222,0.3933593039820845,5.462066765223969,11.746711339526827,0.5406108727017371,0.3260606643300297,0.2577514809536938,Sales,1 -saugeen/D/short,Xihe-max,1136.7445833333334,1299.9498958333334,13.671761067708333,3.037739104607543,0.3103393046061198,0.3505864342365721,41.379588159004086,33.715643006375146,1.0920577102482782,0.44283159849094383,0.3677954808954955,Nature,1 -saugeen/M/short,Xihe-max,411.44137477671546,430.63926588927654,12.894425055670752,0.7555269757230162,0.3543082868485412,0.3696013134786186,9.672913194813075,20.284017717817036,0.6089870633720866,0.3871293230840993,0.32600890261138943,Nature,1 -saugeen/W/short,Xihe-max,937.6302734,988.184375,15.009390258789063,1.2176192637701422,0.38748023509979246,0.39893393804474114,17.005084951561724,30.620749067217478,0.9243127725223652,0.45307092564970286,0.3813146503384794,Nature,1 -solar/10T/short,Xihe-max,24.39489491409339,24.732438188167674,2.2901127325647357,1.007020243635241,4.454455625223954,1.3155952365919041,10.256712703978689,4.939118839843134,1.4367891170163136,0.6661935372654462,0.5486101887069333,Energy,1 -solar/10T/medium,Xihe-max,22.294157516658373,23.738765308335406,2.013663624926126,0.879111796,2.8958603832804117,1.2097935424422432,10.035519367725984,4.7216689334025075,1.0477570961359193,0.4468399589224801,0.3666599803761932,Energy,1 -solar/10T/long,Xihe-max,20.940847399635036,23.001189220270682,1.9817128771606347,0.8631169025612137,2.0318637254427627,1.243496254512787,9.873492148657405,4.576117065770394,0.9890545076858577,0.4283155404296599,0.3473585172078025,Energy,1 -solar/D/short,Xihe-max,122081.0387449835,129059.37964211908,247.8095823722363,0.9670981233246094,1.086865646736774,0.4210585607213189,5.880530678459711,349.4009712994277,0.504765418,0.3580004572294081,0.2883828229645581,Energy,1 -solar/H/short,Xihe-max,653.7347854338743,657.3359950257716,10.852439424112003,0.8360415072527302,4.059328744270328,1.0749686347320797,8.835005388842356,25.56823782418089,0.9424537591220852,0.4000245304831342,0.3260290295241634,Energy,1 -solar/H/medium,Xihe-max,690.3404197080292,706.8772924270073,11.244993988556873,0.8585847271401199,4.202688444779036,1.1886911870443309,7.851355457324129,26.274330052506176,0.9464626930777348,0.4050709294122343,0.324720559,Energy,1 -solar/H/long,Xihe-max,720.8105991484185,749.2569900648824,12.671451981320965,0.9634456738032543,4.5506880224791395,1.199695554824788,6.937880841791617,26.847916104391018,0.9308663336501123,0.4393424056456485,0.3384275128458303,Energy,1 -solar/W/short,Xihe-max,1272287.4620644862,1269842.2574933993,880.1364718358428,0.964771616,0.19331544619254587,0.17323088364743516,5.773291058243982,1127.9572075502183,0.23024425566246204,0.17965785003435691,0.14146456068052113,Energy,1 -temperature_rain/D/short,Xihe-max,171.24074118765589,188.43248006968878,5.956156966522984,1.3708054539036223,25.965293896541144,1.4468768697293175,26.234013919350655,13.08589856248534,1.5405562412164802,0.7011971508588011,0.570111747,Nature,1 -us_births/D/short,Xihe-max,183687.73333333334,185055.14666666667,256.7973438,0.3781654010349133,0.024780575,0.024378858111298706,4.855872146318785,428.5880695181952,0.040176654,0.024072667186309186,0.019677576,Healthcare,1 -us_births/M/short,Xihe-max,73251468.20509897,79890660.89018841,7579.365011652815,0.8559174352593576,0.023305034909234117,0.023678674763163456,4.431343574921141,8558.707157339768,0.026583286078169104,0.023541456050735824,0.016783034847838385,Healthcare,1 -us_births/W/short,Xihe-max,3079889.4655481535,3150273.605110883,1389.4261538911555,1.263213700535578,0.018798167,0.018853562519207855,10.733624106233625,1754.961385771252,0.023823428730114008,0.018861324,0.014881743,Healthcare,1 \ No newline at end of file diff --git a/results/Xihe-max/config.json b/results/Xihe-max/config.json deleted file mode 100644 index 7e5399bb93e6ecf8084c7335d4fd6abc9dfc7a02..0000000000000000000000000000000000000000 --- a/results/Xihe-max/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "model": "Xihe-max", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "To your HF model link", - "org": "Ant", - "replication_code_available": "No", - "testdata_leakage": "No" -} \ No newline at end of file diff --git a/results/Xihe-ultra/all_results.csv b/results/Xihe-ultra/all_results.csv deleted file mode 100644 index 312d0b79e45abe68fa5500093aca57f9586f2fa8..0000000000000000000000000000000000000000 --- a/results/Xihe-ultra/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -loop_seattle/5T/short,Xihe-ultra,39.34097461050878,40.32860606285072,3.4896258807895846,0.5525362998379605,0.094297795,0.074050939,4.822610106944024,6.272238405107763,0.10755504105421235,0.059839379601151696,0.047383994,Transport,1 -loop_seattle/5T/medium,Xihe-ultra,70.70040965234779,73.64114866022479,4.972976076738261,0.7891794207190156,0.15848267904622143,0.11360184718339666,7.058404419723909,8.408353563709591,0.14960188775951558,0.088479463,0.070789544,Transport,1 -loop_seattle/5T/long,Xihe-ultra,85.22036113,90.05738919,5.416399793892794,0.8572834423092317,0.1683974280946231,0.12249189501806863,7.882430169533041,9.231487482220977,0.16321459728026513,0.095763062,0.076725057,Transport,1 -loop_seattle/D/short,Xihe-ultra,16.337564242380374,16.41358982946727,2.6972937100182213,0.8156672950509517,0.05026079,0.04986258,10.258198365673604,4.041975289679586,0.072235498,0.048204241715563236,0.040016383750949784,Transport,1 -loop_seattle/H/short,Xihe-ultra,55.85335249825968,56.65669669439879,4.064133133830489,0.8297236668002693,0.10991025101801177,0.093006592,9.400679467238401,7.473510052061192,0.13228872839318825,0.07193929,0.058627793705812184,Transport,1 -loop_seattle/H/medium,Xihe-ultra,61.60918698148462,62.63858120409451,4.3977422897291625,0.9104377164081833,0.11437020351028049,0.099454172,8.622050446128046,7.849151991233487,0.13873288396643974,0.077729603,0.061905687,Transport,1 -loop_seattle/H/long,Xihe-ultra,57.59299662,58.548485353914046,4.255037760332683,0.8954051820383957,0.12301649396100485,0.097998629,8.232442723421666,7.589004981932263,0.13562565711279403,0.076043209,0.060832148,Transport,1 -m_dense/D/short,Xihe-ultra,8126.747953292943,8172.730366933826,46.464174836391216,0.7043517277360063,0.12510775311764208,0.10280287337204327,7.8535984570922945,90.14847726552536,0.15606034259939686,0.080436356,0.065114109,Transport,1 -m_dense/H/short,Xihe-ultra,45465.97222222222,45216.50738715278,88.69915920681423,0.7916560735820326,0.28938815347836294,0.2094913044808564,12.060428467501541,213.22751281723055,0.37821251342712214,0.15733022206771616,0.1334740172149549,Transport,1 -m_dense/H/medium,Xihe-ultra,44500.24055555555,44613.32777777778,85.02190538194445,0.7031574416225225,0.26314022565800077,0.1934854074176172,8.870896652398848,210.95080126786803,0.36702776280543337,0.14792738180769122,0.12138226762979448,Transport,1 -m_dense/H/long,Xihe-ultra,37076.328641975306,36327.01753,82.43189236111111,0.7148072681487547,0.27660185698957396,0.19591595142835877,8.581513132,192.55214525415008,0.33333624562822173,0.14270179895121288,0.11891402503711118,Transport,1 -sz_taxi/15T/short,Xihe-ultra,16.630913570665573,16.843553703041547,2.7365195728633696,0.5462189537812815,848127617922.1898,0.3978250172352898,4.920805596460368,4.078101711662618,0.38131996929488093,0.2558762957076115,0.2036165907762556,Transport,1 -sz_taxi/15T/medium,Xihe-ultra,16.318150799386608,16.344076882436365,2.7502277523122136,0.5366250260285236,8.24306E+12,0.40636193496480416,4.366802200577904,4.039573096180661,0.375949692,0.25595459033288426,0.20328156169399011,Transport,1 -sz_taxi/15T/long,Xihe-ultra,15.819299133654075,15.615498829072774,2.696805781514074,0.5093306830530901,5304816053489.923,0.4019625261718791,3.809638651956226,3.977348254007194,0.3681362552517857,0.24961152962849423,0.19749012920230472,Transport,1 -sz_taxi/H/short,Xihe-ultra,7.180808700757921,7.264489279022027,1.8484015947661638,0.5636197936304952,1.0594996940319579,0.2998066327640939,5.040299545121094,2.6797030993671522,0.24963087985316815,0.17219001483128735,0.13770351219023647,Transport,1 -bitbrains_fast_storage/5T/short,Xihe-ultra,1740656.9357593034,1892030.2557939012,170.7166456,0.7214019874809705,2.098241080430293,0.5111373083253393,14.146971203244998,1319.3395831852024,4.142399983278953,0.5360080444022636,0.4214194016966749,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Xihe-ultra,3065239.988,3737769.4114507237,299.76141837666535,0.9761514890953042,4.911699487594148,0.5934678123536729,21.543849279856726,1750.782678685016,5.319361259380567,0.9107579686403086,0.6425679957485908,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Xihe-ultra,3925795.310068557,4620252.107380255,373.74073252946386,0.8848255286825402,5.188026568555966,0.617577207,16.35580747161677,1981.3619836033388,5.236148127033329,0.9876851644600814,0.6962672346450743,Web/CloudOps,2 -bitbrains_fast_storage/H/short,Xihe-ultra,3452800.4797025668,3081815.2541445065,344.74713438665026,1.0699317805783952,3.458673907185198,0.43965908487033417,24.590110366040065,1858.1712729731257,5.296391660568114,0.9826413066050114,0.7206755493589037,Web/CloudOps,2 -bitbrains_rnd/5T/short,Xihe-ultra,1509881.7125270413,1782566.6722925613,127.90360169965928,1.6547397376366055,0.8996768625453858,0.4797972762097815,53.94488883113205,1228.772441,5.027782132,0.5233446174647571,0.42800308223909583,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Xihe-ultra,2209326.1475203903,2262989.9485829254,146.68188255432295,4.382693427175122,0.4230648905367195,0.5191792713028323,161.0632905835478,1486.380216337795,6.253959180851816,0.6171654439302363,0.6129070867805502,Web/CloudOps,2 -bitbrains_rnd/5T/long,Xihe-ultra,2400978.5497138854,2344627.651069595,162.49573269282857,3.3021078255405243,0.8490898851438152,0.49751478259037724,116.49943273646113,1549.5091318588236,5.935820402,0.6224845439923974,0.6129977297103549,Web/CloudOps,2 -bitbrains_rnd/H/short,Xihe-ultra,1837965.7806080913,1772938.9382906691,146.92298148364438,5.8293048824970635,1.2349270134135604,0.4419023798861622,211.61750653367537,1355.7159660519203,5.933276495373147,0.643006865,0.6128791462198312,Web/CloudOps,2 -bizitobs_application/10S/short,Xihe-ultra,308054.3644444444,304969.92,297.6815277777778,1.0864350750778309,0.015135137769911024,0.015331965,11.058388250651394,555.0264538239996,0.021405152,0.011480386659316843,0.009110071,Web/CloudOps,2 -bizitobs_application/10S/medium,Xihe-ultra,744430.6133333333,1137550.2933333332,586.3482813,1.395300264927214,0.019421405792236328,0.019760717376114352,15.17861989,862.8039251958311,0.033879771,0.023024171,0.016975704558060425,Web/CloudOps,2 -bizitobs_application/10S/long,Xihe-ultra,5825904.071111111,6023439.928888889,1480.8793055555554,3.125557401569899,0.045064417521158855,0.046117891251138875,75.8035093,2413.690964293298,0.093086413,0.057111596,0.052742588,Web/CloudOps,2 -bizitobs_l2c/5T/short,Xihe-ultra,20.592596290225075,20.701712617420014,2.6786615280877974,0.27563674887882766,0.1408420770539473,0.1920023591734577,3.193351130745122,4.537906597785489,0.155850074,0.091996076,0.074610059,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Xihe-ultra,152.93063350340137,158.73026147959183,8.440059657,0.8267371123610818,0.5844651824300799,0.8971080968853361,10.363730534977945,12.366512584532527,0.6506604421324771,0.4440712699152879,0.3663172065816551,Web/CloudOps,7 -bizitobs_l2c/5T/long,Xihe-ultra,332.6765922619048,342.1043353174603,12.718284505208333,1.1896701400927294,0.9342604960444133,1.2036974400538856,15.879639587280348,18.239424120895507,1.0241163964002387,0.7141126610979567,0.6044385431661644,Web/CloudOps,7 -bizitobs_l2c/H/short,Xihe-ultra,72.27359579521315,77.55674885685316,5.234942212,0.5257759839227331,0.43369256770310394,0.5995372319468615,3.9084780165232087,8.501387874648065,0.4582466028044661,0.28217680689384883,0.21934974788809425,Web/CloudOps,7 -bizitobs_l2c/H/medium,Xihe-ultra,73.74225303104609,75.33625246839371,4.868167906332825,0.4947444858525362,0.4709387297384978,0.7279243468162165,5.144867957370987,8.587330960842612,0.5199759330857271,0.2947749610508805,0.2360158450982762,Web/CloudOps,7 -bizitobs_l2c/H/long,Xihe-ultra,84.46303721484117,94.26989930959664,5.621138409674368,0.5902358580942464,0.5954186480981857,0.7621828894157814,6.8877484359576355,9.190377425048503,0.5613747781846542,0.3433553576674284,0.2787145090927939,Web/CloudOps,7 -bizitobs_service/10S/short,Xihe-ultra,2805.463739976025,2919.575693617725,19.76095381,0.7909498339019101,0.063112086,0.051778554,7.981752010886218,52.96662855021098,0.039239372410971064,0.014639547,0.011556168755544056,Web/CloudOps,2 -bizitobs_service/10S/medium,Xihe-ultra,26531.98126984127,28202.31472222222,37.336243799603174,1.0327667301739094,0.075290192,0.067814741,11.719496561330356,162.88640603144657,0.12259214395396015,0.028100136,0.023266825,Web/CloudOps,2 -bizitobs_service/10S/long,Xihe-ultra,132517.27174603174,133449.8503174603,76.28880042989418,1.4023801133588474,0.094642025,0.090374194,26.34039755570615,364.02921825868833,0.2696844984640459,0.056517186,0.052416530268453235,Web/CloudOps,2 -car_parts/M/short,Xihe-ultra,1.3629633488962944,1.3584304776860094,0.4694027857674751,0.8537271281871612,0.8083668784520265,1.8379574559764493,18.76746463295968,1.167460212982136,2.799449832149264,1.1255797287262774,0.9744766195694426,Sales,1 -covid_deaths/D/short,Xihe-ultra,542669.2081748028,468878.28074409615,118.53235830444086,37.16358485,0.098963274,0.13409218218638003,873.5701250212493,736.6608501710966,0.2771722168122525,0.044598375,0.040718852333222394,Healthcare,1 -electricity/15T/short,Xihe-ultra,144240.57420615738,147404.5490538293,55.63040565765656,1.0594491433135707,0.1557191282034682,0.16080610365866752,14.691749191085608,379.7901712869323,0.748641301,0.10965849670286543,0.092759967,Energy,1 -electricity/15T/medium,Xihe-ultra,272739.35083165736,268842.5935965433,55.36527467607378,0.8620942065477268,0.14297060017191252,0.12583244569118712,8.550964297,522.2445316436136,0.8998799280432236,0.095399945,0.07778859,Energy,1 -electricity/15T/long,Xihe-ultra,345903.11261332233,338775.2666979988,59.846355300857496,0.9007781198316951,0.14785390454168199,0.12770199088235382,8.558542906509379,588.1352842784747,0.9278555266354562,0.094414964,0.076901067,Energy,1 -electricity/D/short,Xihe-ultra,1047135459.564124,1032558832.463216,4021.2353593849352,1.392854968419864,0.27193246660044607,0.089531972,14.262401859589756,32359.472485875354,0.5317736127548208,0.066082253,0.052883138,Energy,1 -electricity/H/short,Xihe-ultra,819618.7203934733,702661.2640844369,87.96363757709125,0.4413557195412455,0.10484083253877081,0.067224346,4.381596944984832,905.3279628916105,0.4273794513089091,0.041525119,0.035983388,Energy,1 -electricity/H/medium,Xihe-ultra,1683524.7636585021,898279.6811796171,89.30575314633481,0.39211313295173117,0.1339224812129966,0.057424678,4.590550455651751,1297.507134,0.5064597880850181,0.034858978,0.033395714,Energy,1 -electricity/H/long,Xihe-ultra,1924899.7185307809,916486.4880518018,93.67298233,0.42458047232902496,0.19120607616860252,0.062367064,5.055280110123524,1387.4075531475173,0.5471931285216745,0.036944596521012556,0.036251388,Energy,1 -electricity/W/short,Xihe-ultra,31689626613.704926,24473103888.00234,21629.090278191426,1.139842112,0.18469137420560905,0.07445832,9.831124301503692,178015.8043930508,0.4061733225130575,0.049350446670626213,0.039829833677869325,Energy,1 -ett1/15T/short,Xihe-ultra,4.260638482,4.3063309987386065,1.0728944097246442,0.690471116,0.42708784913363523,0.24443248513685717,5.6142740397934725,2.0641314110855564,0.39020675112869385,0.20282170004990951,0.15907384691091855,Energy,7 -ett1/15T/medium,Xihe-ultra,9.149213247147818,9.548173285590277,1.6298758758060516,1.0252467472794304,0.7142985386839823,0.4025024989122202,8.597291194310326,3.0247666434202523,0.5765234788848003,0.31065593509970535,0.24332451816962974,Energy,7 -ett1/15T/long,Xihe-ultra,8.694540395585317,9.039675990513393,1.6198936535063244,1.0361203536890342,0.7758445681348971,0.40255830417747673,8.662784855,2.9486506058848883,0.5620156877286555,0.3087533137719547,0.2410556896133796,Energy,7 -ett1/D/short,Xihe-ultra,37958.22771253912,37711.28509635299,132.8418844843328,1.698397637,1.2856150235990937,0.47305672187870146,12.05950182117837,194.82871377838308,0.5117240792733835,0.34891361600842646,0.2692610560183823,Energy,7 -ett1/H/short,Xihe-ultra,99.79998808361235,100.81412062872023,4.887616657075428,0.8132978201905047,0.4730857519876389,0.2559846116142078,6.568740335872557,9.989994398577625,0.4662739093415104,0.22812506545366265,0.17856865374373238,Energy,7 -ett1/H/medium,Xihe-ultra,138.99210379464284,141.1240699404762,6.796685500372024,1.271515120497746,2736731812412.6904,0.4372213012460566,11.957758806399232,11.789491244097128,0.5646886837018307,0.3255451239816686,0.25758118873972685,Energy,7 -ett1/H/long,Xihe-ultra,150.61878306878307,156.58411044973545,7.207098420965608,1.396815170728968,2394095846500.849,0.4811075564970007,11.584646782059975,12.272684427980012,0.5877122764669113,0.34513233388858044,0.2701140712720095,Energy,7 -ett1/W/short,Xihe-ultra,1118537.387486904,1114984.3101786522,828.9507365488582,1.504317575590085,0.6028105351522284,0.5328958880122039,10.443714771374802,1057.6092792174736,0.4208479749533822,0.3298592832608515,0.25311450817707165,Energy,7 -ett2/15T/short,Xihe-ultra,7.968356759207589,8.026721918015253,1.7525847662062872,0.7522992938019447,0.10570319397745719,0.1271988375154672,8.063022528022277,2.822827794819866,0.1340096830526044,0.083201437,0.06721522,Energy,7 -ett2/15T/medium,Xihe-ultra,11.788340308779762,11.88093036954365,2.147454194568452,0.8874042623701659,0.13433878852166006,0.16143173647253167,8.317444143,3.433415254346576,0.18297290476748243,0.11444171553029438,0.09114545,Energy,7 -ett2/15T/long,Xihe-ultra,12.47842540922619,12.336372922867064,2.254948459201389,0.9321516602255125,0.14232416590592187,0.16898038564694082,9.130651701713594,3.5324814803797895,0.18825231990378014,0.12017027720198453,0.095926079,Energy,7 -ett2/D/short,Xihe-ultra,108100.53309870197,105904.86688793018,200.24180913122615,1.3535025951469044,0.4125959780050286,0.1420188217500554,11.307518391563333,328.7864551630769,0.18817316302784134,0.11460367056772731,0.092895067,Energy,7 -ett2/H/short,Xihe-ultra,106.19329252697173,106.64000186011904,6.415441603887649,0.7391487451530276,0.12130242627891988,0.11103710466770694,6.8073925368384565,10.305012980436839,0.1306962315133268,0.081365646,0.064918661,Energy,7 -ett2/H/medium,Xihe-ultra,261.52769717261907,250.22892485119047,10.01505650111607,1.067258062400856,0.18141355753070307,0.1734918867395755,8.934020725,16.171817992193056,0.21792564715179116,0.13495932679437112,0.10835020144716596,Energy,7 -ett2/H/long,Xihe-ultra,241.64530423280422,237.54778439153438,9.704125331,1.0830957131923122,0.18830697099720978,0.18019866406971363,7.911778793543689,15.544944651969791,0.2128760527357232,0.1328905275573177,0.1059619792855713,Energy,7 -ett2/W/short,Xihe-ultra,4865259.677177786,4933087.408863495,1510.6623838511102,1.0055537332869773,0.15064093700097736,0.16200757502744315,6.265137834945838,2205.733364932803,0.1850272133643811,0.1267214141392286,0.096978845,Energy,7 -hierarchical_sales/D/short,Xihe-ultra,28.159158791882422,29.43654448569355,2.302524477464802,0.7431484433132276,0.6148441499280561,1.0580245740382166,8.863077976298529,5.306520403417141,1.627981693599927,0.7063890107658125,0.5818644309389445,Sales,1 -hierarchical_sales/W/short,Xihe-ultra,458.0234594695428,473.0339304961683,8.922949056782949,0.7175628054321711,0.5065322377724856,0.45766175268691167,8.756691952438885,21.40148264652575,0.9825525365457857,0.4096569557359224,0.3484804474257159,Sales,1 -hospital/M/short,Xihe-ultra,2632.0020592036635,2502.0169269857192,17.82671178,0.767976626,0.1943471041889709,0.17767517927982862,5.902434340882258,51.30304142254788,0.18624195365700566,0.064715103,0.051303862269502024,Healthcare,1 -jena_weather/10T/short,Xihe-ultra,852.8823357082549,912.5142005110544,5.807576629093715,0.2895513290626613,0.3250734222723997,0.35403822281453257,2.841655099995239,29.204149289240647,0.18139437183634416,0.03607233,0.030346302340889204,Nature,21 -jena_weather/10T/medium,Xihe-ultra,1565.6715193403058,1610.7880269985005,9.811163567877434,0.614209971,0.7105762719606802,0.4489670378687378,7.936057199656179,39.56856731472983,0.24283161854783494,0.060210942,0.05085375,Nature,21 -jena_weather/10T/long,Xihe-ultra,1378.3645688657407,1464.8181805968916,9.403868304863177,0.6160430266889381,0.6890062217914378,0.4351105353940802,7.855558531795256,37.12633255340124,0.22718239428268824,0.057543882,0.048397793,Nature,21 -jena_weather/D/short,Xihe-ultra,358.73418826043223,377.98197327288364,9.759493921671133,1.0360840764147228,0.6992073770920585,0.4492025888286172,8.893344012,18.940279519068145,0.11405143915626441,0.058768105,0.046278646,Nature,21 -jena_weather/H/short,Xihe-ultra,1117.528823,1146.7484338389104,8.38072225,0.5121741541179023,1.0150919220697359,0.36685848769918433,5.499009023279326,33.42946040657761,0.2049204589453812,0.051373293,0.042260626142665875,Nature,21 -jena_weather/H/medium,Xihe-ultra,1428.9564484126984,1577.7227678571428,11.039999534970239,0.8184800904335158,1.981482324943964,0.4546793499832719,7.3707688137793355,37.801540291537044,0.23131433365225276,0.067555716,0.053919094,Nature,21 -jena_weather/H/long,Xihe-ultra,1589.805191798942,1850.054794973545,12.363274016203704,0.9639472061702166,1.884502952508058,0.45252440185341386,10.76177176367806,39.87236125186144,0.24007580528684322,0.074440612,0.060101553771869605,Nature,21 -kdd_cup_2018/D/short,Xihe-ultra,2757.570635337626,3010.2399770848506,21.72747438112672,1.2234100180682035,0.5041918572545948,0.48092943484168904,11.426541218868598,52.51257597,1.176337479859773,0.4867185047306106,0.38806646988222016,Nature,1 -kdd_cup_2018/H/short,Xihe-ultra,4283.361326359506,4410.762774326932,22.369625439539472,0.9458608796544428,0.839509992,0.5003059568821207,9.233030405704833,65.44739357957279,1.3699324767747911,0.46823677318262913,0.37885680617596507,Nature,1 -kdd_cup_2018/H/medium,Xihe-ultra,4866.814232995309,5169.453232,25.51976239260311,1.060943294578158,0.9054029475243793,0.5758564478194169,9.918730083255399,69.7625561,1.4602566575306746,0.5341748499197597,0.42709019311192614,Nature,1 -kdd_cup_2018/H/long,Xihe-ultra,3831.5001640487226,4174.003635436868,24.158918492319003,1.0252590932213814,0.8978229028557092,0.6215887210991505,8.903214506,61.89911278886574,1.4524033169241166,0.566865853,0.450068242,Nature,1 -m4_daily/D/short,Xihe-ultra,395367.4990180084,356322.1734,173.79005759960728,3.246458191750085,0.035817587,0.030935586553347723,30.038756062985456,628.7825530483558,0.097129527,0.026845761,0.021349966950719668,Econ/Fin,1 -m4_hourly/H/short,Xihe-ultra,1189643.203,1141256.0389917884,195.5065312108638,0.7439293956315816,0.078980642,0.074230568,6.9514040881146455,1090.7076615030444,0.14890568640947974,0.02669096,0.021217149746001165,Econ/Fin,1 -m4_monthly/M/short,Xihe-ultra,1861687.684591674,1884883.1578736748,554.1433499230952,0.9283490505012215,0.15499396063782472,0.13084213194147845,8.848394636,1364.4367645998382,0.28357680042482325,0.11517001170367262,0.09301896,Econ/Fin,1 -m4_quarterly/Q/short,Xihe-ultra,1765062.289514122,1814001.521946145,563.0612539902414,1.1829328735162112,0.11233595486304529,0.10420648440574239,10.42340768071629,1328.556468319703,0.2223769574836663,0.094246539,0.075217666,Econ/Fin,1 -m4_weekly/W/short,Xihe-ultra,302164.47202565224,325163.26729211037,275.7665702153883,2.0732618992708964,0.063938987,0.065408315,14.651009066462853,549.6948899395484,0.1001458842748208,0.050240393,0.039268817,Econ/Fin,1 -m4_yearly/A/short,Xihe-ultra,3503978.194543601,3599349.066222891,898.3659318652897,3.3786454870854623,0.1680590909115393,0.15276711644068547,38.71500837613783,1871.8916086524885,0.30016649479171814,0.1440571406815687,0.11769839727380092,Econ/Fin,1 -restaurant/D/short,Xihe-ultra,137.6293524,141.3862269815212,7.082669923940781,0.6791176103861518,0.6307656485410431,0.3933847340531922,5.584243792021378,11.731553708360288,0.5399132833955744,0.32596088028108533,0.2573412677222934,Sales,1 -saugeen/D/short,Xihe-ultra,1259.6886458333333,1453.4838541666666,14.460890299479166,3.213345540430402,0.3046558634440104,0.36568684053251727,49.04846795439426,35.49209272265209,1.1495973398250232,0.4683916823302077,0.39481831273327983,Nature,1 -saugeen/M/short,Xihe-ultra,400.7090374751538,429.82669013536827,12.882558399880637,0.7547769059101183,0.35616022598106784,0.36981501972417696,9.354173562784688,20.017718088612245,0.6009919496119536,0.3867730504776461,0.32003004982143235,Nature,1 -saugeen/W/short,Xihe-ultra,922.8039063,974.8629883,15.692205810546875,1.273077472183066,0.4315593719482422,0.42107019762791165,16.638537558292164,30.37768763829795,0.916975761,0.47368228085793795,0.38721562260123754,Nature,1 -solar/10T/short,Xihe-ultra,23.484304044774557,23.712475162701015,2.2364191709643735,0.9863113424762433,4.812250012349236,1.134197072117013,9.684598488,4.846060673,1.4097185066613507,0.6505740862129794,0.5311131418858885,Energy,1 -solar/10T/medium,Xihe-ultra,20.828982716074982,21.94608005522838,1.9355253014145792,0.8480618409779851,3.1105616900292854,1.0292941784064018,10.051331849232554,4.563878035,1.0127426688008956,0.42950075448139474,0.35153238422087346,Energy,1 -solar/10T/long,Xihe-ultra,18.486019375221765,19.833804526719636,1.8234916841703417,0.7981416962771917,2.2153290073711807,1.0767716512020074,9.139279396,4.299537111739096,0.9292761745841127,0.39411856035040616,0.31907743326913696,Energy,1 -solar/D/short,Xihe-ultra,137741.55705058423,151342.47595774467,255.22337634549964,0.9952286512140318,1.1817010690281302,0.42114366682096427,6.537962339584063,371.13549688837935,0.5361644061871809,0.36871086482069365,0.30783974379546547,Energy,1 -solar/H/short,Xihe-ultra,679.7500630282367,683.0369153,10.889312325984541,0.8382970095109584,4.140658296079375,0.845854771,8.559950353619033,26.072016857700838,0.9610232220304261,0.4013836779229569,0.32673232032016214,Energy,1 -solar/H/medium,Xihe-ultra,700.3135530717761,727.2067290145985,11.125348806265206,0.8488050409128891,4.222109002543264,0.9666431793133483,7.410135252188334,26.46343804330375,0.953274804,0.4007610306928664,0.3194871062084086,Energy,1 -solar/H/long,Xihe-ultra,735.4945052716951,759.7573246147607,12.563765571142538,0.9534993997102684,4.173594417322932,1.033339556561243,5.625638831,27.120001940849765,0.9403000470168288,0.4356087209366763,0.33118782731384583,Energy,1 -solar/W/short,Xihe-ultra,1282316.5057157362,1233997.9218613966,846.5371859055591,0.9259448562764692,0.18488082538452424,0.16615775649046527,5.515354083060117,1132.3941476869863,0.23114994603118427,0.17279939607171935,0.13851561892296277,Energy,1 -temperature_rain/D/short,Xihe-ultra,172.74661158286605,193.36762446759374,6.004136919102857,1.3759068414149989,26.541821841024518,1.4569862056374001,24.692916243047343,13.143310526000139,1.5473151472473412,0.7068456598279975,0.5717960169197225,Nature,1 -us_births/D/short,Xihe-ultra,171756.8,172695.46666666667,244.73848958333335,0.36039609982303916,0.023612133661905923,0.023213101084301992,4.20966931,414.4355197132601,0.038849967022847716,0.022942247460141733,0.018689506,Healthcare,1 -us_births/M/short,Xihe-ultra,116122227.84624624,121569396.90059654,9216.367799512283,1.0382850650156994,0.028368682070555568,0.028955752195864593,5.472455663,10776.002405634765,0.03347019,0.028625975548883322,0.021726858901776905,Healthcare,1 -us_births/W/short,Xihe-ultra,2805580.456362487,2873316.185631762,1290.0598496131056,1.1728447853359485,0.017423548500697435,0.017469691,10.11065824502198,1674.9867033390105,0.022737780257967882,0.017512435962420286,0.013973287,Healthcare,1 \ No newline at end of file diff --git a/results/Xihe-ultra/config.json b/results/Xihe-ultra/config.json deleted file mode 100644 index 63ddee985b75bc883d21cee2120d4f48868b93a2..0000000000000000000000000000000000000000 --- a/results/Xihe-ultra/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "model": "Xihe-ultra", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "To your HF model link", - "org": "Ant", - "replication_code_available": "No", - "testdata_leakage": "No" -} \ No newline at end of file diff --git a/results/YingLong_110m/all_results.csv b/results/YingLong_110m/all_results.csv deleted file mode 100644 index e106cc19a2b198a2f4c3f46127c6d529ee71d4cc..0000000000000000000000000000000000000000 --- a/results/YingLong_110m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -car_parts/M/short,YingLong_110m,1.843900305255248,1.6918696857977946,0.5801768418848395,1.042972189100596,0.7036974853004412,1.8118402604620656,15.124621236334674,1.35790290715325,3.2561118770763025,1.3912045520443417,1.1609190089229378,Sales,1 -solar/D/short,YingLong_110m,120124.25498783456,131737.36642335766,249.5613290754258,0.9732760927275498,1.0896855041058393,0.4227962893001064,5.250237033751217,346.5894617379971,0.5007037223242208,0.3605311188333409,0.2842868608780193,Energy,1 -bizitobs_application/10S/short,YingLong_110m,1279220.0533333332,1388033.5644444444,529.7608333333334,1.6094207599473829,0.0231995264689127,0.0237628661261664,10.486369538670612,1131.0261063889432,0.043619156082872,0.0204307578270988,0.0160324211305041,Web/CloudOps,2 -bizitobs_application/10S/medium,YingLong_110m,6811315.626666667,6560801.706666667,1474.25625,3.6654339255789594,0.0516912905375162,0.0526861190795898,46.45101624411667,2609.849732583596,0.1024811177518351,0.0578897039420983,0.0493313648740927,Web/CloudOps,2 -bizitobs_application/10S/long,YingLong_110m,9162671.786666667,7306552.32,1622.5309722222223,4.3767350951622985,0.0592618391248914,0.0611039733886718,73.29213639082548,3026.990549484201,0.1167389262665924,0.062574534156966,0.0575771418541374,Web/CloudOps,2 -ett1/D/short,YingLong_110m,42552.828571428574,42676.48888888889,139.67012648809524,1.7278003705101963,1.2535656156994048,0.4905046735491071,10.87539346641944,206.283369594906,0.5418101127302961,0.3668482201264097,0.2799443289488829,Energy,7 -ett1/W/short,YingLong_110m,1051475.5714285714,1083364.857142857,812.9561941964286,1.5250727690609622,0.7231317247663226,0.5172080993652344,8.476241409068537,1025.4148289490315,0.4080370347825129,0.3234946731050985,0.2528383972571947,Energy,7 -hospital/M/short,YingLong_110m,2993.247501086484,2879.5432420686657,18.77489746305954,0.780682014214452,0.1940906828043513,0.175651129200789,5.249674961654481,54.71057942561461,0.1986121078867453,0.0681572375881279,0.054137476375947,Healthcare,1 -electricity/W/short,YingLong_110m,47025732067.4018,49203407619.22883,30903.73063063063,1.6906766671378068,0.2551323863093288,0.1062141109157253,13.11570548512226,216854.17235414637,0.4947896592008528,0.0705121150345822,0.0568398686961493,Energy,1 -hierarchical_sales/W/short,YingLong_110m,498.3057468220339,466.9772245762712,9.377093816207626,0.7420677178441749,0.5937939546080364,0.4801955142263638,6.707491989947868,22.32276297464169,1.024848967041315,0.4305069637798339,0.3687032562737851,Sales,1 -ett2/W/short,YingLong_110m,6688521.142857143,5669832.0,1705.195591517857,1.1859454986372289,0.176666396004813,0.1952486889702933,10.109138397730945,2586.217535873025,0.2169440021284078,0.1430397678866268,0.1128394040690246,Energy,7 -bizitobs_service/10S/short,YingLong_110m,6259.745185185186,6483.464973544974,27.44454365079365,1.0746929494411337,0.0941598333126653,0.0679244171626984,7.816723963908653,79.11855146035717,0.0586135522783464,0.0203317953165151,0.0158236069161317,Web/CloudOps,2 -bizitobs_service/10S/medium,YingLong_110m,79450.71746031746,85633.0361904762,70.39871527777778,1.8281844017773472,0.1630574738033234,0.1044094122023809,17.778647770566124,281.8700364712742,0.2121420317466209,0.0529837320715061,0.0448172043138526,Web/CloudOps,2 -bizitobs_service/10S/long,YingLong_110m,140205.97164021165,137503.37015873016,88.94500661375662,1.89335579626426,0.2043532856936177,0.1174223012772817,26.90652190611916,374.4408786981086,0.2773977735135658,0.0658933043998598,0.0601586077501251,Web/CloudOps,2 -us_births/W/short,YingLong_110m,3334228.571428572,3300954.571428572,1429.017857142857,1.2990108718894304,0.0192109942436218,0.0193279428141457,8.854691384000024,1825.987012940829,0.0247875946545639,0.019398777289189,0.0152737308948069,Healthcare,1 -us_births/M/short,YingLong_110m,50607674.66666666,46719130.66666666,5761.875,0.6502323111027395,0.017891256759564,0.0180254578590393,3.857575419670789,7113.907130871661,0.0220957470464813,0.0178963444660325,0.0135742571718928,Healthcare,1 -solar/H/short,YingLong_110m,628.0685987322321,690.0911048149571,11.383386507155205,0.8804431656373053,4.218507305138539,1.4016672154542604,6.0146237700365095,25.061296828620662,0.9237677734507826,0.4195954295563567,0.3430711896492029,Energy,1 -solar/H/medium,YingLong_110m,652.3963503649635,678.9728710462288,12.2689762013382,0.9391865615780406,4.646920398663614,1.406865948932927,6.913374973875051,25.54205062959831,0.920084279116056,0.4419571587027988,0.3672737353222672,Energy,1 -solar/H/long,YingLong_110m,677.4370235198702,645.0960259529603,13.07360604217356,0.9975936672777356,5.069801592934555,1.408540045807264,7.285826330718738,26.02762039679905,0.9024250331472882,0.4532857474521184,0.3695688623863586,Energy,1 -loop_seattle/H/short,YingLong_110m,60.858273416707405,61.76369935093151,4.378860739673293,0.884904762340182,0.1304836147839394,0.1008803440288966,7.167057891058617,7.801171284923015,0.1380886648590238,0.0775102880145323,0.0625666537078707,Transport,1 -loop_seattle/H/medium,YingLong_110m,66.9749613003096,63.77637383900929,4.805631208075336,0.9752768181998628,0.1354313680138996,0.1095736004619775,7.272618869116842,8.183823146935032,0.1446481417088102,0.0849389840793782,0.0675196517741423,Transport,1 -loop_seattle/H/long,YingLong_110m,66.06846835225318,62.81858875128999,4.76953528121775,0.9799117689366712,0.1578273147252322,0.111065071245055,7.561400268775225,8.12825124810086,0.1452627319847349,0.0852379809752005,0.0682400740590624,Transport,1 -bitbrains_rnd/5T/short,YingLong_110m,1429166.7504790674,1626556.2823995843,128.40966224866665,1.7586382192009924,1.232069256738781,0.6738116679194635,54.80158042631478,1195.4776244158932,4.891549304445742,0.5254152660226297,0.4293571966948413,Web/CloudOps,2 -bitbrains_rnd/5T/medium,YingLong_110m,2181518.301122295,2375060.339156109,184.90390293294303,4.513589574300016,1.7640844087507292,0.7407258189766511,162.0242568357637,1476.9963781683066,6.214475867189912,0.7779848749196192,0.6734731764383273,Web/CloudOps,2 -m4_quarterly/Q/short,YingLong_110m,2149980.7386666667,2117257.156,654.1768684895833,1.4733073524974245,0.1304903262456258,0.1188058980305989,12.473758867339992,1466.2812617866557,0.2454296628490938,0.1094976881048318,0.0882082277748772,Econ/Fin,1 -hierarchical_sales/D/short,YingLong_110m,28.503154712691195,28.95848232740802,2.333928986668045,0.7557288351703992,0.626868412230361,1.01677911197809,6.907991797036802,5.338834583754323,1.6378953262694147,0.7160235664054078,0.5847970303492365,Sales,1 -solar/W/short,YingLong_110m,5132242.686131387,5374631.7080291975,1965.0248631386864,2.1729912508262035,0.4503507126856894,0.3491745552007299,12.500431608881795,2265.445361541829,0.4624340372200045,0.4011107026127813,0.3172612244118224,Energy,1 -bitbrains_rnd/5T/long,YingLong_110m,2271092.1071532364,2337607.7655438865,192.9968373725689,3.4640875677908043,2.1271747879066623,0.7269518750794107,118.47955108327018,1507.0143022391117,5.773032517961116,0.7393274346235844,0.6479758135832705,Web/CloudOps,2 -jena_weather/10T/short,YingLong_110m,740.4624007936508,815.5128968253969,6.045567491319445,0.3244455062148568,0.340106691267744,0.5535157509893068,2.910167493160289,27.2114387858057,0.169017139538176,0.0375505511601541,0.0306395209585856,Nature,21 -jena_weather/10T/medium,YingLong_110m,1612.6080808080808,1556.7261183261182,10.586028814935064,0.6405198409986587,1.0609466420006453,0.6560839085345505,6.401859123674272,40.15729175141273,0.2464446129763671,0.0649662778656273,0.0557072030209336,Nature,21 -jena_weather/10T/long,YingLong_110m,1754.6976190476191,1692.8407407407408,11.351511863425928,0.647390345084517,1.0158162019373005,0.6491443095564889,7.031053205526741,41.88911098421187,0.2563266528997248,0.0694618475527007,0.059911383129092,Nature,21 -ett2/H/short,YingLong_110m,107.60275297619047,105.81012834821428,6.420584542410714,0.7430531378811844,0.1210455605198008,0.1112654731387183,5.256734516054753,10.373174681657996,0.1315607173193783,0.0814308766536893,0.0646473544645334,Energy,7 -ett2/H/medium,YingLong_110m,247.20969122023808,236.4450892857143,9.41319115048363,1.0296640197015383,0.1720335914613535,0.1707638331821986,7.271572223720213,15.72290339664523,0.2118762342906105,0.1268488041494753,0.1041337052597714,Energy,7 -ett2/H/long,YingLong_110m,273.4015211640212,259.3898148148148,10.023366195436507,1.0983183735154745,0.186180762808678,0.1862538332661623,7.549927260006394,16.534857760622593,0.2264321508653267,0.1372622854941352,0.1108865766785165,Energy,7 -sz_taxi/H/short,YingLong_110m,7.214532835870727,7.199728941306089,1.851638402694311,0.5647840552609904,1.2481388189853766,0.2953322973006811,4.051169520996518,2.6859882419457324,0.2502163796692724,0.1724915434637727,0.1361426119807096,Transport,1 -bitbrains_rnd/H/short,YingLong_110m,2108299.8555901283,2196922.7105408977,185.93737479220837,5.915073307974757,2.532121997766209,0.5655497275287612,200.3061379767857,1451.998572860913,6.35465644980072,0.813752960964861,0.6922823514290165,Web/CloudOps,2 -temperature_rain/D/short,YingLong_110m,168.42888534974455,179.74203293980327,5.994417313635806,1.4092067762158809,43.05256994026332,1.4657871271208651,18.447379060434574,12.978015462687065,1.5278555309201594,0.7057013973834304,0.5695113851444837,Nature,1 -saugeen/W/short,YingLong_110m,788.195947265625,846.0529296875,13.776004028320312,1.1173300535021184,0.3653525829315185,0.3688099384307861,10.117556462639788,28.07482764445091,0.8474620173780107,0.415840136691114,0.3386746927817969,Nature,1 -jena_weather/D/short,YingLong_110m,383.0036458333333,372.2569940476191,9.961922588045637,1.119835715322047,0.7614433688762426,0.4450140574621776,7.276908702603315,19.57047893724968,0.1178462696669062,0.0599870559875349,0.047642228426339,Nature,21 -bitbrains_fast_storage/5T/short,YingLong_110m,1646127.6025572307,1837058.534358384,173.4253635675557,0.7746741240390985,2.140447317106568,0.6970448224612723,14.6627237639006,1283.0150437766624,4.02834997747612,0.5445127575158513,0.4190319350630545,Web/CloudOps,2 -m4_yearly/A/short,YingLong_110m,5335253.9982008645,5441574.807463509,1210.3423761643596,4.716552389805132,0.2106766300993446,0.2040163579638429,46.482874293828445,2309.81687546889,0.370389842776649,0.1940840103709212,0.1588989513840055,Econ/Fin,1 -covid_deaths/D/short,YingLong_110m,2225042.157393484,2112747.340350877,242.95526315789476,46.26887410004859,0.1288096924356339,0.1849375042032819,1107.9592552876968,1491.6575201410958,0.5612434011933948,0.0914131001194397,0.0787579692151736,Healthcare,1 -saugeen/D/short,YingLong_110m,1083.0783333333334,1276.1467708333332,13.531979166666666,3.006842335499278,0.3131029764811198,0.3444833628336589,28.575796289368288,32.91015547415924,1.0659677771599811,0.4383040294110111,0.3600684836259978,Nature,1 -bizitobs_l2c/H/short,YingLong_110m,76.98164682539682,80.9236576140873,5.459219796316964,0.544079942342998,0.4506522345600082,0.6493920110887097,3.1917779476069663,8.77391855589034,0.4729366805562608,0.2942659283250179,0.2286716253990275,Web/CloudOps,7 -bizitobs_l2c/H/medium,YingLong_110m,101.10079985119047,100.93075706845238,6.324358258928571,0.6549118588142214,0.5744248679293306,0.8125947780217326,4.330943289945748,10.0548893505195,0.60883882267385,0.3829494987266835,0.3056147204016453,Web/CloudOps,7 -bizitobs_l2c/H/long,YingLong_110m,127.51450892857144,111.91805555555555,7.303777591765873,0.7759753162582762,0.8824020441286401,0.8202975802951389,5.61243380948337,11.292232238515618,0.6897621365177885,0.4461358153053192,0.3580713160303097,Web/CloudOps,7 -kdd_cup_2018/D/short,YingLong_110m,2641.803311390385,2844.666533679101,20.860599940155595,1.175281319525241,0.5278960250224417,0.4562750780262983,8.945472781917406,51.39847576913527,1.1513803983148565,0.4672995747201962,0.3749615411328965,Nature,1 -bizitobs_l2c/5T/short,YingLong_110m,21.63178013392857,21.427143787202382,2.7550249372209823,0.2880674767032388,0.1536247137713749,0.2131534785914942,2.7375406337990342,4.65099775681827,0.1597340887873169,0.0946187078435359,0.077275721016552,Web/CloudOps,7 -bizitobs_l2c/5T/medium,YingLong_110m,160.88086734693877,173.8452168367347,9.424848533163264,0.9018982754305164,0.5122450865934992,0.8432613865858843,5.950172826255396,12.683882187522036,0.6673587276290389,0.4958856312443436,0.3862613771152057,Web/CloudOps,7 -bizitobs_l2c/5T/long,YingLong_110m,298.20722222222224,352.9578968253968,13.497110615079364,1.2567234613394984,0.8347074233885171,1.088086232112377,9.350956603048376,17.268677489090535,0.9696104242932112,0.7578425828200893,0.5918333986329958,Web/CloudOps,7 -restaurant/D/short,YingLong_110m,147.13038199308863,149.52120108340338,7.436090291398151,0.7097982312540012,0.7350719648214953,0.405928857143691,4.923523323043857,12.12973132402728,0.5582383398368355,0.3422261044582373,0.2703437743348678,Sales,1 -bitbrains_fast_storage/5T/medium,YingLong_110m,2932373.6551262,3267816.9807411344,302.2288904408331,1.059909032669241,5.002733475089009,0.7522894046457049,22.88002887017856,1712.4174885600182,5.202797581522949,0.9182548944733104,0.6557298339121019,Web/CloudOps,2 -electricity/H/short,YingLong_110m,1944081.4857657657,2079194.669189189,207.620185106982,1.1389196928115577,0.1812231216702186,0.1434392298420292,10.57890264352921,1394.303225903808,0.6582106952284659,0.0980115543332733,0.079940864424704,Energy,1 -bitbrains_fast_storage/5T/long,YingLong_110m,3831983.867698639,4409853.582701519,400.0782705183567,0.995757746542128,5.283809294674965,0.7659855844544042,18.343332336100374,1957.545367979664,5.173207965674115,1.057287422193827,0.7196851456747015,Web/CloudOps,2 -ett1/H/short,YingLong_110m,99.30339471726192,104.8042224702381,4.941267903645834,0.8253969858404319,0.4627948579334077,0.2599838075183687,5.633980002590112,9.96510886630256,0.4651123997393038,0.2306291886274444,0.179656885094139,Energy,7 -ett1/H/medium,YingLong_110m,136.56222098214286,135.01561569940475,6.831343005952381,1.2686854868617772,3853813528516.11,0.4550558907645089,8.41479905221366,11.685983954384964,0.5597309298877828,0.3272051363436405,0.2535399540538676,Energy,7 -ett1/H/long,YingLong_110m,151.84737103174604,154.01362433862434,7.408285796957672,1.4119925613288111,3872398931859.4976,0.5106130125661376,8.918774871107171,12.322636529239432,0.5901043744074848,0.3547667615832361,0.2729411016142002,Energy,7 -electricity/H/medium,YingLong_110m,7619830.620540541,7453080.60972973,294.38128096846845,1.3000870172048071,0.2470112886589681,0.1519741369682371,10.45759363785584,2760.4040683458898,1.077476674986685,0.1149067150833084,0.0921588430756736,Energy,1 -solar/10T/short,YingLong_110m,25.458110933698297,27.031522772201946,2.5718477940617395,1.1395126606310668,4.472551976386037,1.558937234038504,6.638171239118119,5.045603128833886,1.4677654037728944,0.7481502447787418,0.5700685263101939,Energy,1 -electricity/H/long,YingLong_110m,10095027.529033031,9583277.077525526,329.76702702702704,1.444133612638563,0.3441131514403158,0.1687742574398107,11.59887223743284,3177.2673052535306,1.2531132854398166,0.1300600179230385,0.1037060848487748,Energy,1 -m4_weekly/W/short,YingLong_110m,337790.21727019496,322943.9451467752,287.17623741161344,2.312578906208,0.0718715578630544,0.0693259851207614,18.299805806386555,581.1972275142019,0.1058851278425693,0.0523190599888635,0.0416169408903756,Econ/Fin,1 -ett2/15T/short,YingLong_110m,7.498291596912202,7.571107700892857,1.729876999627976,0.7462979204359157,0.1085578173003524,0.1276543571835472,5.397103975973882,2.73830085945869,0.1299968831830147,0.0821234151334548,0.0647997474034762,Energy,7 -ett2/15T/medium,YingLong_110m,12.128210565476191,12.200902777777776,2.169119853670635,0.8945756200628378,0.1355653714114049,0.1622458418588789,6.65633130351552,3.4825580491179458,0.1855918223247291,0.1155963233937895,0.0912416193690419,Energy,7 -ett2/15T/long,YingLong_110m,12.159797867063492,11.908239087301586,2.1875968812003967,0.9064558057674303,0.139153641765881,0.1659342060391865,7.0699985720637,3.487090171914614,0.1858333473523007,0.1165809976366959,0.0927031899970379,Energy,7 -jena_weather/H/short,YingLong_110m,1083.419695071011,1138.2622180451128,8.647916177161655,0.5382559556516687,1.6086323952505086,0.6101880199083264,4.506082416898583,32.91534133304728,0.2017689233870139,0.053011169440773,0.0437685577017571,Nature,21 -jena_weather/H/medium,YingLong_110m,1434.5523809523809,1486.8956349206348,11.310623604910717,0.8849009643070249,3.0977005882841278,0.6892198372227872,6.145043432664278,37.87548522398599,0.2317668114200839,0.0692117118125467,0.0566642850114182,Nature,21 -jena_weather/H/long,YingLong_110m,1199.3948412698412,1184.2037037037037,11.636966765873016,1.0097983231584071,3.803772474315069,0.6546696309702974,9.875330096630517,34.63228033597905,0.2085246976150432,0.0700674328247747,0.0603954391211289,Nature,21 -m4_monthly/M/short,YingLong_110m,2124889.5944444444,2104369.640407407,624.3422008101852,1.0851097148969189,0.1816056700812445,0.147112749311659,10.271842581025457,1457.7001044262995,0.302960125247973,0.1297597432974679,0.1054375180146469,Econ/Fin,1 -m_dense/D/short,YingLong_110m,12796.386666666667,12255.59037037037,60.13559606481481,0.8671633461373934,0.1517253222511466,0.1262981499565972,7.978575124196496,113.12111503457994,0.1958293829296734,0.1041036120080754,0.0833182725876339,Transport,1 -solar/10T/medium,YingLong_110m,26.257030385976552,26.577802200840523,2.627392912934085,1.1379832379438883,2.623091047284714,1.4972573596180625,7.652865015652232,5.124161432466443,1.1370717280344853,0.5830288992861634,0.4569334412231867,Energy,1 -m4_daily/D/short,YingLong_110m,364378.0656324986,360539.007874548,183.8920455025516,3.543748673772196,0.0428049835985834,0.0340059473715273,31.538818489347257,603.6373626876476,0.0932452907395394,0.0284062390890253,0.0226415174904807,Econ/Fin,1 -bitbrains_fast_storage/H/short,YingLong_110m,2891610.768801035,2610107.800015423,300.8787582791387,1.1046303884623228,3.795777175365274,0.5500887496021343,20.674034815140484,1700.4736895350763,4.846902260900079,0.8576021744615674,0.6399073021233013,Web/CloudOps,2 -electricity/15T/short,YingLong_110m,146139.65227477477,148886.21432432433,55.86144320101352,1.0489393678544376,0.1730247960245589,0.1694501277593406,10.66752529448951,382.2821631658673,0.7535534968302381,0.1101139156310653,0.0909779068139793,Energy,1 -solar/10T/long,YingLong_110m,30.30442977620641,32.5395356536395,2.8617998926018857,1.2375559443485704,2.179261924975194,1.5324585701628723,8.354899939932718,5.504945937627944,1.1898058752920673,0.6185322008076112,0.4813229342949809,Energy,1 -m4_hourly/H/short,YingLong_110m,749210.383252818,717480.5539452496,211.8672252415459,0.9638390525024426,0.12359827996836,0.1075143123018568,7.537097839151907,865.5693982880969,0.1181693283349051,0.0289245527308261,0.022922751640237,Econ/Fin,1 -loop_seattle/D/short,YingLong_110m,18.943187242002065,19.119933565531475,3.118676631836945,0.9304386245067072,0.0582435757506127,0.0575321791956914,7.541500154312252,4.352377194361957,0.0777828014905089,0.0557349224422685,0.0443372585836093,Transport,1 -saugeen/M/short,YingLong_110m,404.8509579613095,425.13257998511904,13.560282389322916,0.7947187874178349,0.4134275345575242,0.4136600040254138,5.383273345142056,20.120908477534247,0.6040900346556595,0.4071203577936292,0.3223218449697684,Nature,1 -ett2/D/short,YingLong_110m,110078.70476190477,109161.5746031746,194.42872023809525,1.265202085176369,0.460411871047247,0.1295007857065352,11.129440348945582,331.78110971226914,0.1898870828376392,0.1112766864210114,0.0905975178997903,Energy,7 -m_dense/H/short,YingLong_110m,57181.484444444446,58352.39111111111,110.0059375,0.9481986678278248,0.3765221804507182,0.2631729125976562,8.327033240738814,239.1265030155471,0.4241508298442539,0.1951231214023421,0.1589397861698921,Transport,1 -m_dense/H/medium,YingLong_110m,56845.386666666665,46238.07111111111,112.59459201388889,0.9497984247475948,0.5246484551925824,0.2914479573567708,8.042056551789994,238.42270585383991,0.4148253866772913,0.1959003652888921,0.1616473374171873,Transport,1 -m_dense/H/long,YingLong_110m,72637.4162962963,58406.763456790126,127.50513117283953,1.0276977408385104,0.6006675419517632,0.3136951135706018,8.334321690062021,269.51329521249284,0.4665673553225138,0.2207302307459761,0.1803886234454,Transport,1 -loop_seattle/5T/short,YingLong_110m,47.50567424132482,49.14025836074561,3.851548163981876,0.6053939536032894,0.113768793308821,0.0814232156003591,4.735004991270783,6.892436016483926,0.1181900645187271,0.0660455497750451,0.0524655617089425,Transport,1 -electricity/15T/medium,YingLong_110m,282733.474027027,275577.52007207205,56.71742623873874,0.8804414343468178,0.1462577081588512,0.1354073623051717,7.434214337292104,531.726879165448,0.9162189567864776,0.097729836756855,0.0790904614363773,Energy,1 -loop_seattle/5T/medium,YingLong_110m,113.91570554373064,115.6213668246904,6.600064963074045,1.0335103121146776,0.240231029719145,0.1453582803039236,7.5971683231511635,10.673130072463778,0.1898969105068196,0.1174287146434775,0.0934372085902928,Transport,1 -loop_seattle/5T/long,YingLong_110m,124.5887650498796,124.61946451095058,6.909670192638459,1.0803133892896442,0.2572309386108244,0.1504683244825708,8.099115069579383,11.161933750469924,0.1973452714506879,0.1221643821119755,0.0975790263086008,Transport,1 -sz_taxi/15T/short,YingLong_110m,16.64501446361798,16.796248253157433,2.741521441747272,0.5468035215974539,1079318087152.523,0.3946966769785526,3.918143195430888,4.079830200341428,0.3814815724429811,0.2563439798049376,0.2018717640127759,Transport,1 -sz_taxi/15T/medium,YingLong_110m,16.41774172008547,16.29446781517094,2.7590465578258545,0.5375678696241388,10190647795473.94,0.4063753568209134,3.7108302648962153,4.051881256908384,0.3770951829455409,0.2567753348409959,0.2018226805281084,Transport,1 -sz_taxi/15T/long,YingLong_110m,15.671237313034188,15.4512775997151,2.6950200876958688,0.5080189682874582,6628394884256.456,0.3973311117232015,3.5659556213184054,3.9586913637001535,0.3664094209390731,0.2494462586314947,0.1962359118574886,Transport,1 -ett1/15T/short,YingLong_110m,5.136664109002976,5.460441080729167,1.140609232584635,0.7041810358768417,0.5033803079865247,0.2458350590297154,4.867918564445051,2.2664209911230038,0.4284479074510879,0.215622622996504,0.1676887519959624,Energy,7 -ett1/15T/medium,YingLong_110m,9.287917906746031,9.688941592261903,1.6029811507936509,1.009852841367202,0.7418127876548242,0.3889829799107143,7.195348188018159,3.0476085553669834,0.5808772039534426,0.3055297922770673,0.240882488272878,Energy,7 -ett1/15T/long,YingLong_110m,8.228581969246031,8.670805431547619,1.569231460813492,1.0137601444292446,0.8001605865673274,0.3896575055803571,6.783171848764059,2.8685504996855173,0.5467485614982224,0.2990970679970918,0.2329755020422297,Energy,7 -electricity/15T/long,YingLong_110m,342645.7764804805,337793.8654414414,61.17964283033033,0.9112326708689272,0.148884438010155,0.1351548392835597,7.609666871529001,585.3595275388285,0.9234763931119212,0.0965183844027915,0.0775482475996398,Energy,1 -us_births/D/short,YingLong_110m,358157.8933333333,347439.73333333334,388.5833333333333,0.5717365850482113,0.0364318148295084,0.036666940053304,4.616306166578116,598.4629423225246,0.0561010445960638,0.0364265343281222,0.0298180647071071,Healthcare,1 -electricity/D/short,YingLong_110m,1297677372.4436758,1266918193.723964,4237.877837837837,1.4186643866874746,0.5282984296377244,0.0975097330705704,11.631629748682268,36023.28930627624,0.5919822769855417,0.0696424069079309,0.055195447196494,Energy,1 -kdd_cup_2018/H/short,YingLong_110m,4418.228545304156,4548.484061050581,21.99192171600269,0.9172755726602436,0.8375542966641358,0.4853699780605646,7.743883423661001,66.46975662136997,1.3913323924162797,0.460330752062966,0.3736459314080033,Nature,1 -kdd_cup_2018/H/medium,YingLong_110m,4890.508440364557,5138.728227720904,25.029952653218835,1.0340430992906833,0.9328160726903276,0.5543136609066536,8.841430063525657,69.93217028209948,1.4638069770650932,0.5239222403879714,0.421088051598333,Nature,1 -kdd_cup_2018/H/long,YingLong_110m,3825.816441197321,4073.1029229967016,23.835689092517622,1.0066094026393546,0.998323822848604,0.6002705705380212,7.802349250918457,61.8531845679535,1.4513256521506386,0.5592815836127703,0.4437795527641178,Nature,1 diff --git a/results/YingLong_110m/config.json b/results/YingLong_110m/config.json deleted file mode 100644 index 76e1e72057e30ced6daf27e0e22377c3a97e6584..0000000000000000000000000000000000000000 --- a/results/YingLong_110m/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "YingLong_110m", - "model_type": "zero-shot", - "model_dtype": "bf16", - "model_link": "https://huggingface.co/qcw2333/YingLong_110m", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/yinglong.ipynb", - "org": "Alibaba", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/YingLong_300m/all_results.csv b/results/YingLong_300m/all_results.csv deleted file mode 100644 index f13f330c8dd017ddf526cd5f1953fd3c90199ab9..0000000000000000000000000000000000000000 --- a/results/YingLong_300m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -car_parts/M/short,YingLong_300m,2.140451681887912,1.8299107668772627,0.580173079388161,1.0651554318735337,0.7310368808628804,1.8206315619134603,15.415473223745524,1.46302825737848,3.508191683111761,1.3911955299632646,1.1909752401446398,Sales,1 -ett1/W/short,YingLong_300m,1266404.2857142857,1252840.5714285714,876.1472516741071,1.5891326412551992,0.7785827772957938,0.5494017260415214,8.218942503845803,1125.346295908191,0.4478021506247268,0.3486399031037617,0.2702477427520409,Energy,7 -hospital/M/short,YingLong_300m,3526.4856584093877,3309.5495436766623,19.362625964254672,0.7925509856820293,0.1971561012450429,0.1785367976474223,5.817898089598588,59.38422061801761,0.2155785107008762,0.0702908285263521,0.0571739399990139,Healthcare,1 -electricity/W/short,YingLong_300m,56024946654.78919,55650156780.16576,32040.56216216216,1.5985525104586955,0.2532723454411594,0.1020619658736495,12.953576291218274,236695.89488368656,0.5400619222233788,0.0731059894339027,0.0571041755428816,Energy,1 -us_births/W/short,YingLong_300m,2994969.1428571427,2996735.1428571427,1360.232142857143,1.2366136774693737,0.0183419585227966,0.0184230485132762,7.65173526097592,1730.5979148424808,0.023492697001202,0.0184650179625041,0.0146625855770976,Healthcare,1 -bitbrains_rnd/H/short,YingLong_300m,2016634.5495929413,2029023.1120583096,181.98897105835212,5.891558599317641,2.444099645239502,0.5708674190770602,199.5416414615088,1420.0825854833026,6.214976467442832,0.796472813640522,0.6726041862317288,Web/CloudOps,2 -sz_taxi/H/short,YingLong_300m,7.258240789429754,7.306476821247329,1.865656143579728,0.5678432184047901,1.2825355203742654,0.2966730818789229,3.9790141391573983,2.6941122451430557,0.2509731806994151,0.1737973825291552,0.136745560885467,Transport,1 -bitbrains_fast_storage/H/short,YingLong_300m,2828396.814641544,2454481.489242475,296.4950003855744,1.116041561674029,4.006003154639603,0.5545716818021557,20.78528469074592,1681.783819235262,4.793630060824541,0.8451070407959799,0.6314508563229755,Web/CloudOps,2 -loop_seattle/H/short,YingLong_300m,61.74249684291999,62.71841392374124,4.418428308823529,0.8946830528220908,0.1302929391788961,0.1014617551940649,7.371551018983021,7.8576393937950595,0.1390882078092375,0.0782106742252841,0.0630410510266255,Transport,1 -loop_seattle/H/medium,YingLong_300m,66.5091847265222,64.02369711042311,4.718248919633643,0.9658017991621825,0.1293049752281669,0.1070802418932211,7.47193216776216,8.155316347421612,0.1441442872752958,0.0833945120869589,0.0668446691847725,Transport,1 -loop_seattle/H/long,YingLong_300m,65.67417870657034,64.11391038871689,4.7273988433092535,0.980583866721561,0.1591987853940918,0.1101696873925008,7.602251006171908,8.103960680221144,0.1448286270162778,0.0844849464171062,0.0680189711717979,Transport,1 -solar/H/short,YingLong_300m,672.2013221923422,746.2710494301447,11.799409885628762,0.9113878939471538,4.744568416277159,1.4033726929429309,7.281014323939222,25.926845588932377,0.9556721899139048,0.4349301902698205,0.3553887270331152,Energy,1 -solar/H/medium,YingLong_300m,723.4724452554744,826.7110097323601,12.653674346107056,0.9667742024331428,5.460635722061639,1.4120378747476288,6.491292193523192,26.897443098842583,0.96890867935974,0.4558148837672371,0.3741601759997349,Energy,1 -solar/H/long,YingLong_300m,688.6034874290349,731.6656934306569,12.617222982562854,0.9605376905649284,6.969098705162071,1.4051437800137203,6.16670622014489,26.24125544689192,0.9098321496728228,0.4374621150409241,0.3519056717370039,Energy,1 -hierarchical_sales/D/short,YingLong_300m,28.81281779661017,28.70021444811906,2.3483247855518807,0.7628497507907794,0.6420465861655035,1.0103917974079126,6.993795589397193,5.367757240841855,1.646768477164764,0.7204400380791133,0.5886648287710353,Sales,1 -electricity/H/short,YingLong_300m,1984023.0149549549,2107041.9585585585,201.48073902027028,1.092141316358599,0.1818618676648489,0.1407964399552443,11.213333626226326,1408.5535186690474,0.6649378510823422,0.0951132973386854,0.0783453463677904,Energy,1 -m4_quarterly/Q/short,YingLong_300m,2063543.7866666664,2028244.752,629.7319420572917,1.3902340567998526,0.1276051349639892,0.1145791289011637,13.006706666329526,1436.504015541435,0.2404454761879113,0.1054060378812329,0.0858312457511847,Econ/Fin,1 -solar/D/short,YingLong_300m,114811.94355231144,128232.76690997567,249.15471411192212,0.9718423636674904,1.0682757375304135,0.4241992662712895,5.156159962169716,338.839111603592,0.4895071061831731,0.3599436987059138,0.2780932732986481,Energy,1 -jena_weather/10T/short,YingLong_300m,736.9854662698413,785.972123015873,5.79593990265377,0.3190417201037026,0.3542642892024339,0.5516509426646484,2.7715034516383183,27.1474762412612,0.1686198519709293,0.0360000509709435,0.0297604143954231,Nature,21 -jena_weather/10T/medium,YingLong_300m,1453.4891774891776,1474.2339105339106,9.960123556998555,0.6168747497369166,0.8123377555588925,0.6476998426857323,6.01765140831233,38.12465314582124,0.2339703446024747,0.0611251079977263,0.0505212199444347,Nature,21 -jena_weather/10T/long,YingLong_300m,1428.3079365079366,1394.7974867724868,10.071142319775133,0.6389360993754846,0.857084003881742,0.6403959273913475,6.651505448945648,37.79296146781748,0.2312616116361389,0.0616270467682569,0.0517809597026227,Nature,21 -electricity/H/medium,YingLong_300m,6697165.174774774,6783113.28,262.5517004504504,1.1394256712480204,0.238575305449455,0.1358960838391234,9.37409242522388,2587.888168908149,1.0101380342999615,0.1024825808864167,0.0823824332110503,Energy,1 -electricity/H/long,YingLong_300m,10788752.264072072,10874608.261765763,310.58578978978977,1.2904492927112967,0.3256845672558769,0.1510404453444876,10.5750235895188,3284.623610715857,1.2954545805609838,0.1224949436299781,0.0973975950166174,Energy,1 -bizitobs_service/10S/short,YingLong_300m,7097.78708994709,7754.496507936508,29.26458333333333,1.1375627888817756,0.1028141211454199,0.0737919624772652,9.36021751103007,84.24836550311875,0.062413882517257,0.0216801388985467,0.0168141396060885,Web/CloudOps,2 -bizitobs_service/10S/medium,YingLong_300m,79036.43936507936,83061.78539682539,71.17019345238096,2.024475608191864,0.199401138547867,0.1157342432415674,20.704316738600188,281.1342016992585,0.2115882251571748,0.0535643647256803,0.0454953792638194,Web/CloudOps,2 -bizitobs_service/10S/long,YingLong_300m,143507.8162962963,138727.52423280422,91.6943716931217,2.209092702254538,0.2256087239583333,0.1289170231894841,30.394555370464637,378.82425515837326,0.280645118928353,0.0679301219456458,0.0621107889211743,Web/CloudOps,2 -temperature_rain/D/short,YingLong_300m,169.8573192622022,179.59816848414107,5.988265373235231,1.4169223545586886,43.17501018061756,1.4562404630752914,18.35573106066468,13.03293210533233,1.5343206716380424,0.704977151354141,0.570969453004899,Nature,1 -jena_weather/H/short,YingLong_300m,1097.892439431913,1140.749582289056,9.048075918964075,0.5433205131930411,2.0868194362760684,0.611689395497154,4.561840662804369,33.13445999909932,0.2031121067037529,0.0554641228970155,0.0452479824277923,Nature,21 -jena_weather/H/medium,YingLong_300m,1460.507738095238,1471.3926587301587,11.443912760416667,0.8864643831135963,3.157411993222846,0.6841133754278699,5.952747439205969,38.21658982817852,0.2338540909890952,0.0700273317943319,0.0573387430363795,Nature,21 -jena_weather/H/long,YingLong_300m,1266.8092592592593,1151.557671957672,12.092056258267196,1.0297684460143597,3.487787066881547,0.6550576079496593,9.710695016466998,35.59226403671533,0.2143048630840619,0.0728075757743191,0.060212749301546,Nature,21 -ett2/W/short,YingLong_300m,5192896.0,3621361.142857143,1325.118024553571,0.920251381772724,0.1422298039708818,0.1493746212550572,9.130618127945622,2278.792662793173,0.1911557683875646,0.1111570869625623,0.0913238713292226,Energy,7 -m4_weekly/W/short,YingLong_300m,322603.114634669,310905.3627598029,284.32563745446754,2.2478891646814683,0.0702888353084991,0.0683257121561629,15.625143070590372,567.9816146977549,0.1034774479942459,0.0517997248533857,0.0409224454402673,Econ/Fin,1 -us_births/M/short,YingLong_300m,61678576.0,59888624.0,6444.708333333333,0.7297496794067668,0.0198117395242055,0.0200638261934121,4.488259551838323,7853.570907555365,0.0243931376938969,0.020017220143867,0.0147512790400836,Healthcare,1 -ett2/15T/short,YingLong_300m,7.8107073102678575,7.9362060546875,1.755917794363839,0.7526828731917684,0.1089503491888287,0.1278966540382022,5.480821304810825,2.7947642673878343,0.1326774020234941,0.0833596642985447,0.0656481454070023,Energy,7 -ett2/15T/medium,YingLong_300m,11.774804067460318,11.666077628968257,2.1428738839285715,0.887583082146435,0.1351627821972178,0.1633011881510416,6.635062649703288,3.431443437893202,0.1828678321684062,0.1141976281575843,0.0902405995156334,Energy,7 -ett2/15T/long,YingLong_300m,12.091335565476191,11.55820560515873,2.1818497333829363,0.9056944228890282,0.1384972828893669,0.1660144430493551,6.927418405885268,3.4772597782558887,0.1853094678799243,0.1162747217264114,0.0923770822217173,Energy,7 -bitbrains_fast_storage/5T/short,YingLong_300m,1623703.492605833,1833989.1113537804,174.27381623355913,0.8032211889881609,2.1813455291714123,0.7021580837758448,14.560358990451094,1274.2462448859064,4.000818117280552,0.5471766890843569,0.4236067899107265,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,YingLong_300m,2884643.2224165485,3210277.6705504307,299.1882810123437,1.071828860739069,5.004896097549624,0.7550169182353349,22.36133822426951,1698.423746423886,5.160280725540856,0.909016682713278,0.6451886415427616,Web/CloudOps,2 -m4_monthly/M/short,YingLong_300m,2115643.5334444446,2077160.3182962963,613.2118324652778,1.0481616578806128,0.1776379590564304,0.1442686033602114,10.328364197653602,1454.525191753118,0.3023002693982013,0.1274464706444784,0.1040263144155038,Econ/Fin,1 -m_dense/D/short,YingLong_300m,10192.354814814817,9848.373333333331,50.06791666666667,0.7452390651863391,0.1358169544352877,0.1083497789171007,7.522153103474118,100.95719298204963,0.1747718345769766,0.0866749697650207,0.0727910211679623,Transport,1 -bitbrains_rnd/5T/short,YingLong_300m,1399005.2873932128,1607149.0893850003,127.8545537579058,1.785717368673824,1.287260952103256,0.6904630002774508,54.44360319841742,1182.795539133122,4.839657872789569,0.5231439223383848,0.4254771941136492,Web/CloudOps,2 -bitbrains_rnd/5T/medium,YingLong_300m,2154692.144613578,2237569.5868353653,170.8998017476835,4.498291984870307,1.302720488480961,0.742535158557506,161.37266202016482,1467.8869658844917,6.176147931090124,0.7190624901772755,0.6523731293826035,Web/CloudOps,2 -electricity/15T/short,YingLong_300m,148046.51135135136,151065.45126126127,56.60763900619369,1.0720575790780826,0.1753974046930317,0.1731773346699035,11.37966539568751,384.76812673524734,0.7584538210441255,0.1115848146488403,0.0928686267327801,Energy,1 -bitbrains_rnd/5T/long,YingLong_300m,2495840.3812932014,2664985.362653644,219.90548587644668,3.4700286239149376,3.13265080653652,0.7299715676745755,117.8975914945948,1579.8228955465868,6.051945847533729,0.8424084091017047,0.6887456579557683,Web/CloudOps,2 -hierarchical_sales/W/short,YingLong_300m,526.165717690678,466.37486758474574,9.348125910354874,0.7470547667388,0.5960424784768493,0.474093744310282,6.830222255351081,22.938302415189277,1.053108683839589,0.4291770330528773,0.3712959863530989,Sales,1 -bitbrains_fast_storage/5T/long,YingLong_300m,3764342.004972065,4058291.128677089,387.3317370508165,1.009582292054786,5.223704907595844,0.7684563330215575,17.863852992642816,1940.191228969986,5.127346157498022,1.0236021398256,0.7091054073249635,Web/CloudOps,2 -saugeen/M/short,YingLong_300m,435.9256417410714,451.20191592261904,13.39521716889881,0.7850987702047343,0.3970942270188105,0.3903743198939732,6.467658630148655,20.87883238452456,0.6268451840939336,0.4021646046854742,0.3277683276184382,Nature,1 -ett1/D/short,YingLong_300m,44104.08253968254,45032.7492063492,142.73630952380952,1.7279010891714286,1.193524654327877,0.5022412496899802,10.298170440655811,210.0097201076239,0.5515974959562591,0.3749016515760436,0.284186477558998,Energy,7 -m_dense/H/short,YingLong_300m,56680.333333333336,58421.52,108.44579861111112,0.9290678683994812,0.3376354017763754,0.2654865180121528,8.735681521715637,238.0763182959056,0.4222880638408689,0.1923558237751465,0.1564930420470122,Transport,1 -kdd_cup_2018/H/short,YingLong_300m,4379.112130473512,4493.657248686884,22.089731872523508,0.927031062482016,0.8771597718772081,0.4876331401105727,7.713783069151611,66.17486026032478,1.3851596774796953,0.4623780957872693,0.3740945141201524,Nature,1 -m_dense/H/medium,YingLong_300m,41800.30666666666,36464.53333333333,91.74852430555556,0.7876295415627308,0.3532931748005365,0.2336693996853298,7.203348506611248,204.4512329790815,0.3557193157147895,0.1596308410972175,0.1340288035217407,Transport,1 -m_dense/H/long,YingLong_300m,48228.20345679012,40142.02074074074,99.05324074074076,0.8425839120449523,0.3943751161026657,0.2482007137345679,7.669552484776432,219.60920622048184,0.3801760001115421,0.1714758024538058,0.1450698737138724,Transport,1 -kdd_cup_2018/H/medium,YingLong_300m,4827.251245101868,5073.050808932262,24.9392565809393,1.0340730004504828,0.970128537306293,0.552104804941354,8.735375001149587,69.47842287431305,1.454309222045284,0.5220238073369216,0.4169610440086544,Nature,1 -kdd_cup_2018/H/long,YingLong_300m,3780.132152963716,4039.107903047205,23.72086670455411,1.0039256698315402,1.0027367671188725,0.5979061908619002,7.661119924565232,61.48277932042204,1.4426344482750642,0.556587386406003,0.4394137146588515,Nature,1 -m4_daily/D/short,YingLong_300m,371833.55571327184,372676.6711953767,182.73388389688736,3.51317346219464,0.0425897182039283,0.0342663411028246,30.821644341592496,609.7815639335711,0.0941944000342118,0.028227335127278,0.0225400668982757,Econ/Fin,1 -solar/10T/short,YingLong_300m,24.90353938564477,27.648199893552317,2.5009282095878955,1.10731808027984,5.283718417722162,1.5479865117989604,6.1093779056853625,4.990344615920304,1.4516907083504589,0.7275197453354145,0.553421584303218,Energy,1 -m4_yearly/A/short,YingLong_300m,5000833.433736688,4916918.619424857,1118.2171567133862,4.312498587797275,0.197400464768342,0.1904156140444542,51.570179669169846,2236.2543311834384,0.3585937478127971,0.1793113043998976,0.1515941545796118,Econ/Fin,1 -saugeen/W/short,YingLong_300m,825.80419921875,866.23046875,14.6233642578125,1.1859630934214436,0.4206714630126953,0.4009381294250488,10.372063312467995,28.736809134257584,0.8674444791022005,0.4414184098198283,0.360389737554416,Nature,1 -solar/10T/medium,YingLong_300m,19.27412076974121,21.211836429993365,2.034577224687569,0.8896344776528704,3.385049980445471,1.45993566937038,5.788836723467309,4.390230149974054,0.9742094680060194,0.4514807488376786,0.3482281957351236,Energy,1 -solar/10T/long,YingLong_300m,20.30869037662206,21.51395533949209,2.027292605212135,0.8863020744614358,2.819984049958087,1.4675204210037165,6.560452865681614,4.506516434744476,0.9740113330649068,0.4381668194287328,0.3512486118243954,Energy,1 -loop_seattle/D/short,YingLong_300m,17.843253353973168,18.015608875129,3.013710373129515,0.9072788780417246,0.0559316786696175,0.0555139689253579,7.063688757702453,4.224127525770638,0.0754908083870195,0.0538590350134818,0.0428155122248671,Transport,1 -ett2/D/short,YingLong_300m,110412.06349206348,110744.29206349206,198.483878968254,1.299525306770167,0.481592765687004,0.133729989188058,13.158700287625312,332.2831074431312,0.1901743893837749,0.1135975607540354,0.092369399197667,Energy,7 -ett1/H/short,YingLong_300m,101.64408482142858,107.17801339285714,5.043887183779762,0.8324057044261892,0.5000860305059524,0.259050550914946,5.5807663009249255,10.081869113484291,0.4705620781612315,0.2354188502641587,0.1820929093721376,Energy,7 -ett1/H/medium,YingLong_300m,134.91138392857144,134.96585751488095,6.794502766927083,1.2602670254898185,4084204535571.4873,0.4403334844680059,8.777061283906667,11.615135984075756,0.5563374800544929,0.3254405762238074,0.251884158597903,Energy,7 -ett1/H/long,YingLong_300m,139.63209325396826,138.84641203703703,7.119143208498677,1.3700696073732894,3946868136624.5366,0.4871406353339947,9.286321544286984,11.816602441225154,0.5658714979262243,0.3409203492613036,0.2635700437027675,Energy,7 -bizitobs_application/10S/short,YingLong_300m,1445730.1333333333,1696495.36,572.6089930555555,1.817812998221645,0.0260726356506347,0.0266502592298719,13.865050401736086,1202.385185093917,0.0463711905181336,0.0220832400785958,0.0167920703778449,Web/CloudOps,2 -bizitobs_application/10S/medium,YingLong_300m,7385415.68,6472782.933333334,1427.790625,3.8681272482165134,0.0535245768229166,0.0546028264363606,46.89840908821573,2717.6121283214798,0.1067126298687804,0.056065135604854,0.0482781477326485,Web/CloudOps,2 -bizitobs_application/10S/long,YingLong_300m,11097452.08888889,7829531.875555555,1706.585,4.599798576803993,0.0632481596204969,0.0646431435479058,80.28070935257594,3331.283849942675,0.128474302570193,0.065816162034804,0.0606065105006177,Web/CloudOps,2 -m4_hourly/H/short,YingLong_300m,680276.8180354268,711011.2979066023,221.3985507246377,0.9250319562832946,0.1259653825498817,0.1112879140365527,6.632736911403227,824.7889536332472,0.1126018975042944,0.0302257890415205,0.024848441847268,Econ/Fin,1 -electricity/15T/medium,YingLong_300m,278774.65377477475,275488.4064864865,56.76663063063063,0.8830048993986506,0.1490606893164232,0.1366740812645642,7.348494437699491,527.9911493337504,0.9097819181048782,0.0978146208788825,0.0788211182303169,Energy,1 -loop_seattle/5T/short,YingLong_300m,46.9253664376935,48.42047601627,3.857237383800148,0.6071451751382285,0.112301051136879,0.081556795354475,4.697483555905233,6.850209225833434,0.1174659682631413,0.0661431073375346,0.0524441375006667,Transport,1 -loop_seattle/5T/medium,YingLong_300m,110.54684412087204,110.4436410764964,6.519525495436661,1.022574347155637,0.2322959196013932,0.1443163168073561,7.40182114791904,10.514125932328946,0.1870679002010895,0.1159957520566482,0.0920612168122998,Transport,1 -electricity/15T/long,YingLong_300m,354041.6156996997,339702.3521441441,62.13003922672673,0.918390176709199,0.1527356066870213,0.1376696520456582,7.614968616067751,595.0139626090296,0.9387074476292172,0.0980177511934217,0.0781690330929683,Energy,1 -ett2/H/short,YingLong_300m,106.4244605654762,104.88314732142857,6.394745163690477,0.7407379492686809,0.1214849699767802,0.1111269905453636,5.261647960582421,10.316223173500862,0.1308384137338789,0.0811031614359429,0.0644934721112446,Energy,7 -ett2/H/medium,YingLong_300m,250.7732328869048,239.66867559523808,9.44057384672619,1.0184489875158365,0.1694063243810091,0.1668871924990699,7.150306133250036,15.835821193954695,0.2133978742240738,0.1272178035904968,0.1043022206096907,Energy,7 -ett2/H/long,YingLong_300m,261.07665343915346,244.61593915343917,9.685083912037037,1.0566042765999606,0.1798375119565762,0.177695154260706,7.226095618540751,16.157866611627707,0.2212695472336587,0.1326297699842532,0.1074672897321776,Energy,7 -solar/W/short,YingLong_300m,3862026.510948905,3956599.124087591,1628.1543111313867,1.7994194368063492,0.3761719195512089,0.2988925014969206,6.805938593758485,1965.203936223644,0.4011472559064694,0.3323470007686349,0.2550112813871233,Energy,1 -us_births/D/short,YingLong_300m,296733.0133333333,282991.52,343.6533333333333,0.5056032079818389,0.0324939346313476,0.0324668312072753,4.374781613832464,544.732056458341,0.0510642100472872,0.0322147114140432,0.026229993355089,Healthcare,1 -loop_seattle/5T/long,YingLong_300m,121.08967205595688,119.8049902534113,6.827265436876505,1.0697318088504102,0.2490078142916523,0.1493035153114608,7.929456457836664,11.004075247650611,0.1945543008369836,0.1207074491773945,0.0963152927793633,Transport,1 -ett1/15T/short,YingLong_300m,4.989907691592262,5.426839192708333,1.141652352469308,0.7168367160193663,0.5069723305352889,0.2472874959309896,4.930563499085433,2.233810128814054,0.4222830970424437,0.2158198160747358,0.1664695648975054,Energy,7 -ett1/15T/medium,YingLong_300m,9.615633060515872,10.269906374007936,1.6350288318452382,1.0285318121682634,0.745706922602835,0.395733894469246,6.911382373698324,3.10090842504513,0.5910362118139507,0.3116381119724082,0.2433109284140673,Energy,7 -ett1/15T/long,YingLong_300m,8.188919270833333,8.516012524801587,1.5882097904265873,1.0330601058192537,0.7845586517624605,0.3966448102678571,6.774528152155858,2.861628779355095,0.545429274759444,0.3027143563860313,0.234386699323145,Energy,7 -bizitobs_l2c/H/short,YingLong_300m,75.45882936507937,79.87349640376983,5.463115389384921,0.5544664321973307,0.438568115234375,0.6435810235464935,3.342718499167766,8.686704171610737,0.4682356018836918,0.294475910768167,0.2293816088144503,Web/CloudOps,7 -bizitobs_l2c/H/medium,YingLong_300m,113.69686569940475,110.27954799107142,6.904127139136905,0.7066118148803214,0.6698913210481228,0.827325504925879,4.58752951107538,10.662873238457108,0.645653170562967,0.4180553850416573,0.3300024212056354,Web/CloudOps,7 -bizitobs_l2c/H/long,YingLong_300m,160.58421378968254,147.89649057539685,8.480064174107143,0.8682625214808085,1.075153924125715,0.8582013811383928,5.829721432727708,12.672182676622151,0.7740534920595641,0.5179867947268668,0.4064934051278263,Web/CloudOps,7 -electricity/D/short,YingLong_300m,1282925632.1337657,1243548164.7602162,4178.923099099099,1.3960895808857603,0.4919655604049015,0.0964651131501923,11.42025889604222,35817.95125539379,0.5886078908799401,0.0686735847612102,0.0544394748229051,Energy,1 -restaurant/D/short,YingLong_300m,144.66475203138134,145.78415989539553,7.351071436910432,0.7028993806063948,0.7281346041870972,0.4021850939367937,4.760680038577257,12.027666109074584,0.5535410621619854,0.3383133397880059,0.2660543066940184,Sales,1 -covid_deaths/D/short,YingLong_300m,2211933.930827068,2010009.6,239.6091478696742,45.40439854073493,0.1270707583128245,0.179707526749991,1027.51135296984,1487.257183820965,0.5595877532384465,0.090154107958177,0.0782561074650183,Healthcare,1 -kdd_cup_2018/D/short,YingLong_300m,2642.2679699448104,2860.213046080192,20.99158853647184,1.1828928942779466,0.5280815777416218,0.4594672016818771,8.995735851027813,51.40299572928421,1.1514816502185126,0.4702338582751935,0.3737529937229737,Nature,1 -jena_weather/D/short,YingLong_300m,443.6748511904762,421.8808531746032,10.51873992435516,1.1223821038179145,0.7764330249147366,0.4529628208705357,7.463217046476573,21.063590652841604,0.1268372425727124,0.0633400064278558,0.0498921349319168,Nature,21 -saugeen/D/short,YingLong_300m,1158.5739583333334,1324.6688541666667,14.110291341145832,3.1354633168907866,0.3316891988118489,0.3666178385416667,31.62129070008756,34.037831281286614,1.1024934652389848,0.4570356985341836,0.3812078377644103,Nature,1 -sz_taxi/15T/short,YingLong_300m,16.783106541895606,16.917020506620116,2.7601179372114437,0.5506225498685873,1435920906467.7017,0.3976650820255862,3.961607921482223,4.096718997184894,0.3830607471782375,0.2580828316647546,0.2029748081706777,Transport,1 -sz_taxi/15T/medium,YingLong_300m,16.644449452457266,16.39372329059829,2.7757130158253207,0.5409075762721375,10721217586650.363,0.4071870983156384,3.82111000336121,4.079760955308199,0.3796898542357274,0.2583264269460883,0.2033638725101678,Transport,1 -sz_taxi/15T/long,YingLong_300m,15.92368011039886,15.55902666488604,2.7135422231125355,0.5112924792403588,7093442563239.458,0.3995969593015491,3.699541891371406,3.9904486101688943,0.3693488151024904,0.251160634491863,0.1984183486358434,Transport,1 -bizitobs_l2c/5T/short,YingLong_300m,21.18319847470238,21.35645461309524,2.7568990071614583,0.2863604562118306,0.154044101423466,0.2115164802187965,2.582952677078044,4.602520882592754,0.1580691967068341,0.0946830709909545,0.0769831298202451,Web/CloudOps,7 -bizitobs_l2c/5T/medium,YingLong_300m,156.47931547619046,165.9232674319728,9.161406515731292,0.8765213191442975,0.5612127561711044,0.8780798256802721,5.991860301799829,12.509169256037367,0.6581662581680209,0.4820247070447808,0.378771384090228,Web/CloudOps,7 -bizitobs_l2c/5T/long,YingLong_300m,281.7161507936508,324.4997619047619,13.082249503968256,1.2142850755593633,0.8502009243068485,1.1255081288206574,8.598363535277072,16.784402008819104,0.9424190800691136,0.7345487516496757,0.5755636810205136,Web/CloudOps,7 diff --git a/results/YingLong_300m/config.json b/results/YingLong_300m/config.json deleted file mode 100644 index a0ee2cb50c12fd9be6014d9c44bdd03487ad4723..0000000000000000000000000000000000000000 --- a/results/YingLong_300m/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "YingLong_300m", - "model_type": "zero-shot", - "model_dtype": "bf16", - "model_link": "https://huggingface.co/qcw2333/YingLong_300m", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/yinglong.ipynb", - "org": "Alibaba", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/YingLong_50m/all_results.csv b/results/YingLong_50m/all_results.csv deleted file mode 100644 index 45fccda0749752760a83b092903e9a1827899fde..0000000000000000000000000000000000000000 --- a/results/YingLong_50m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -,dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -0,car_parts/M/short,YingLong_50m,2.345947492672213,2.159477054478876,0.7046011231312068,1.2694769664624517,0.7896742665264872,1.8166425885386976,15.541603892619609,1.5316486191918215,3.672736271633272,1.6895612149756591,1.4066018375789853,Sales,1 -1,hierarchical_sales/W/short,YingLong_50m,506.3414327330509,469.1426112288136,9.835895085738876,0.7762142640077655,0.6525183068950571,0.4975210610082594,6.810471240877617,22.502031746779014,1.0330792840570182,0.4515707544804065,0.3884757395322085,Sales,1 -2,m4_weekly/W/short,YingLong_50m,341134.72809085064,329510.0955646025,285.9197289479323,2.2658492182747123,0.0715250489745989,0.0683844194234452,17.1217089367556,584.0674002979885,0.1064080288437343,0.0520901436193186,0.0414693148038312,Econ/Fin,1 -3,solar/H/short,YingLong_50m,644.0886156998336,693.0945703675246,11.36824447352414,0.8779149150095859,4.844241733675308,1.4027870115210337,7.285883170317518,25.378900994720667,0.9354747691167784,0.4190372891381436,0.3410760500052428,Energy,1 -4,solar/H/medium,YingLong_50m,699.8939172749392,783.696897810219,12.152376064476886,0.9282774260861631,6.186506391005668,1.4087745690411022,6.9437785740605165,26.455508259622214,0.9529891549700594,0.437756949627003,0.3593152234850635,Energy,1 -5,bitbrains_fast_storage/H/short,YingLong_50m,2779094.920323197,2396860.934049645,285.2965966634964,1.11194384477403,3.75212288793506,0.5462979873385828,21.42075720979008,1667.0617626000535,4.751667418280039,0.8131879533951912,0.623302913456145,Web/CloudOps,2 -6,solar/H/long,YingLong_50m,683.3649635036496,695.7158150851582,12.816001368613138,0.9753705831403884,7.868069870639642,1.4131793830220591,6.751444721076464,26.14125022839668,0.9063647864932614,0.4443541239486017,0.3515286755533442,Energy,1 -7,bizitobs_service/10S/short,YingLong_50m,8828.645925925926,8728.380105820106,30.63292989417989,1.1677389729333216,0.11160320302166,0.0703267738301918,10.776195731661758,93.96087444210984,0.0696092196403678,0.0226938537757618,0.0177720397177592,Web/CloudOps,2 -8,bizitobs_service/10S/medium,YingLong_50m,84027.80952380953,84558.35428571429,71.37823412698413,2.11970590850764,0.2086375984312996,0.1171220179966517,21.88575470696702,289.875506940151,0.2181671374712877,0.053720941039888,0.0466532159788119,Web/CloudOps,2 -9,bizitobs_service/10S/long,YingLong_50m,150605.20634920636,143200.4673015873,96.26845238095238,2.3440938635306883,0.2365248584242724,0.1303954819775132,29.702485428959225,388.0788661460533,0.2875012306105031,0.0713187471488748,0.0652027971221242,Web/CloudOps,2 -10,jena_weather/H/short,YingLong_50m,1058.9135338345864,1098.6013993316624,8.542908377715122,0.5349701924956606,1.5319804349726602,0.6067653079627356,4.510380017575527,32.54095164303875,0.1994739386892743,0.0523674783902325,0.0437365212110991,Nature,21 -11,jena_weather/H/medium,YingLong_50m,1433.5343253968254,1443.396626984127,11.270207093253967,0.8804535156623439,3.437548347597054,0.6806450719007342,5.971317705796436,37.862043333618765,0.231684558108966,0.0689643960097257,0.056707822484238,Nature,21 -12,jena_weather/H/long,YingLong_50m,1265.4759259259258,1147.4583333333333,11.866300843253969,1.0546677344811275,3.406120945809831,0.6630678835119225,10.351846591977424,35.573528443576215,0.2141920540557199,0.0714482780557197,0.060924276451579,Nature,21 -13,jena_weather/10T/short,YingLong_50m,929.2649801587302,1088.3414682539683,7.2913194444444445,0.3355857765130435,0.36007344982751,0.5732276958533923,2.867170582323744,30.48384785683609,0.189342901249565,0.0452882321166316,0.0369492828071498,Nature,21 -14,jena_weather/10T/medium,YingLong_50m,1749.5961038961038,1838.22886002886,10.89862012987013,0.6360092629119818,0.8894896365168352,0.6662731209017687,6.367873579534676,41.82817356634286,0.2566987861103216,0.0668846454215329,0.0577173812096614,Nature,21 -15,jena_weather/10T/long,YingLong_50m,1826.5833333333333,1982.88082010582,11.474210482804232,0.6418221135358174,0.8611013964834784,0.6545774939021869,6.865585264621856,42.73854622391048,0.2615244927853477,0.0702126614439889,0.0602355313198452,Nature,21 -16,ett2/H/short,YingLong_50m,107.4489490327381,106.79140625,6.4644891648065474,0.7520336049129952,0.1244616642814223,0.1131137393769763,5.303594440600407,10.365758488057596,0.1314666593496833,0.0819877094260368,0.0649365320886791,Energy,7 -17,ett2/H/medium,YingLong_50m,240.5033110119048,232.72336309523808,9.28452380952381,1.010321635022601,0.1702085426368962,0.1661948976062593,7.061515230128147,15.508169170211705,0.2089825524990052,0.125114928987171,0.102082007057541,Energy,7 -18,ett2/H/long,YingLong_50m,256.7726851851852,248.11107804232805,9.654962384259258,1.060695091675036,0.1827469019936677,0.1782821736007771,7.263948572087742,16.02412821919449,0.2194381028819546,0.1322172788445764,0.1068073671577059,Energy,7 -19,hierarchical_sales/D/short,YingLong_50m,29.1740104381976,29.092078338156263,2.364517233360893,0.7687472673995499,0.6495397235299631,1.006649990957007,7.098579794570063,5.401297107010278,1.6570581366736246,0.725407701746376,0.591013011304968,Sales,1 -20,m4_daily/D/short,YingLong_50m,360999.3035249586,360344.2695596336,193.09807170739128,3.708535578366285,0.0435554471905981,0.0339308457044369,31.566226549101973,600.8321758402745,0.0928119668942469,0.0298283157249116,0.023740030190624,Econ/Fin,1 -21,ett2/15T/short,YingLong_50m,8.028053501674107,8.161034574962798,1.7920915876116073,0.7730018202384242,0.1120487116501903,0.1315044766380673,5.499145219882313,2.8333819900737187,0.1345107227002379,0.08507696295069,0.0670705947508995,Energy,7 -22,ett2/15T/medium,YingLong_50m,12.2265091765873,12.068477182539684,2.1940415736607144,0.909095213257458,0.1392216305434914,0.1669336906312004,6.4364931291520415,3.4966425577383946,0.1863424112839223,0.1169244469636455,0.0919309167813532,Energy,7 -23,ett2/15T/long,YingLong_50m,12.33646949404762,11.64891121031746,2.199874441964285,0.9193657819633304,0.1421542997484582,0.1693870132688492,6.754968759117255,3.512331062705738,0.1871784801118855,0.1172352910738002,0.0931966929661242,Energy,7 -24,us_births/W/short,YingLong_50m,2937836.285714286,3116259.714285714,1357.3392857142858,1.2339246271767508,0.0182938064847673,0.0183947086334228,7.413372415602534,1714.0117519183718,0.0232675414658539,0.0184257477104473,0.0143340729500147,Healthcare,1 -25,ett2/W/short,YingLong_50m,6623990.285714285,4113258.5714285714,1388.510602678571,0.9505465144564608,0.1422410437038966,0.1490610837936401,12.31331514823101,2573.711383530462,0.2158949276778677,0.1164747523997943,0.109217580130966,Energy,7 -26,bitbrains_fast_storage/5T/short,YingLong_50m,1625323.5851269332,1807582.7612112092,171.9134607317288,0.8170880115615706,2.188540019955796,0.6879133037170319,14.894242836302633,1274.881792609391,4.002813579976022,0.5397657564700009,0.4157287325802631,Web/CloudOps,2 -27,temperature_rain/D/short,YingLong_50m,172.66202881747603,177.03384917584646,6.184557662395261,1.4583460504277337,55.47725823619599,1.4426676192473649,18.659867846409888,13.140092420431298,1.5469362738146486,0.7280859433364272,0.5855959025173892,Nature,1 -28,saugeen/D/short,YingLong_50m,1106.04375,1263.1034375,13.639124348958331,3.030752481941585,0.3296083831787109,0.3492827606201172,29.81457571490539,33.2572360547295,1.0772097998619516,0.4417744873944299,0.3670472305631288,Nature,1 -29,saugeen/W/short,YingLong_50m,912.7845703125,931.8578125,15.276841735839843,1.2393039893169084,0.4460285186767578,0.4208316326141357,11.57681965079929,30.212324808139144,0.9119841466469698,0.4611441708771578,0.3761910558095485,Nature,1 -30,bitbrains_fast_storage/5T/medium,YingLong_50m,2870817.369644257,3254084.6308887354,298.25332927733194,1.0843732914669508,5.019108437274568,0.7493626578769188,22.878649997043716,1694.3486564589523,5.147899476017486,0.906176040954912,0.643268657087845,Web/CloudOps,2 -31,kdd_cup_2018/H/short,YingLong_50m,4438.921620014656,4562.074270052173,22.455049457146558,0.933982231959928,0.8712875178771664,0.4929287654149919,7.8220884471991505,66.62523260758385,1.3945867863972063,0.4700248544763467,0.3798612742472347,Nature,1 -32,kdd_cup_2018/H/medium,YingLong_50m,4856.280447402948,5070.645750417959,24.90104127814369,1.0303783760583116,0.9588919119228289,0.5491351752796269,8.970134709056229,69.68701778238862,1.458675491254284,0.5212238918382718,0.4193356351147657,Nature,1 -33,electricity/H/short,YingLong_50m,1963977.596036036,2063356.6875675677,202.36813837274772,1.0953413361090494,0.1902020470908761,0.1394756007306852,10.844345889803993,1401.4198500221253,0.6615702500380454,0.0955322132056848,0.0786484151989655,Energy,1 -34,kdd_cup_2018/H/long,YingLong_50m,3813.569594741067,4078.6333335286295,23.86171278584025,1.0096902984019391,0.9942768613202848,0.601537233567298,7.891046642279318,61.75410589378707,1.4490008660882356,0.5598922046171155,0.4445782346906876,Nature,1 -35,bizitobs_l2c/H/short,YingLong_50m,95.82296316964286,101.37220982142856,6.389646499875992,0.6454355930506707,0.527434936816172,0.7312637518416564,4.273780109539138,9.788920429222156,0.5276478809935692,0.3444183105108198,0.268816795141293,Web/CloudOps,7 -36,bizitobs_l2c/H/medium,YingLong_50m,129.63949032738094,134.55741257440476,7.505065336681548,0.7737421733293213,0.6573928147968812,0.8902070027584474,4.899897526602889,11.3859338803359,0.6894355906947746,0.4544431056756912,0.3582639308630035,Web/CloudOps,7 -37,bizitobs_l2c/H/long,YingLong_50m,143.0390625,121.79383680555556,7.483891369047619,0.7900304220953858,0.8887128190205733,0.8322667681981647,6.230490059681806,11.95989391675361,0.7305448388148346,0.4571376846620619,0.3770265652827749,Web/CloudOps,7 -38,restaurant/D/short,YingLong_50m,149.56512094891193,150.78612122910246,7.516571372700103,0.7169512887675535,0.747977100991174,0.4093964885705613,4.910284339555336,12.229681964340362,0.5628382998873502,0.345930030292545,0.2713240697110609,Sales,1 -39,sz_taxi/15T/short,YingLong_50m,16.772697029532967,16.85576893267323,2.7634824722532243,0.5513067916951179,1255242130478.4165,0.3971661275414174,3.967935671947546,4.095448330712154,0.3829419344286209,0.2583974300806815,0.2032656653026313,Transport,1 -40,sz_taxi/15T/medium,YingLong_50m,16.9963374732906,16.79073684561966,2.8291869073851497,0.5522790594283759,11016456208215.96,0.4117452768179087,3.994914936733256,4.122661455090704,0.3836824618143321,0.263303065115379,0.2082080492337384,Transport,1 -41,sz_taxi/15T/long,YingLong_50m,16.179738693019942,15.89880698005698,2.7589086983618234,0.5209434231991072,7478219016552.588,0.4030867791243768,3.936516157070564,4.02240459091573,0.3723066037566794,0.2553596746288545,0.2043178718052039,Transport,1 -42,jena_weather/D/short,YingLong_50m,435.4188988095238,373.4659226190476,9.950619264632936,1.0802152985573867,0.8180907067106863,0.4451001364087301,7.25892292119835,20.866693528432428,0.1256516000703413,0.059918991506165,0.0489986381024393,Nature,21 -43,kdd_cup_2018/D/short,YingLong_50m,2654.8831704235654,2825.608085643992,20.737810858434734,1.1712600846547196,0.5410068262621351,0.453783898842177,9.20092703933402,51.52555841932784,1.1542271845319196,0.4645489689929856,0.3718150430487579,Nature,1 -44,m_dense/D/short,YingLong_50m,11828.093333333334,11316.230370370373,56.79396990740741,0.8310552920904485,0.1478149323573017,0.1208352322048611,7.51145069906171,108.75703808643068,0.1882745202008855,0.0983187628383452,0.0802924636732568,Transport,1 -45,bitbrains_fast_storage/5T/long,YingLong_50m,3655905.308869216,3892526.2015919313,378.6719158481997,1.007303638308903,5.08547967450603,0.7616484369449431,18.311073355116317,1912.042182816377,5.052956632652286,1.0007168178506989,0.6903949173762876,Web/CloudOps,2 -46,m_dense/H/short,YingLong_50m,54753.804444444446,55595.96444444444,108.64254340277778,0.9371542769194656,0.3416542823024893,0.260730726453993,8.538897815674979,233.9953085949469,0.4150493695538056,0.1927048000099034,0.1575588611417751,Transport,1 -47,m_dense/H/medium,YingLong_50m,52862.44,46354.77333333333,107.02614583333332,0.8893852585453887,0.4622745488663682,0.2679356384277344,8.161983479242796,229.91833332729252,0.4000288528944082,0.1862119724331516,0.156467421056435,Transport,1 -48,m_dense/H/long,YingLong_50m,64903.02814814815,55913.53283950617,122.08861882716047,0.9842398728974996,0.5558742264659896,0.2963102213541667,8.394663164590755,254.76072724842845,0.4410284793506166,0.2113534471694824,0.1747923377750923,Transport,1 -49,bizitobs_application/10S/short,YingLong_50m,1780789.9022222222,1970887.68,599.4136111111111,1.885777229108565,0.0269266404045952,0.0274116579691569,15.091735876583115,1334.4624019515209,0.0514648808446078,0.0231169870558781,0.0176288755103418,Web/CloudOps,2 -50,bizitobs_application/10S/medium,YingLong_50m,8732044.8,6254582.613333333,1417.6616666666666,4.080549996140289,0.0554265276590983,0.056578197479248,44.76144069936248,2955.003350251908,0.1160342844698754,0.0556674012224096,0.0489637633139335,Web/CloudOps,2 -51,bizitobs_application/10S/long,YingLong_50m,12305676.515555557,8048074.524444444,1721.628611111111,4.865825362311909,0.0655826059977213,0.0675031195746527,67.58463998206184,3507.944770881599,0.1352874081569255,0.0663963339843274,0.0601136315919493,Web/CloudOps,2 -52,ett1/H/short,YingLong_50m,97.85457589285714,100.60135788690476,4.928938802083334,0.8304398933425667,0.4636341276622954,0.2600812457856677,5.630882199233684,9.8921471831376,0.4617069794873857,0.2300537390170789,0.1793112094888531,Energy,7 -53,ett1/H/medium,YingLong_50m,133.38772321428573,132.90990513392856,6.761934988839286,1.2665184884028238,4272706268616.796,0.4422677902948289,8.465435611100284,11.54936029459146,0.5531869804489171,0.3238806568550475,0.2512376460359611,Energy,7 -54,ett1/H/long,YingLong_50m,135.28277116402117,136.3495701058201,6.983536396329365,1.3444550410010925,3239410691356.6665,0.4766467543506117,9.208496499410815,11.631112206664554,0.5569887723370733,0.3344264327304795,0.2592799199082443,Energy,7 -55,ett1/W/short,YingLong_50m,1364905.7142857143,1400135.7142857143,923.1282087053572,1.733683991631802,0.850813456944057,0.5598954132625035,9.790559825487488,1168.2917932972543,0.4648911890481847,0.3673347472361833,0.2902033017798772,Energy,7 -56,electricity/H/medium,YingLong_50m,7920814.653693694,7897946.254414414,272.0144481981982,1.1566413945307807,0.2453377881846304,0.1373175527187671,9.572521634062108,2814.394189464883,1.0985507984646588,0.1061762031703422,0.086222377009911,Energy,1 -57,m4_yearly/A/short,YingLong_50m,5003386.283624968,5004212.268245263,1134.3502510083863,4.356100991573442,0.2020312156753626,0.1948036310521314,44.14944905738486,2236.82504537681,0.3586852644791607,0.1818983208525387,0.1503592848334618,Econ/Fin,1 -58,ett1/15T/short,YingLong_50m,5.336710611979167,5.779307919456845,1.1641288394019718,0.7179857460793408,0.5370524839818158,0.247701172601609,4.932365771034554,2.3101321633142917,0.4367111384796321,0.2200688076922992,0.1699979607237043,Energy,7 -59,ett1/15T/medium,YingLong_50m,9.55883308531746,9.986096230158733,1.6359509858630952,1.0279384672562828,0.7153626003996103,0.4004739040798611,7.047193897866914,3.091736257399305,0.5892879875917432,0.3118138754399848,0.2452464456609152,Energy,7 -60,ett1/15T/long,YingLong_50m,8.809058159722222,8.916981026785715,1.6117877294146823,1.0351120073786877,0.7736472612527088,0.3960380239335317,7.09317100956946,2.9680057546646066,0.5657048314328871,0.3072083348696756,0.2422142447896858,Energy,7 -61,electricity/H/long,YingLong_50m,12860407.7587988,12703120.227171171,330.4777117117117,1.3246994223916422,0.3215943027644249,0.1537320356114761,10.851398147439134,3586.1410678888246,1.414372976489062,0.1303403117524763,0.1046904349320689,Energy,1 -62,electricity/D/short,YingLong_50m,1468617804.8276756,1457061033.079928,4508.499171171171,1.4785628623154317,0.5245328362423753,0.1005155816066071,11.861664471410982,38322.54956063956,0.6297667588297139,0.0740896141505971,0.058313981731172,Energy,1 -63,solar/10T/short,YingLong_50m,26.09515710538321,27.7849409785584,2.5307373046875,1.1219539696809864,3.741759049011934,1.5730944926769048,7.540801049475076,5.1083419135158925,1.4860161094427018,0.736191207871765,0.5696661614103602,Energy,1 -64,solar/10T/medium,YingLong_50m,35.16942324706923,32.85600116124751,2.838931773252599,1.2271129636096665,2.4403789970388856,1.5303478816768263,9.623389839344616,5.930381374504445,1.315975128083644,0.6299702106829513,0.5363962280170552,Energy,1 -65,bitbrains_rnd/5T/short,YingLong_50m,1413526.150639543,1617651.6682116392,128.62127531770662,1.8033149761164065,1.1654302492101956,0.6559372417056644,55.5143854083498,1188.9180588415431,4.864709455863586,0.5262811256083998,0.4295090572542891,Web/CloudOps,2 -66,electricity/15T/short,YingLong_50m,147501.08966216215,149329.09209459458,56.5797974732545,1.0781102571023975,0.1780705432119419,0.1734740378651663,10.355844945135749,384.05870601011264,0.7570554129580094,0.1115299334994566,0.0931792249724961,Energy,1 -67,solar/10T/long,YingLong_50m,39.365362524077455,41.476229692568936,3.161267119544556,1.362179537472376,1.9718213520460528,1.5801417159006357,10.068966022347482,6.27418221954682,1.3560639744786465,0.683257244452153,0.5594268831323339,Energy,1 -68,hospital/M/short,YingLong_50m,3306.233811386354,3063.4567579313343,19.830595053780964,0.8277984503138229,0.2061077119577765,0.186392229155632,5.845252568523061,57.49985922927424,0.2087378412843228,0.071989664990386,0.0578865357962606,Healthcare,1 -69,loop_seattle/D/short,YingLong_50m,18.65639834881321,18.91279831011352,3.120340113841589,0.9369963193505348,0.0582251644232939,0.057477707365841,6.617972802598223,4.319305308589937,0.0771917626602223,0.0557646510904927,0.0438613585539295,Transport,1 -70,saugeen/M/short,YingLong_50m,352.7041248139881,365.5207868303572,12.7148684547061,0.7450967247924493,0.402371951511928,0.3969873700823103,6.10345306099761,18.780418653852955,0.5638445087169407,0.3817384952583827,0.3148663600775754,Nature,1 -71,ett1/D/short,YingLong_50m,40004.263492063496,39826.63492063492,133.8670386904762,1.6953082577971474,1.4413104344928076,0.4696768624441964,11.177447030439016,200.01065844615252,0.5253346288301407,0.3516062175355745,0.2713516288121217,Energy,7 -72,bitbrains_rnd/5T/medium,YingLong_50m,2108276.8289839784,2271763.29469824,162.83312770682485,4.562232492964939,0.9072726409906402,0.7263479851234265,162.53609101742742,1451.990643559379,6.109264008470763,0.6851218848404009,0.6379403883415798,Web/CloudOps,2 -73,m4_hourly/H/short,YingLong_50m,610484.0450885668,701484.5732689211,214.16810084541063,0.9642894045509044,0.1272972094647934,0.1155495021654211,7.699171827419456,781.3347842561259,0.1066694442327237,0.0292386730373303,0.0240692645292731,Econ/Fin,1 -74,ett2/D/short,YingLong_50m,115713.23174603174,111845.38412698412,200.70328621031743,1.310885593087438,0.4282265314980158,0.1362759786938864,12.50183286594022,340.1664765170603,0.1946862495004617,0.1148677860757533,0.0940236788229717,Energy,7 -75,bitbrains_rnd/5T/long,YingLong_50m,2403903.1805204945,2571618.586995048,204.93930604546205,3.5096062818253326,2.5363544050581006,0.7185924567258016,118.96630136303492,1550.452572805919,5.939434753250842,0.7850763435031533,0.6614450212712936,Web/CloudOps,2 -76,us_births/M/short,YingLong_50m,49769536.0,40883354.66666666,5374.541666666667,0.6073190193919541,0.0167430688937505,0.0168678288658459,4.00501332323018,7054.752724227831,0.0219120138627552,0.0166932897735045,0.0128850775476625,Healthcare,1 -77,electricity/W/short,YingLong_50m,78800425941.56396,79297323436.05045,35512.92972972973,1.7377926176399563,0.2567041640255809,0.1087707746427741,13.061693300587018,280714.1356283363,0.6404970215356374,0.0810287863380408,0.0631888170242435,Energy,1 -78,loop_seattle/5T/short,YingLong_50m,55.46593369049923,56.872313596491225,4.197359930300245,0.6588380813048039,0.12175173725014,0.0874300621119323,5.350901955219515,7.4475454809285475,0.1277089665819101,0.0719754582826837,0.0574285432218288,Transport,1 -79,electricity/15T/medium,YingLong_50m,293482.7428468469,291231.64663063065,60.58548001126126,0.9499317309341716,0.1648060511921295,0.1467763945386848,8.37699985883902,541.7404755478834,0.9334733917053883,0.1043948829132895,0.0849946371244059,Energy,1 -80,loop_seattle/5T/medium,YingLong_50m,126.56000669182148,129.6099725877193,7.018233570772058,1.0945748166882907,0.2608085923642689,0.1530180621860689,7.923993573446473,11.249889185757407,0.2001586400067423,0.1248687932458795,0.0994782709867671,Transport,1 -81,loop_seattle/5T/long,YingLong_50m,131.4119894507511,133.67282536406375,7.0721103514505215,1.1035848649746882,0.2718466672579119,0.1531503306046468,8.383921989740612,11.463506856575394,0.2026771456417622,0.1250363573406324,0.1004954530569604,Transport,1 -82,bizitobs_l2c/5T/short,YingLong_50m,21.605775669642856,21.62736235119048,2.78084222702753,0.2877068481078589,0.1540279500589987,0.2104753351041046,2.612555731886186,4.648201337038108,0.1596380484130131,0.0955053780759954,0.0779665012791904,Web/CloudOps,7 -83,bizitobs_l2c/5T/medium,YingLong_50m,142.88671343537416,149.1109056122449,8.69811330782313,0.8418890347772198,0.5265403205971789,0.8457501422146139,5.890603479443451,11.953523055374683,0.6289310968819162,0.4576486712871379,0.3589505826234094,Web/CloudOps,7 -84,bizitobs_l2c/5T/long,YingLong_50m,266.64021825396827,306.94638888888886,12.69935763888889,1.187276933414684,0.8327570502377598,1.064004944329774,7.577885517520446,16.329121784528656,0.9168557761203632,0.7130499458498359,0.5499149616032214,Web/CloudOps,7 -85,electricity/15T/long,YingLong_50m,361712.035003003,339027.2284084084,65.0468125,1.004045441403098,0.1701865799081604,0.1494019458655707,8.851890312349786,601.4250036396916,0.948821650556743,0.1026193184955768,0.083771915015824,Energy,1 -86,solar/D/short,YingLong_50m,121799.3498783455,129854.88661800486,253.6771593673966,0.989843435216182,1.0813892373783456,0.4291314118099908,5.553985313786724,348.99763592085475,0.5041827137837863,0.3664770917354322,0.2884009884155804,Energy,1 -87,bitbrains_rnd/H/short,YingLong_50m,2060177.1368654363,2117281.9518349604,184.44738075955843,5.867890053533356,2.520784680684783,0.5681762230904476,198.2342075690009,1435.3317166653276,6.28171412933239,0.8072320177857726,0.6792077052027463,Web/CloudOps,2 -88,m4_monthly/M/short,YingLong_50m,2155715.8595555555,2111149.437962963,619.4549046585648,1.0533479404780126,0.1806442969640096,0.1456027859935054,9.55228028430166,1468.235628077304,0.3051497687522639,0.1287439953739253,0.1047997367605427,Econ/Fin,1 -89,covid_deaths/D/short,YingLong_50m,1793368.2847117796,1472458.105263158,194.51929824561404,43.57945993519842,0.1186095936102697,0.1703691975257426,890.1879936255372,1339.167011508191,0.5038681052161073,0.0731888326046813,0.0593550539782383,Healthcare,1 -90,loop_seattle/H/short,YingLong_50m,61.95739851515398,62.46728178806149,4.4340315614646135,0.8965250597533395,0.13190690814934,0.1018883100798372,7.382973603801189,7.871302212159941,0.1393300535881994,0.0784868676641886,0.0634633846793447,Transport,1 -91,loop_seattle/H/medium,YingLong_50m,69.45846233230134,66.83237229102167,4.966542505159959,1.0132323030102617,0.1342672909007353,0.1121721612164602,7.526380705270291,8.334174364164776,0.1473055823432752,0.0877830729221292,0.0696447769590137,Transport,1 -92,loop_seattle/H/long,YingLong_50m,65.69606553147575,62.03056415548676,4.751472738218094,0.9835414056424908,0.1565596815069229,0.110881079748237,7.709076965029921,8.105310946007917,0.1448527580736363,0.0849151791494899,0.0681958697173843,Transport,1 -93,solar/W/short,YingLong_50m,5121434.627737226,5059590.072992701,1883.7143020072992,2.080787091174768,0.4297032043011519,0.3356885144310276,11.17355031713386,2263.0586885313483,0.4619468575887953,0.3845131842215099,0.3045864028182746,Energy,1 -94,sz_taxi/H/short,YingLong_50m,7.290910248063568,7.258826622596154,1.862273387419872,0.5666622235072498,1.1346744635166266,0.2970678867437901,4.108246478491044,2.7001685591946973,0.25153736372616,0.173482257918256,0.13685702572313,Transport,1 -95,m4_quarterly/Q/short,YingLong_50m,2202503.0186666665,2156719.5866666664,647.456951171875,1.4015457984951658,0.1313249570528666,0.1182681825955708,11.334767466026577,1484.083224979875,0.2484093980052535,0.1083728922797461,0.0875069168020507,Econ/Fin,1 -96,us_births/D/short,YingLong_50m,390455.28,379573.06666666665,410.8233333333333,0.6041797786177266,0.038495709101359,0.0388207817077636,4.469942328335667,624.8642092486975,0.0585759491364682,0.0385113538609314,0.0307806063870929,Healthcare,1 diff --git a/results/YingLong_50m/config.json b/results/YingLong_50m/config.json deleted file mode 100644 index c60adcb95af47970a001535eaa354322d3992ad0..0000000000000000000000000000000000000000 --- a/results/YingLong_50m/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "YingLong_50m", - "model_type": "zero-shot", - "model_dtype": "bf16", - "model_link": "https://huggingface.co/qcw2333/YingLong_50m", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/yinglong.ipynb", - "org": "Alibaba", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/YingLong_6m/all_results.csv b/results/YingLong_6m/all_results.csv deleted file mode 100644 index 394491323a6cd9a079dc498160bb5db3741e074a..0000000000000000000000000000000000000000 --- a/results/YingLong_6m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -,dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -0,car_parts/M/short,YingLong_6m,1.4256717159372094,1.3781951381277402,0.6313851595843505,1.1116554622469257,0.6382586963869232,1.7779277178192112,16.78015743895224,1.1940149563289435,2.8631253826976617,1.5139968449160262,1.2105464409696856,Sales,1 -1,hospital/M/short,YingLong_6m,4015.826162538027,3838.0612777053457,21.013732140916996,0.8359615058789307,0.2085148737980769,0.1878369135319901,5.688476650672462,63.370546490763566,0.2300497993002959,0.0762847273578859,0.0619081969301056,Healthcare,1 -2,bizitobs_application/10S/short,YingLong_6m,2320305.635555556,1858212.6933333331,613.4881944444444,2.149902125424402,0.0298106257120768,0.0304501024881998,25.748018490394287,1523.254947655039,0.0587458547070207,0.0236597874773274,0.0198596027188103,Web/CloudOps,2 -3,bizitobs_application/10S/medium,YingLong_6m,11444406.613333331,8742759.253333334,1521.3734375,4.596128402307508,0.0628451538085937,0.0632885360717773,46.49517702247203,3382.9582636109085,0.1328388143709146,0.0597398572210546,0.0523605555669505,Web/CloudOps,2 -4,bizitobs_application/10S/long,YingLong_6m,13244051.91111111,10289149.724444443,1901.4120833333332,5.507461199402938,0.0754376898871527,0.0766571892632378,69.45892650247822,3639.23782008144,0.1403508562712475,0.0733298638928626,0.064830209163392,Web/CloudOps,2 -5,ett2/15T/short,YingLong_6m,8.20115269252232,8.28952404203869,1.8478413899739583,0.7985663190479825,0.1153554605776208,0.1349930536179315,5.37556537228766,2.8637654744273875,0.1359531347904118,0.0877236043962933,0.0688702376315566,Energy,7 -6,ett2/15T/medium,YingLong_6m,12.236853918650796,12.131070188492064,2.207648499503968,0.921881067240266,0.1424043299788135,0.1685027785528274,6.493926040291256,3.498121484261345,0.1864212259553815,0.1176495846721545,0.0928659678493093,Energy,7 -7,ett2/15T/long,YingLong_6m,12.351812996031748,11.778629712301589,2.236570560515873,0.94641495562293,0.1467957712771317,0.1739019290984623,6.895564035671887,3.514514617416144,0.1872948456949237,0.1191908936562061,0.0946459018278231,Energy,7 -8,jena_weather/H/short,YingLong_6m,1093.234857978279,1123.939640768588,8.614744869987469,0.5534368630739412,1.9122423732753688,0.6048327551283558,4.840350400390098,33.06410225574375,0.2026808182663901,0.052807831463258,0.0463518219474755,Nature,21 -9,jena_weather/H/medium,YingLong_6m,1481.1103174603174,1477.3252976190477,11.698043774801588,0.9287345089526038,4.576984468393407,0.6886579995473037,6.47764261574141,38.48519608187436,0.2354977402936103,0.071582404542275,0.0615594287683964,Nature,21 -10,jena_weather/H/long,YingLong_6m,1238.1851851851852,1178.522222222222,11.791174768518518,1.0847386146010818,3.86556487963336,0.6651762179768127,11.902302009548809,35.1878556491467,0.2118698765364644,0.0709959358517055,0.0640282769833356,Nature,21 -11,bizitobs_service/10S/short,YingLong_6m,14452.285291005292,12271.09417989418,34.80434854497354,1.3457522148879075,0.1474375310019841,0.0980934490224041,15.064629365476948,120.21765798336487,0.0890610842960005,0.025784174069172,0.0217088115020862,Web/CloudOps,2 -12,bizitobs_service/10S/medium,YingLong_6m,99495.53777777778,100139.45904761904,80.52168650793651,2.56449512959571,0.1497442626953125,0.1331268310546875,25.507747086169825,315.42913273472027,0.2373993984182764,0.0606025187682515,0.0528472399110938,Web/CloudOps,2 -13,bizitobs_service/10S/long,YingLong_6m,158795.78412698413,156422.89439153438,106.1684126984127,2.8186576443050546,0.1849508876901455,0.1412263609871031,32.81866787443415,398.4918871532821,0.2952155294685723,0.0786529542458256,0.0703410124367075,Web/CloudOps,2 -14,m4_weekly/W/short,YingLong_6m,386603.2243411185,352563.78144418256,314.4123098350118,3.007684597800195,0.0753146887387508,0.0726404674359197,26.088581357810188,621.7742551289161,0.1132776334380136,0.0572810502977567,0.0460643184049049,Econ/Fin,1 -15,loop_seattle/H/short,YingLong_6m,66.51445255214274,67.7222079361523,4.7398130066604205,0.9552631427715462,0.1485241628937651,0.1089159061257218,8.08626601773549,8.155639309836031,0.1443631093632809,0.0838995101974141,0.0683763311341952,Transport,1 -16,loop_seattle/H/medium,YingLong_6m,75.21790505675955,70.65750773993808,5.423755563080495,1.0987527117909544,0.1504415695143189,0.1207601793287216,8.391361001844356,8.672825667379666,0.1532912055438018,0.0958642616288985,0.0767024007197392,Transport,1 -17,bitbrains_fast_storage/H/short,YingLong_6m,2707360.0128867524,2258208.1280449666,283.7603740928292,1.2550857824673614,3.7548346359841647,0.5633419689119171,22.842427385021125,1645.4057289576797,4.689940689387193,0.8088092201652473,0.626181025860868,Web/CloudOps,2 -18,us_births/W/short,YingLong_6m,3451126.285714286,3316488.285714286,1418.0,1.2886437559112136,0.0190166447843824,0.0192207012857709,8.768189706544595,1857.7207232827773,0.0252183766608251,0.0192492109588244,0.015425635124951,Healthcare,1 -19,loop_seattle/H/long,YingLong_6m,77.03095115239078,77.21209150326797,5.328770532335741,1.0925809377147078,0.1867445497936016,0.122045201372764,8.66671102574153,8.776727815785948,0.1568518764359512,0.0952322636222274,0.0776206161634525,Transport,1 -20,us_births/D/short,YingLong_6m,291700.4266666667,278277.17333333334,360.48333333333335,0.5302691111278264,0.0340310351053873,0.0339402770996093,4.642845159482257,540.0929796494921,0.0506293342403947,0.0337923873464703,0.0276366354815189,Healthcare,1 -21,m4_daily/D/short,YingLong_6m,487054.95204298897,453533.78647470346,259.19196534185,4.848722708823648,0.0514984758713499,0.0418586928208943,44.35645812622036,697.8932239554908,0.1078052165014137,0.0400379957563352,0.0323670658642097,Econ/Fin,1 -22,electricity/W/short,YingLong_6m,121555547433.05225,148941582800.95135,38160.5963963964,1.789992584226223,0.2676569298852848,0.1128648620468002,13.950345574134747,348648.17141790985,0.7955000729021929,0.0870698879385118,0.0705048804977863,Energy,1 -23,ett2/W/short,YingLong_6m,6308226.285714285,5570140.571428572,1630.1685267857142,1.0456053839619526,0.1521472930908203,0.160482747214181,12.255732843150977,2511.6182603481534,0.210686266588469,0.1367461473906063,0.1208295737805687,Energy,7 -24,bitbrains_rnd/5T/short,YingLong_6m,1411730.3493651769,1572914.8245244725,133.84738430405437,1.8703412026772617,1.2843956277336963,0.6748114196869792,56.965790015981455,1188.1625938250947,4.861618311119202,0.5476648548016729,0.4486004945457798,Web/CloudOps,2 -25,bitbrains_rnd/5T/medium,YingLong_6m,2120893.5838062745,2242790.2817224977,184.4575629429833,4.716103863100836,1.737886768178811,0.763055867398377,163.4471494050614,1456.3288034665368,6.127516856243132,0.7761068953001928,0.67858376583898,Web/CloudOps,2 -26,bitbrains_rnd/5T/long,YingLong_6m,2554275.375110407,2890143.4071811587,235.6223082051662,3.66585199150068,3.708038489884353,0.7385314958387462,119.27105109574332,1598.210053500605,6.1223829101573015,0.9026160171170392,0.7269553805930001,Web/CloudOps,2 -27,bitbrains_fast_storage/5T/short,YingLong_6m,1660855.3978340656,1746959.7239333368,182.2329702517244,0.839242730401285,2.3897731391605705,0.6932475103115835,15.5882633909942,1288.74178865825,4.046330540312583,0.5721664645864641,0.4380864950051189,Web/CloudOps,2 -28,bitbrains_fast_storage/5T/medium,YingLong_6m,2883905.47595491,3269161.232629103,320.7089593357298,1.1248485031990805,5.561914312475275,0.7682795851990999,23.09546434583923,1698.20654690615,5.159620813379753,0.9744024510096536,0.6903823536588576,Web/CloudOps,2 -29,temperature_rain/D/short,YingLong_6m,173.95016437325032,183.78863917195116,6.196907171296386,1.4494874376699838,44.49994781273643,1.4786377054508604,19.25694551667524,13.18901680843763,1.5526959677391374,0.7295398070286827,0.5965084148536008,Nature,1 -30,covid_deaths/D/short,YingLong_6m,2521502.73283208,1429387.8696741855,190.8111058897243,43.99552852963286,0.1187168827274223,0.1744953259505021,990.3348606461624,1587.9240324499406,0.5974641449363226,0.0717936071846385,0.0645705954238884,Healthcare,1 -31,bitbrains_fast_storage/5T/long,YingLong_6m,3557613.766486999,3612921.949718246,375.43088978436447,1.0492137198460367,5.260199865673104,0.7790442698512434,17.98245515654289,1886.163769794924,4.984567713257915,0.992151753600007,0.7163534024934399,Web/CloudOps,2 -32,m_dense/H/short,YingLong_6m,58425.37333333334,58457.79555555555,115.704453125,0.9912151708333264,0.3735685572615229,0.2771017711118441,8.980336145676718,241.71341157108623,0.4287393610054854,0.2052308681420126,0.1681412299803744,Transport,1 -33,m_dense/H/medium,YingLong_6m,63860.333333333336,58787.55111111111,130.19722222222222,1.0937945539709597,0.5707514801886628,0.3245477973090278,8.963314518927264,252.7060215612864,0.4396765515030423,0.2265267179953562,0.1823752116213765,Transport,1 -34,m_dense/H/long,YingLong_6m,76637.2661728395,70293.0962962963,145.43271604938272,1.1986693071260404,0.6752887811929162,0.3559894627700617,9.28459470404453,276.8343659534334,0.4792412110261316,0.2517655303462199,0.1996784596665274,Transport,1 -35,electricity/H/short,YingLong_6m,2086959.0506306307,2143283.6264864863,218.35821860923423,1.1828402007772778,0.2308749496262992,0.1515115087414799,11.424128323613807,1444.631112301902,0.681969051717956,0.1030806729909616,0.0844180273551706,Energy,1 -36,solar/10T/short,YingLong_6m,30.63052530033455,32.533699247262774,2.694103083181265,1.184814199998189,2.774939421239352,1.6279012530401566,9.084060634000648,5.534485098031664,1.6099811156698989,0.7837142951441862,0.6442934399924607,Energy,1 -37,electricity/H/medium,YingLong_6m,8272421.735495496,7590927.824144145,300.43863738738736,1.2885369002786058,0.2831753755766807,0.1536465776676589,11.089191636103468,2876.181798060668,1.1226685382653605,0.117271100909394,0.0949288107500603,Energy,1 -38,solar/10T/medium,YingLong_6m,54.561721687679714,60.905640345056405,4.133981955734351,1.7762782629536509,1.6602498323897497,1.6801763122034226,11.411678287324456,7.38659066739722,1.6391137408834482,0.9173469782367212,0.7501030995299636,Energy,1 -39,m4_monthly/M/short,YingLong_6m,2213308.539185185,2133837.343148148,614.0911432291666,1.050113930620086,0.175333191341824,0.143268569946289,10.907367839620989,1487.7192407121663,0.3091991323396145,0.1276292216083775,0.1049130441022329,Econ/Fin,1 -40,jena_weather/D/short,YingLong_6m,456.9145833333333,381.391121031746,10.172071087549604,1.2265688762946565,0.8045389398100412,0.4825322711278522,8.080186964146192,21.375560421503184,0.1287158104709992,0.061252493426345,0.0515118234851049,Nature,21 -41,kdd_cup_2018/D/short,YingLong_6m,2804.9000598444045,3063.910100405612,22.16047858900193,1.235084621197566,0.529840773364253,0.48696614713204,9.390973827828384,52.9613071953894,1.1863894806491615,0.496418235810306,0.3931704265712783,Nature,1 -42,saugeen/D/short,YingLong_6m,1239.9059375,1419.1428125,14.692249348958333,3.264783412067605,0.3566678873697916,0.3851711018880208,34.43539381044104,35.212298100237646,1.1405347253396367,0.4758854570676986,0.3974965662985171,Nature,1 -43,ett1/15T/short,YingLong_6m,4.8676272437686015,4.990239606584821,1.1404894147600446,0.7283943219594146,0.4423740879988088,0.2559108007521856,5.036526940554215,2.2062699843329696,0.41707686337269,0.2155999724402206,0.1682085004302174,Energy,7 -44,ett1/15T/medium,YingLong_6m,9.993009052579366,10.509025297619049,1.7060029141865078,1.077458748159638,0.718713344201674,0.4138988095238095,7.440758873505662,3.1611721010693747,0.6025225280494372,0.325165842241755,0.2547738538940717,Energy,7 -45,electricity/H/long,YingLong_6m,12435422.78918919,11308260.072168168,349.5649129129129,1.447709422068271,0.3546584281847244,0.1710208984494093,12.161615080735844,3526.3894834787025,1.3908069692706584,0.1378682982607375,0.1097024346345031,Energy,1 -46,ett1/15T/long,YingLong_6m,8.801176215277778,8.903399057539682,1.655717540922619,1.0939685705011315,0.7376511119753871,0.4196805245535714,7.341803746068504,2.966677639258734,0.5654516913233636,0.3155814003783656,0.2463456125350392,Energy,7 -47,hierarchical_sales/W/short,YingLong_6m,593.9550450211865,536.1059984110169,10.393634343551376,0.7924142448297052,0.6735013868940373,0.5134435104111493,6.942178945762895,24.371192933896086,1.118893388432303,0.477176836617141,0.4238735763580704,Sales,1 -48,us_births/M/short,YingLong_6m,70465296.0,59346416.0,6786.041666666667,0.7683487106292306,0.0210015053550402,0.0212315022945404,4.335564397122846,8394.360964361731,0.0260728279232775,0.021077399150019,0.0166777022892018,Healthcare,1 -49,sz_taxi/15T/short,YingLong_6m,16.908707205338064,17.10427457741911,2.7716585748507137,0.553238366544088,1046035122388.7012,0.3983898279285547,4.012979207869956,4.112019844959174,0.3844914418841482,0.2591619306412892,0.2040290328973894,Transport,1 -50,sz_taxi/15T/medium,YingLong_6m,16.695684762286326,16.662596821581197,2.7991895365918804,0.5460415234734954,11066967960346.691,0.4087407038762019,3.838801948937468,4.086035335418225,0.3802737900338029,0.2605113090618455,0.2046726696245767,Transport,1 -51,sz_taxi/15T/long,YingLong_6m,15.90224915420228,15.79834290420228,2.7299501424501424,0.5143364474420461,8393565261042.143,0.4010567602608618,3.6545857891035154,3.987762424493501,0.3691001865413246,0.252679322278043,0.1992851416979396,Transport,1 -52,saugeen/W/short,YingLong_6m,911.599609375,960.18984375,15.580357360839844,1.2639585670474955,0.4376567840576172,0.4278181076049804,11.711795701281426,30.19270788410672,0.911391993483082,0.4703060423987124,0.3854927856982198,Nature,1 -53,m_dense/D/short,YingLong_6m,13341.748148148148,12546.767407407408,64.25738425925925,0.9290418115872155,0.1582925851650817,0.1377927879050925,7.827893388393261,115.5064853077443,0.1999588117150047,0.1112390370649989,0.0872931534659928,Transport,1 -54,bizitobs_l2c/H/short,YingLong_6m,114.42757936507935,126.6770755828373,6.601813422309028,0.6749339592467777,0.5073041298406588,0.7045015169432383,4.831414432867954,10.697082750221172,0.5766001559393172,0.3558546509988396,0.2866723190263739,Web/CloudOps,7 -55,bizitobs_l2c/H/medium,YingLong_6m,134.3912388392857,144.24291294642856,7.931118629092261,0.8366655400172881,0.7647520299463647,0.9057084583100818,4.982667404934468,11.592723529839123,0.7019570180676354,0.480241279668949,0.3697913207905763,Web/CloudOps,7 -56,bizitobs_l2c/H/long,YingLong_6m,163.82595486111111,163.56238839285714,8.791390749007936,0.9157663167342724,0.98063049475104,0.890716068328373,6.1518163495244025,12.799451350003684,0.7818274299497194,0.5370035204656358,0.4142741912553321,Web/CloudOps,7 -57,electricity/D/short,YingLong_6m,1533642808.236973,1485540612.298955,4602.094126126126,1.5200583704182054,0.4938796391668024,0.102420258687438,12.38869562345204,39161.751853523776,0.6435576395005285,0.0756276900902348,0.0594639002034952,Energy,1 -58,electricity/15T/short,YingLong_6m,169974.29711711712,168176.98011261263,62.85628994228603,1.166133278671916,0.1918052702106809,0.1839524637051659,10.652215801644042,412.2793920597016,0.8126839478587841,0.1239021373415051,0.1025828727259494,Energy,1 -59,solar/10T/long,YingLong_6m,62.130653449412,74.06619874036902,4.560471019141575,1.9646781716857096,1.5017374238717625,1.7485805326707158,12.36408185977238,7.882300010112023,1.7036328728937742,0.9856727521309536,0.7950374809879271,Energy,1 -60,ett1/H/short,YingLong_6m,96.08734188988096,97.44706101190476,4.944199335007441,0.8355688178660703,0.4380381992885044,0.2681375231061663,5.863344281192608,9.802415104956582,0.4575188162895408,0.230766010522073,0.1821882587743685,Energy,7 -61,ett1/H/medium,YingLong_6m,144.43982514880952,144.67147507440475,7.006554594494047,1.3020510597474362,4901045378767.825,0.4599154881068638,8.750688141923838,12.018312075695553,0.5756486591175269,0.3355973560971806,0.2617993112689566,Energy,7 -62,ett1/H/long,YingLong_6m,150.16276455026454,154.31660052910053,7.399727182539682,1.4042094601479322,3797929727094.4585,0.4959285675533234,8.927667085037163,12.254091747259956,0.5868219132561785,0.3543569080754304,0.2741528388126436,Energy,7 -63,ett1/D/short,YingLong_6m,40297.57777777778,40178.444444444445,135.17145337301588,1.7178217424849256,1.3497804671999007,0.4811476934523809,10.745280435240572,200.7425659340285,0.5272570081246664,0.355032305967142,0.2731873254716103,Energy,7 -64,ett2/D/short,YingLong_6m,111597.38412698412,104680.0507936508,196.2031498015873,1.3196671681082288,0.4579360235305059,0.1361082410055493,13.11068159392628,334.06194654133253,0.1911924659327621,0.1122922392769428,0.0934227546884637,Energy,7 -65,loop_seattle/D/short,YingLong_6m,19.58421536377709,19.838204656862747,3.15921617002064,0.9511760644378504,0.0582751578221749,0.0583228395696271,8.306610462454897,4.425405672226795,0.079087918060891,0.0564594182086612,0.0455482690902607,Transport,1 -66,saugeen/M/short,YingLong_6m,422.8811383928572,431.0413876488095,13.497557140531994,0.7908826313922438,0.4121134621756417,0.4043764840988886,6.196140045153902,20.564073973628307,0.6173951923324396,0.4052371576509418,0.330658762144861,Nature,1 -67,m4_hourly/H/short,YingLong_6m,1257200.1288244766,1117143.085346216,255.2020933977456,1.162580452863786,0.1515378230243873,0.1439845012965793,9.289869046444084,1121.249360679629,0.153075286752863,0.0348407187524388,0.0274400106343165,Econ/Fin,1 -68,ett1/W/short,YingLong_6m,1530810.5714285714,1635119.4285714286,958.4317801339286,1.616038709138826,0.8329011372157505,0.5475828647613525,10.051207219177323,1237.2592983803238,0.4923350054198341,0.3813828809243914,0.3001318502614921,Energy,7 -69,kdd_cup_2018/H/short,YingLong_6m,4400.338209159561,4512.022298842,22.4222406458294,0.9355944265482335,0.8910455518531085,0.4936754558677208,7.899186291474847,66.33504510558171,1.388512636408817,0.469338106633982,0.3778594455561658,Nature,1 -70,kdd_cup_2018/H/medium,YingLong_6m,4861.196719513105,5068.561403816703,25.01046982575156,1.0362150684167428,0.9895599268400797,0.5509716319720219,9.123059247672622,69.72228280480427,1.4594136520413743,0.5235144295240369,0.4199486741754273,Nature,1 -71,kdd_cup_2018/H/long,YingLong_6m,3808.0005624527626,4054.502235163844,23.896696175907103,1.0145360063015778,1.045777418031586,0.6001156763201528,8.031144110772448,61.70899903946557,1.4479424770138187,0.5607130563122684,0.443385552750686,Nature,1 -72,restaurant/D/short,YingLong_6m,155.43575464649294,158.31785280657513,7.730971852526385,0.7359646030201444,0.7514430906766602,0.4180397097109368,5.113213956181143,12.467387643227148,0.5737780659882443,0.355797237135071,0.2792917209108109,Sales,1 -73,bizitobs_l2c/5T/short,YingLong_6m,25.520079985119047,23.281775483630952,3.0645804268973214,0.3140058967085348,0.1640628778096951,0.2186434474080827,2.8339294917401223,5.0517402927228,0.1734972095536489,0.1052500963450835,0.0862905443981476,Web/CloudOps,7 -74,bizitobs_l2c/5T/medium,YingLong_6m,151.3609906462585,147.68826530612245,8.864945392219388,0.855587909665183,0.5131953472364185,0.7810369997608418,8.138101425179208,12.30288546017797,0.6473126969712253,0.4664264922984302,0.3777525911006123,Web/CloudOps,7 -75,bizitobs_l2c/5T/long,YingLong_6m,255.129126984127,270.9380357142857,12.081734871031744,1.1310586877828337,0.7863878990967644,0.9703994605654762,12.098654228981454,15.972762033666156,0.8968467088682183,0.6783713507823539,0.5570813152362109,Web/CloudOps,7 -76,m4_yearly/A/short,YingLong_6m,5946464.286933055,6271486.670264937,1350.404881605293,5.270358387025677,0.2338295769959213,0.2302942916654481,50.81537036373012,2438.5373253106163,0.3910307635723022,0.2165436823562338,0.177432191591014,Econ/Fin,1 -77,electricity/15T/medium,YingLong_6m,323643.0010810811,297403.4505765766,65.74022888513514,1.029878923600594,0.1769668669185374,0.1593413279700174,8.818135834764806,568.8963008150793,0.980265613185778,0.1132770342973414,0.0919811912232511,Energy,1 -78,loop_seattle/5T/short,YingLong_6m,59.32077467911507,61.91805824303405,4.440177182098168,0.6930516664203245,0.136395591405154,0.092089331974309,5.74872904092521,7.701998096540603,0.1320722673052282,0.0761392381984669,0.0606622336274395,Transport,1 -79,loop_seattle/5T/medium,YingLong_6m,143.55213412667698,152.0149394511094,7.543936413586816,1.1698861469748054,0.294770224052563,0.1614691643965871,8.945093428100222,11.981324389510409,0.2131723749172696,0.1342221268057066,0.1073335596907305,Transport,1 -80,electricity/15T/long,YingLong_6m,427130.7330570571,377739.4658858859,71.10913194444444,1.0776169606145911,0.1803762871579164,0.1602488576761639,9.394873488037192,653.5523950358204,1.0310589989283163,0.1121833703834594,0.0910301376974757,Energy,1 -81,solar/D/short,YingLong_6m,121620.9907542579,124691.05206812652,252.89779501216543,0.9876464983558,1.03813322118309,0.4334201190883516,5.563061242944722,348.7420117425744,0.5038134239128848,0.3653511756970328,0.2886610173572294,Energy,1 -82,sz_taxi/H/short,YingLong_6m,7.401238857171474,7.458362025073451,1.872837686131143,0.5705636704225204,1.1252373589409723,0.2969712803506443,4.165126290372363,2.7205217986944112,0.2534333935831153,0.1744663875343128,0.1380375211958415,Transport,1 -83,hierarchical_sales/D/short,YingLong_6m,28.1761290822654,29.048646134766432,2.31473781779661,0.7547697733321663,0.6108387172504037,1.0257519410138487,7.139694087114035,5.308119166170386,1.628472176307393,0.7101359283250264,0.5806447906103447,Sales,1 -84,jena_weather/10T/short,YingLong_6m,1294.21875,1514.3083333333334,8.747539992559524,0.3614230021215637,0.408061852448748,0.6081725513985363,3.290174630413293,35.97525191016736,0.2234514029149214,0.0543331868327899,0.04751797037363,Nature,21 -85,jena_weather/10T/medium,YingLong_6m,2153.5473304473303,2340.593217893218,12.827224251443,0.6620934052029728,0.9885680672350696,0.6976142815589148,6.9173921247698535,46.40632856030878,0.2847948450432316,0.0787204559455066,0.0684501072410684,Nature,21 -86,jena_weather/10T/long,YingLong_6m,2293.6833333333334,2528.382804232804,13.373459201388888,0.6617058281251397,0.905035600700856,0.681582378917909,7.5457536312034845,47.89241415227816,0.2930618943803265,0.0818344900199733,0.0709459708477972,Nature,21 -87,loop_seattle/5T/long,YingLong_6m,149.38093337919963,156.49258112601763,7.589826568054122,1.1776338798942303,0.3043673048532278,0.1606137316266053,9.674586819463173,12.22214929458807,0.2160900990968317,0.1341896859290376,0.1086805463172179,Transport,1 -88,m4_quarterly/Q/short,YingLong_6m,2223853.2213333333,2197359.973333333,665.29750390625,1.4687421750313596,0.1341299676895141,0.1230659607251485,13.812310096535851,1491.2589383917648,0.2496104860701967,0.1113590866455555,0.0906396630489106,Econ/Fin,1 -89,bitbrains_rnd/H/short,YingLong_6m,2075879.546822386,1939742.5855675377,183.50577021439835,6.06972423368074,2.442471223785261,0.6024472826633097,206.0600628480097,1440.791291902608,6.305607902813446,0.8031110691596235,0.6920231138529406,Web/CloudOps,2 -90,ett2/H/short,YingLong_6m,116.23372395833331,113.91056547619048,6.712948753720238,0.7770926335150614,0.1264926730156087,0.115883799961635,5.602928147235157,10.78117451664397,0.1367352904471069,0.0851388683282542,0.068046699928005,Energy,7 -91,ett2/H/medium,YingLong_6m,283.94916294642854,275.0946986607143,10.345156715029765,1.128187364748504,0.1837476101249718,0.1845985775902157,7.962617794351183,16.85079116677993,0.2270752473106589,0.1394076394563621,0.1124304754724707,Energy,7 -92,ett2/H/long,YingLong_6m,318.9388558201058,306.72460317460315,10.977796378968254,1.2004794387609048,0.1982187573096972,0.1981620990410053,8.088656623438526,17.858859309040593,0.2445633330440612,0.1503324722738818,0.1201268788616959,Energy,7 -93,solar/W/short,YingLong_6m,5541395.620437956,6155899.328467153,2133.059306569343,2.3621086683270733,0.4856537867636576,0.3722002349630759,7.349534061933132,2354.016911672037,0.4805137049996196,0.4354107335853979,0.3259369901003652,Energy,1 -94,solar/H/short,YingLong_6m,660.2696087847355,673.4941573825074,12.277839972307596,0.9476317131566676,4.625272248395754,1.4195587045275764,8.772532354802598,25.695711875422628,0.9471525240179856,0.4525652830962397,0.380009302778114,Energy,1 -95,solar/H/medium,YingLong_6m,715.0728710462287,716.3605231143553,13.413013229927008,1.0211985848631435,5.368780852321784,1.42377526542029,9.668975323556722,26.74084649083175,0.9632677040409178,0.4831680426680602,0.416752720446112,Energy,1 -96,solar/H/long,YingLong_6m,721.4557177615571,720.9772100567722,14.540393096107056,1.1052415852486046,5.611037135556817,1.4333167342057556,8.586769921994845,26.85992773187517,0.9312826452913922,0.5041419277554371,0.4169607668135643,Energy,1 diff --git a/results/YingLong_6m/config.json b/results/YingLong_6m/config.json deleted file mode 100644 index 6c36dc8b7438fcd7aac9966f3e2371b1e847c993..0000000000000000000000000000000000000000 --- a/results/YingLong_6m/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "YingLong_6m", - "model_type": "zero-shot", - "model_dtype": "bf16", - "model_link": "https://huggingface.co/qcw2333/YingLong_6m", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/yinglong.ipynb", - "org": "Alibaba", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/auto_arima/config.json b/results/auto_arima/config.json index cc94ef84d89fb0f1a3ef113835b4a6a49088b185..5eb6a9bb09477f57ec930d1609814035d30d997b 100644 --- a/results/auto_arima/config.json +++ b/results/auto_arima/config.json @@ -1,7 +1,5 @@ { "model": "Auto_Arima", "model_type": "statistical", - "model_dtype": "float32", - "testdata_leakage": "No", - "replication_code_available": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/auto_ets/config.json b/results/auto_ets/config.json index f803ff18a7174edd37efefa1550d909de6afaaf6..c68c6cc9eae4738e51ce65d90579625ee58ddeca 100644 --- a/results/auto_ets/config.json +++ b/results/auto_ets/config.json @@ -1,7 +1,5 @@ { "model": "Auto_ETS", "model_type": "statistical", - "model_dtype": "float32", - "testdata_leakage": "No", - "replication_code_available": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/auto_theta/config.json b/results/auto_theta/config.json index b179cc612bdab04988a184ab44effa0637bcb60d..28984714533528ea26486371d9120787158ae553 100644 --- a/results/auto_theta/config.json +++ b/results/auto_theta/config.json @@ -1,7 +1,5 @@ { "model": "Auto_Theta", "model_type": "statistical", - "model_dtype": "float32", - "testdata_leakage": "No", - "replication_code_available": "No" -} + "model_dtype": "float32" +} \ No newline at end of file diff --git a/results/chronos-2-synth/all_results.csv b/results/chronos-2-synth/all_results.csv deleted file mode 100644 index 8fcac2a4192776dfc2bb9fbde8427e0719a41693..0000000000000000000000000000000000000000 --- a/results/chronos-2-synth/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/5T/long,Chronos-2-Synth,3793345.1402724674,3793345.1402724674,309.1508532782765,0.8759402907378362,2.994418738835243,0.8289809156065816,16.167750626401787,1947.6511854725084,5.147060096870285,0.8169933264697912,0.6200447414208721,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Chronos-2-Synth,3239103.272963466,3239103.272963466,265.9827951276569,0.96950614920306,3.5620362640495196,0.8078988930972284,21.059951745220832,1799.7508919190639,5.468140482883074,0.808129222948785,0.5914591771812167,Web/CloudOps,2 -bitbrains_fast_storage/5T/short,Chronos-2-Synth,1954999.5260345312,1954999.5260345312,159.39402642439936,0.6637699037556243,2.1117972478549456,0.7419216672577367,12.912584747397071,1398.2129759212403,4.3900429010213715,0.5004578173855134,0.3854588920477343,Web/CloudOps,2 -bitbrains_fast_storage/H/short,Chronos-2-Synth,2674974.31183542,2674974.31183542,271.33362894035594,1.0085908073420293,2.661933612801714,0.5818175942675801,20.619732905229526,1635.5348702597018,4.661805650291443,0.7733889796598359,0.6022981568065253,Web/CloudOps,2 -bitbrains_rnd/5T/long,Chronos-2-Synth,3868997.2885057013,3868997.2885057013,246.37250622553236,3.317870488567364,4.287637447380719,0.7180455763817263,115.40701566464033,1966.9766873315255,7.5350442060415945,0.9437975231323885,0.7456062720871048,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Chronos-2-Synth,2251706.009342534,2251706.009342534,151.741557259183,4.366534645426357,0.6850735938620953,0.7430337151250351,159.23433581580989,1500.5685620265854,6.31365695204471,0.6384540914342823,0.6210688094147774,Web/CloudOps,2 -bitbrains_rnd/5T/short,Chronos-2-Synth,1794318.6662009724,1794318.6662009724,121.74398510879313,1.6267822467708557,1.4941304791473826,0.6763744516832255,51.38441208243037,1339.5218050487167,5.480936327385669,0.4981412233145517,0.40949678764882375,Web/CloudOps,2 -bitbrains_rnd/H/short,Chronos-2-Synth,3065537.1157239676,3065537.1157239676,212.03527130130854,5.843239611381821,3.16625013845813,0.5584677513346618,201.46560709543203,1750.867532317613,7.662653658084533,0.9279701732370597,0.7396091701029824,Web/CloudOps,2 -bizitobs_application/10S/long,Chronos-2-Synth,5696007.395555556,5696007.395555556,1454.3155555555556,3.000292144579015,0.043572616577148435,0.044571439954969615,24.984235679050666,2386.630971800114,0.09204281694570038,0.05608713791236913,0.0471631923276947,Web/CloudOps,2 -bizitobs_application/10S/medium,Chronos-2-Synth,1709081.28,1709081.28,732.3341666666666,1.7220286092351975,0.024250977834065754,0.024681353569030763,15.590544703660738,1307.3183544951858,0.05133454411320794,0.028756607336761608,0.020536941662658268,Web/CloudOps,2 -bizitobs_application/10S/short,Chronos-2-Synth,358574.50666666665,358574.50666666665,311.39722222222224,1.0175544449456937,0.014157140519883898,0.014327945709228516,8.99118288813673,598.8109106109096,0.02309374330664817,0.012009346170841565,0.009391937929873147,Web/CloudOps,2 -bizitobs_l2c/5T/long,Chronos-2-Synth,105.08625,105.08625,6.430193452380952,0.6441739873762772,0.555990304055754,0.820685066344246,4.301951311675586,10.251158471119252,0.5755872226439014,0.3610457491947104,0.27981438300165734,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Chronos-2-Synth,95.57163052721089,95.57163052721089,6.0434716464710885,0.5988177381865053,0.4218252909873767,0.7003924519026361,3.6474162680715523,9.776074392475278,0.5143652764440971,0.3179754817037646,0.24079764903764195,Web/CloudOps,7 -bizitobs_l2c/5T/short,Chronos-2-Synth,20.981524367559523,20.981524367559523,2.6334751674107144,0.267162743475183,0.15679867412976825,0.20227314177013578,2.156590717927886,4.580559394611047,0.15731495031614756,0.09044419675191297,0.07254676667010823,Web/CloudOps,7 -bizitobs_l2c/H/long,Chronos-2-Synth,80.60271577380952,80.60271577380952,5.145284598214285,0.5497473060902974,0.6268150579538807,0.7779261997767857,4.165520636610837,8.977901523953664,0.5483961369025667,0.31428883346477127,0.2505023362064724,Web/CloudOps,7 -bizitobs_l2c/H/medium,Chronos-2-Synth,75.52813895089285,75.52813895089285,4.864032273065476,0.49191206950305355,0.4721516248106625,0.7624497186569941,3.6447335671185046,8.690692662319433,0.5262346411075575,0.2945245421748763,0.23569556226067595,Web/CloudOps,7 -bizitobs_l2c/H/short,Chronos-2-Synth,52.51400514632937,52.51400514632937,4.371547638423859,0.4303170757847046,0.39080810546875,0.6016565958658854,2.7738637933171923,7.246654755563381,0.39061329112522913,0.23563763804871823,0.18343025211963043,Web/CloudOps,7 -bizitobs_service/10S/long,Chronos-2-Synth,130956.16338624338,130956.16338624338,77.27796957671957,1.343717417776477,0.08910345491278109,0.08312263609871032,16.963064942220985,361.87865837355395,0.2680912790930465,0.05724999037702389,0.050813511989174276,Web/CloudOps,2 -bizitobs_service/10S/medium,Chronos-2-Synth,14023.939047619047,14023.939047619047,29.982487599206348,0.9460166953408063,0.07152118016803076,0.060839398096478176,10.219241616946563,118.42271339409112,0.08912771409707064,0.022565524012003985,0.017060509831761866,Web/CloudOps,2 -bizitobs_service/10S/short,Chronos-2-Synth,1697.0586243386244,1697.0586243386244,17.232896825396825,0.7436720250917929,0.07022247152984458,0.05426697746155754,5.8999230945052785,41.1953713945951,0.030518848112036662,0.012766680890117843,0.009940853853407166,Web/CloudOps,2 -car_parts/M/short,Chronos-2-Synth,1.3839921481235262,1.3839921481235262,0.45368274147963167,0.8359226680147035,0.8512076452895977,1.9068825535364355,15.474058219114315,1.176431956435869,2.8209631526259273,1.087884675093083,0.9509235410734131,Sales,1 -covid_deaths/D/short,Chronos-2-Synth,429949.3373433584,429949.3373433584,106.54495614035088,29.88446124481787,0.08313788525834324,0.35424593588463343,482.36102002924895,655.7052213787522,0.24671228057233185,0.04008805825519208,0.03305528585678907,Healthcare,1 -electricity/15T/long,Chronos-2-Synth,345423.34145345347,345423.34145345347,59.83367661411411,0.8853972478829217,0.1269324583519535,0.1320987013624953,8.073243201743978,587.7272679172318,0.9272117937359903,0.0943949577425508,0.07708468672235487,Energy,1 -electricity/15T/medium,Chronos-2-Synth,267754.98054054054,267754.98054054054,53.62474141328829,0.8300065425938451,0.12317424524634973,0.12982086044174057,7.463443402025155,517.4504619193422,0.8916192522607155,0.09240082937454829,0.07486100279640806,Energy,1 -electricity/15T/short,Chronos-2-Synth,132087.32486486487,132087.32486486487,49.346680567286036,0.9663953622202821,0.1561490192550028,0.16212189545502534,11.321251323527775,363.43819951246854,0.7164083300571827,0.09727203432797597,0.08204097748186358,Energy,1 -electricity/D/short,Chronos-2-Synth,1406202622.0627027,1406202622.0627027,4262.968216216216,1.3962842131175988,0.23677406344177618,0.09302860184403153,10.826249597562741,37499.368288848586,0.6162391567414234,0.07005472515007087,0.055309948339465426,Energy,1 -electricity/H/long,Chronos-2-Synth,10339854.32984985,10339854.32984985,304.97680480480483,1.254369112068842,0.29643916012087,0.14579578406531532,11.893301674672545,3215.5643874520456,1.2682176433316403,0.12028276161089041,0.09760427734375655,Energy,1 -electricity/H/medium,Chronos-2-Synth,6824298.111711712,6824298.111711712,259.986722972973,1.1144176101174572,0.24051122402711822,0.13279026065860783,10.245041318070836,2612.3357578442537,1.0196807339142613,0.1014813856499874,0.0831267589443106,Energy,1 -electricity/H/short,Chronos-2-Synth,2305910.6194594596,2305910.6194594596,196.86168637387388,1.0568440483812842,0.1627461825798434,0.13838065551207945,12.985249011991181,1518.5225120028545,0.7168510692482831,0.09293277462511955,0.07866249698729341,Energy,1 -electricity/W/short,Chronos-2-Synth,70140608482.47928,70140608482.47928,31369.884684684686,1.4250855317322035,0.12389578224306297,0.09233172476828636,11.433969613072982,264840.72285522724,0.6042791318307777,0.0715757247545918,0.055819043931957836,Energy,1 -ett1/15T/long,Chronos-2-Synth,12.038900049603175,12.038900049603175,1.704739273313492,1.057462354308891,0.8588890776904113,0.3907097904265873,9.12495480212906,3.4697118107420932,0.661330501779421,0.3249249910419426,0.26142114997630717,Energy,7 -ett1/15T/medium,Chronos-2-Synth,11.308136160714286,11.308136160714286,1.6527447606646826,1.0211817552048705,0.7862688156076661,0.3836623031374008,8.035749854006646,3.3627572259552556,0.6409447764215256,0.3150147482933898,0.2462622293809939,Energy,7 -ett1/15T/short,Chronos-2-Synth,5.584258161272321,5.584258161272321,1.13715086437407,0.6937849408185501,0.5150412581661331,0.2320850372314453,4.8888384527913855,2.3631035020227786,0.4467248773463393,0.2149688238272068,0.16834457786201745,Energy,7 -ett1/D/short,Chronos-2-Synth,41705.85396825397,41705.85396825397,135.25633680555555,1.6473303481655612,1.1368063306051588,0.47727738637772815,12.881674897461828,204.2201115665496,0.5363908970798501,0.35525525511831973,0.2757008210030695,Energy,7 -ett1/H/long,Chronos-2-Synth,163.34264219576718,163.34264219576718,7.656326471560846,1.4146557808356803,4134999578627.657,0.4786190941220238,12.811649119504686,12.78055719426063,0.6120332032680461,0.3666448922711783,0.3096447603742983,Energy,7 -ett1/H/medium,Chronos-2-Synth,155.19994419642856,155.19994419642856,7.438834635416667,1.3921444103835061,4399629119783.946,0.44881402878534227,11.541074172028337,12.457926962236877,0.5967051700794436,0.35630254534115147,0.2895745427099732,Energy,7 -ett1/H/short,Chronos-2-Synth,97.9201915922619,97.9201915922619,4.9486310686383925,0.8188669089694061,0.5129561651320684,0.2610577537899926,5.821784514239439,9.895463182300357,0.4618617507346262,0.23097285766160286,0.17904820972191957,Energy,7 -ett1/W/short,Chronos-2-Synth,1277266.857142857,1277266.857142857,901.1335797991071,1.5544519258384428,0.6900087084089007,0.5641817705971854,8.794015398913894,1130.1623145118833,0.4497185593746589,0.3585825594321058,0.26430613170150624,Energy,7 -ett2/15T/long,Chronos-2-Synth,11.959277033730158,11.959277033730158,2.138247612847222,0.8870302940235181,0.13569047509207663,0.16518651568700396,7.298914976644507,3.4582187660311714,0.1842947256924225,0.1139510782669219,0.09380731432190006,Energy,7 -ett2/15T/medium,Chronos-2-Synth,11.529809027777778,11.529809027777778,2.101476779513889,0.8671634129821371,0.130831289897771,0.1591310531374008,6.71641001044967,3.395557248490707,0.18095538021780289,0.11199149412805445,0.08934725301827626,Energy,7 -ett2/15T/short,Chronos-2-Synth,8.156113978794643,8.156113978794643,1.7704764229910714,0.7589512486691441,0.10775694254655459,0.12701343354724703,4.857620242533299,2.855891100653987,0.1355793102546512,0.08405081419115856,0.06571720501058526,Energy,7 -ett2/D/short,Chronos-2-Synth,141351.3396825397,141351.3396825397,233.61190476190475,1.9590256342119123,0.475348142593626,0.1872430226159474,20.642475734108604,375.9672055945035,0.21517592688716347,0.13370224155463814,0.11452455361172492,Energy,7 -ett2/H/long,Chronos-2-Synth,275.0396329365079,275.0396329365079,10.707279265873016,1.4048195895460436,0.23164182587883445,0.26973834971271493,11.523158553489685,16.58431888672272,0.22710948291917382,0.14662794861538947,0.1263864678082209,Energy,7 -ett2/H/medium,Chronos-2-Synth,305.04408482142856,305.04408482142856,11.085732886904761,1.2958629457973854,0.21209201943119507,0.2169320061093285,10.243040241363104,17.465511295734476,0.2353589816110964,0.14938737025762078,0.12334327764259978,Energy,7 -ett2/H/short,Chronos-2-Synth,109.75900297619047,109.75900297619047,6.547845168340774,0.7641742613943926,0.12328892319901438,0.11454206194196428,5.573024034067969,10.476593099676558,0.13287235061160613,0.08304489548087408,0.06595576729366907,Energy,7 -ett2/W/short,Chronos-2-Synth,2755152.285714286,2755152.285714286,1146.5311104910713,0.7490738241518227,0.11520302295684814,0.11881665672574725,9.738629449948728,1659.8651408214723,0.13923721196023262,0.09617636476867028,0.08516094585004161,Energy,7 -hierarchical_sales/D/short,Chronos-2-Synth,29.93920525010335,29.93920525010335,2.344711948377429,0.7522379948739162,0.6570393382671011,1.0760787806169905,6.943891672130294,5.471672984572757,1.6786486765439637,0.7193316596437495,0.5880571537219734,Sales,1 -hierarchical_sales/W/short,Chronos-2-Synth,445.4257150423729,445.4257150423729,8.748261015293961,0.7106052092971481,0.5180295228893289,0.4509641356387381,6.347031843204885,21.105111111822485,0.9689459744214586,0.4016369423048865,0.34014575182035167,Sales,1 -hospital/M/short,Chronos-2-Synth,2729.7179487179487,2729.7179487179487,17.56463392275098,0.7523552122671665,0.1918299814661085,0.1723113576001263,4.972380284574713,52.246702754508334,0.1896676634235546,0.06376369989646502,0.050546917307792254,Healthcare,1 -jena_weather/10T/long,Chronos-2-Synth,1425.414814814815,1425.414814814815,9.570665922619048,0.6440973923078107,0.7976430754771839,0.6528769841269841,6.652453068119601,37.75466613300686,0.23102727591587544,0.05856454587663657,0.05332259249899089,Nature,21 -jena_weather/10T/medium,Chronos-2-Synth,1774.8079365079366,1774.8079365079366,10.225537743506493,0.5883983170481839,0.6694474164636656,0.6547924839917028,5.966215182178999,42.12846942992276,0.25854166807194234,0.06275394337637634,0.05253747537721895,Nature,21 -jena_weather/10T/short,Chronos-2-Synth,985.1757936507937,985.1757936507937,6.222515966021826,0.2788170059661448,0.40503381854857134,0.5525265648251488,2.233221229554565,31.387510153734617,0.19495579170895896,0.03864962593833143,0.03223895604659491,Nature,21 -jena_weather/D/short,Chronos-2-Synth,392.39717261904764,392.39717261904764,9.9750248015873,1.1348972810988334,0.7375609112269987,0.45579611157614086,7.234589382430787,19.809017457184687,0.11928266143004805,0.060065952727630885,0.04742439027727278,Nature,21 -jena_weather/H/long,Chronos-2-Synth,2394.284656084656,2394.284656084656,14.77403480489418,1.267679650655475,3.1166675060435134,0.7130996300429894,9.252843387598933,48.931428101830996,0.29462143888950554,0.08895606691394892,0.06779476185436682,Nature,21 -jena_weather/H/medium,Chronos-2-Synth,1492.922123015873,1492.922123015873,10.89689205109127,0.8118882579809137,2.765196239358812,0.6971216110956101,6.8297068834007275,38.63835041789275,0.2364349008967942,0.06668000999829796,0.05497246892051641,Nature,21 -jena_weather/H/short,Chronos-2-Synth,1081.71835839599,1081.71835839599,8.474234903665414,0.5392669167965849,2.194339662941887,0.613093504431652,4.609704578589101,32.88948704975482,0.20161045442709344,0.05194651857188684,0.04272091692511895,Nature,21 -kdd_cup_2018/D/short,Chronos-2-Synth,2946.7586940621054,2946.7586940621054,21.5647815679234,1.2267338836124602,0.5152686024149711,0.4796934545702008,9.098753811722716,54.28405561545771,1.2160204488827209,0.4830739904189623,0.38230301851734116,Nature,1 -kdd_cup_2018/H/long,Chronos-2-Synth,4045.840837585906,4045.840837585906,23.808732372085938,1.0056682992128088,1.076596149901843,0.5937768684710073,9.066401376874248,63.606924446839166,1.4924754763869323,0.5586490700221827,0.4479404193809887,Nature,1 -kdd_cup_2018/H/medium,Chronos-2-Synth,5122.602203665755,5122.602203665755,25.06738688770607,1.0333821701122667,1.030373902272039,0.5458441571263707,9.974736199921653,71.57235642107752,1.4981390434693884,0.5247058067123463,0.42828807883426495,Nature,1 -kdd_cup_2018/H/short,Chronos-2-Synth,4759.541181607157,4759.541181607157,23.521448148089235,0.9617569356425049,0.9499539801827462,0.4992119404206415,7.9651435504467525,68.98942804232513,1.444073678745789,0.4923465104798509,0.3945809360384573,Nature,1 -loop_seattle/5T/long,Chronos-2-Synth,110.32625386996904,110.32625386996904,5.9895890666208,0.9444327821719863,0.19135297073873409,0.1337311070261438,7.260523282868716,10.503630509017777,0.18570633551131524,0.10589715963693039,0.0835461805348947,Transport,1 -loop_seattle/5T/medium,Chronos-2-Synth,93.02269744904541,93.02269744904541,5.525714351941435,0.8742128961352799,0.16884233787701963,0.12586975947991244,6.591585791776922,9.64482749711188,0.17160129518125328,0.09831381016184394,0.07745264908605397,Transport,1 -loop_seattle/5T/short,Chronos-2-Synth,48.79722339799407,48.79722339799407,3.742391479618163,0.5913434776927697,0.09477528826986181,0.08031280761655762,4.355787760342249,6.985500941091774,0.11978592256043054,0.06417375357167915,0.05033415212973805,Transport,1 -loop_seattle/D/short,Chronos-2-Synth,18.80484068627451,18.80484068627451,2.9243590363777088,0.8806483831727409,0.05431334630377645,0.053937233687677374,8.173845766333885,4.336454852327476,0.07749824794323451,0.052262207124005686,0.04288315051208741,Transport,1 -loop_seattle/H/long,Chronos-2-Synth,62.48869969040248,62.48869969040248,4.261671987874097,0.8896534133828808,0.12443774590428276,0.09878565444132698,7.3880852425383505,7.904979423781094,0.14127256556566875,0.07616178399087375,0.0612031220040873,Transport,1 -loop_seattle/H/medium,Chronos-2-Synth,69.3547342621259,69.3547342621259,4.5149038151444785,0.9257389258531883,0.11823686377305856,0.10304260568845136,7.973493319085672,8.327948982920459,0.14719554943906268,0.07980041053297351,0.06442098508401783,Transport,1 -loop_seattle/H/short,Chronos-2-Synth,62.57557896773668,62.57557896773668,4.181890931796888,0.8462548971023094,0.11863818807582763,0.09581890399384412,7.4272659976702124,7.910472739839047,0.14002341176115551,0.07402372211387612,0.059602970163296236,Transport,1 -m4_daily/D/short,Chronos-2-Synth,399669.5965392545,399669.5965392545,187.5868653152354,3.409987345053648,0.03918911039400183,0.031242069183919963,24.838969339227425,632.1942712009137,0.09765654392155923,0.028976986642042867,0.02285617322566967,Econ/Fin,1 -m4_hourly/H/short,Chronos-2-Synth,2212933.874396135,2212933.874396135,245.33572363123994,0.8475253507724109,0.1071399436864684,0.09156327547082578,5.974099355795926,1487.5933161977218,0.20308932288762546,0.03349374150172082,0.025676262348310278,Econ/Fin,1 -m4_monthly/M/short,Chronos-2-Synth,1814359.5064814815,1814359.5064814815,551.9609208622685,0.9152717496686643,0.1594462887092873,0.13064415645599364,7.435159795542234,1346.9816281157964,0.2799490248519171,0.11471642844001322,0.09115975093117218,Econ/Fin,1 -m4_quarterly/Q/short,Chronos-2-Synth,1713842.6106666666,1713842.6106666666,549.8309615885416,1.1549957558124753,0.11340904029210408,0.09985374609629313,8.918739896482842,1309.1381174905368,0.219126667694775,0.09203202076130969,0.07364921509746819,Econ/Fin,1 -m4_weekly/W/short,Chronos-2-Synth,268961.76215984573,268961.76215984573,258.3934004713949,2.077450928837584,0.061158979018239766,0.06148336642248768,13.836780738282558,518.6152351790735,0.09448365869346564,0.047075273155740074,0.03730339670014975,Econ/Fin,1 -m4_yearly/A/short,Chronos-2-Synth,3685389.7839296595,3685389.7839296595,920.2356286816981,3.5049862211654736,0.17396093327406684,0.1532782643764475,32.57133348396228,1919.7369048725557,0.30783871133677393,0.1475640486675783,0.12069973668436416,Econ/Fin,1 -m_dense/D/short,Chronos-2-Synth,7726.954074074074,7726.954074074074,40.77435763888889,0.617717798937793,0.11451142058715368,0.08892012419524016,6.5849827282838485,87.90309479235685,0.15217325965655212,0.07058644439047965,0.059470267920516524,Transport,1 -m_dense/H/long,Chronos-2-Synth,46976.205432098766,46976.205432098766,103.92342592592593,0.9525571878203053,0.34672881948744544,0.2579320987654321,8.566293208386405,216.73994886060754,0.3752088914682348,0.17990681295364488,0.1485976291830834,Transport,1 -m_dense/H/medium,Chronos-2-Synth,46059.008888888886,46059.008888888886,100.5235329861111,0.9139888536317566,0.3380331183367013,0.2491665988498264,8.112792744029537,214.61362698787067,0.3734005974080157,0.1748982475968297,0.14224952942399563,Transport,1 -m_dense/H/short,Chronos-2-Synth,57251.253333333334,57251.253333333334,101.31924479166666,0.8628583175659563,0.32152140552001257,0.23927371554904514,9.759521191219998,239.27234134628543,0.424409510702401,0.17971509312284167,0.14882894271228334,Transport,1 -restaurant/D/short,Chronos-2-Synth,139.3336368730737,139.3336368730737,7.089800335061175,0.6795262994076543,0.6784850156439712,0.3913678879763239,4.505903622611708,11.803966997288398,0.5432459107319507,0.3262890383218424,0.25532748124844856,Sales,1 -saugeen/D/short,Chronos-2-Synth,1467.0227083333334,1467.0227083333334,14.2811572265625,3.173431267746448,0.28979273478190104,0.3482537333170573,34.74162723364932,38.30173244558702,1.2406022398908698,0.4625700852742611,0.39102529437017286,Nature,1 -saugeen/M/short,Chronos-2-Synth,385.9129231770833,385.9129231770833,12.179532005673362,0.7136291846141731,0.34761120024181547,0.34681401933942524,4.566374263048426,19.644666532600734,0.5897918227563019,0.3656660890641146,0.28270054296690506,Nature,1 -saugeen/W/short,Chronos-2-Synth,955.3931640625,955.3931640625,14.646339416503906,1.1881830354589615,0.3744035720825195,0.38387365341186525,9.0132699390077,30.909434871289704,0.9330269935678432,0.4421119340893548,0.35342588890179716,Nature,1 -solar/10T/long,Chronos-2-Synth,24.5850122604927,24.5850122604927,1.9476960688805252,0.8515239529034765,2.3906434573553565,1.4782443124000466,6.186527518669835,4.9583275668810645,1.0716630712707336,0.42096330323561004,0.3363057070118321,Energy,1 -solar/10T/medium,Chronos-2-Synth,27.341969420482194,27.341969420482194,2.0353347572439726,0.8903335704909898,3.3641751713349226,1.471654501216545,5.364971100377557,5.228954907099715,1.1603258153341194,0.4516488483139158,0.34691033045746733,Energy,1 -solar/10T/short,Chronos-2-Synth,23.32006491408151,23.32006491408151,1.95843954341545,0.8593865848094706,4.738321593485067,1.514620221734337,4.215776608762778,4.82908530822158,1.4047803932041985,0.5697098510937241,0.43306058940099906,Energy,1 -solar/D/short,Chronos-2-Synth,122546.70170316302,122546.70170316302,247.4690389294404,0.9650313836985104,1.05850227151764,0.422403704046913,4.9920261452640124,350.06671036127244,0.5057271622187577,0.3575084722155706,0.2692073299316702,Energy,1 -solar/H/long,Chronos-2-Synth,768.8025141930251,768.8025141930251,12.26878801703163,0.9344835660449682,4.540054033870689,1.4032767513179238,6.365052569492576,27.727288258916072,0.9613556154835421,0.42538123978127645,0.3308750043752249,Energy,1 -solar/H/medium,Chronos-2-Synth,833.2734184914842,833.2734184914842,11.766320711678832,0.899750685749648,4.488155582510605,1.3976797730383212,5.845376977920172,28.86647568532543,1.0398378288694001,0.42385033476161393,0.33455320822860907,Energy,1 -solar/H/short,Chronos-2-Synth,792.829843770009,792.829843770009,12.105636825297733,0.9351976803501281,4.151352159295716,1.4094980801399026,5.870175135572951,28.15723430612476,1.0378850631462895,0.4462178175687237,0.35625857347291656,Energy,1 -solar/W/short,Chronos-2-Synth,1588413.897810219,1588413.897810219,1013.9396669708029,1.1094004069973584,0.22928957695508526,0.19911461154909899,6.952774102376596,1260.3229339380518,0.2572634204451871,0.206970435771491,0.14972938178639336,Energy,1 -sz_taxi/15T/long,Chronos-2-Synth,15.47737045940171,15.47737045940171,2.6929829393696583,0.5082846064566703,5754869507219.619,0.4004704402043269,4.295114259807665,3.934128932737425,0.36413596608264365,0.24925770381122805,0.2016473063849256,Transport,1 -sz_taxi/15T/medium,Chronos-2-Synth,16.31565671741453,16.31565671741453,2.7639354216746796,0.5401290748943555,8826194502402.188,0.408639656784188,4.3057667547856875,4.039264378251878,0.3759209717918787,0.25723032522461714,0.20567466039995114,Transport,1 -sz_taxi/15T/short,Chronos-2-Synth,16.57537595867674,16.57537595867674,2.723099127502814,0.5440025514925346,1008882213134.1923,0.39486388321761245,3.890350356598083,4.0712867693981885,0.3806827251110475,0.25462141463410626,0.20040406575972258,Transport,1 -sz_taxi/H/short,Chronos-2-Synth,7.213133722288996,7.213133722288996,1.8414129273504274,0.5608318283302823,0.9596420679336939,0.29430187257946044,4.074081992224543,2.6857277826110737,0.25019211627498367,0.17153897733522913,0.13561967251201457,Transport,1 -temperature_rain/D/short,Chronos-2-Synth,186.346511214151,186.346511214151,5.784900465951639,1.3516999154066685,18.42305641096644,1.5202821906495776,19.451127278511645,13.650879503319594,1.607069417588225,0.6810357252338202,0.5526510310620748,Nature,1 -us_births/D/short,Chronos-2-Synth,115898.56,115898.56,206.97010416666666,0.30490546617655323,0.019947195053100587,0.019760162035624185,2.4289878839676087,340.43877569983124,0.03191337272053306,0.019401767800099803,0.015448107758455905,Healthcare,1 -us_births/M/short,Chronos-2-Synth,40516794.666666664,40516794.666666664,5376.364583333333,0.6074192832476859,0.016499829789002735,0.01666366308927536,3.7589275695673012,6365.280407544247,0.019770517547898873,0.016698951740242683,0.012380833574658938,Healthcare,1 -us_births/W/short,Chronos-2-Synth,2086647.4285714286,2086647.4285714286,1118.771763392857,1.0176816047982542,0.015155343072754996,0.015174633690289088,6.560831048680451,1444.5232530393648,0.019609261518111445,0.015187216987535331,0.011958782913582013,Healthcare,1 diff --git a/results/chronos-2-synth/config.json b/results/chronos-2-synth/config.json deleted file mode 100644 index ff1a872f34f4c42de8de2733313014796ccd7379..0000000000000000000000000000000000000000 --- a/results/chronos-2-synth/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "Chronos-2-Synth", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/autogluon/chronos-2-synth", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos-2.ipynb", - "org": "AWS", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} diff --git a/results/chronos-2/all_results.csv b/results/chronos-2/all_results.csv deleted file mode 100644 index 7103137e5faaf6fcd887a091342cd2c41a09367b..0000000000000000000000000000000000000000 --- a/results/chronos-2/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/5T/long,Chronos-2,4145797.488463595,4145797.488463595,357.97367393467607,0.8711999929238592,3.7828414394588368,0.8360760492923797,15.432955463966756,2036.1231515955992,5.380865066630999,0.946017452499958,0.7026061445524241,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Chronos-2,2854813.8535229373,2854813.8535229373,267.82598383026493,0.9588490240362445,3.4515445017801873,0.8189011494265185,20.728725080171767,1689.6194404430062,5.133530703852618,0.8137293395024578,0.6233457308991365,Web/CloudOps,2 -bitbrains_fast_storage/5T/short,Chronos-2,1828932.7515211524,1828932.7515211524,159.031559686963,0.6563897676548155,1.9940338153085935,0.7430846129850146,13.074593783884868,1352.3804019288184,4.246139955221287,0.499319761484916,0.3906378769842435,Web/CloudOps,2 -bitbrains_fast_storage/H/short,Chronos-2,3005434.042481728,3005434.042481728,299.2353117583048,0.9309593597654008,2.7620977169722725,0.5810333336412573,18.11250242174507,1733.6187708033528,4.941376627389847,0.8529178389819855,0.6635224677367416,Web/CloudOps,2 -bitbrains_rnd/5T/long,Chronos-2,2314407.0498213544,2314407.0498213544,165.33585963632635,3.2753978851024095,1.2795751991636328,0.7092483613458547,115.21846073163633,1521.3175374724879,5.8278244831833454,0.6333644009242041,0.8730126297517127,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Chronos-2,2303161.976669191,2303161.976669191,157.1222612676236,4.359229717604128,0.8530047134127788,0.7383417746560365,159.03573459737711,1517.617203602144,6.385389278797625,0.6610934563586716,1.004565433626743,Web/CloudOps,2 -bitbrains_rnd/5T/short,Chronos-2,1786863.6929437863,1786863.6929437863,125.62732040874121,1.618136938121414,1.025217456216353,0.671741078144912,51.66287752931445,1336.7362091840657,5.469538473681133,0.514030709724316,0.415745145404104,Web/CloudOps,2 -bitbrains_rnd/H/short,Chronos-2,2064867.4331017432,2064867.4331017432,161.80485699671794,5.804742667777643,1.6657797654211588,0.5561468700716082,208.00435712396174,1436.9646596565078,6.288860980407804,0.7081372842185025,0.8035454819748182,Web/CloudOps,2 -bizitobs_application/10S/long,Chronos-2,5508385.564444444,5508385.564444444,1444.5294444444444,2.9496631708273595,0.04264470418294271,0.043645833333333335,21.627569590086416,2346.995007332662,0.09051421622566853,0.0557097267230176,0.04539597373122299,Web/CloudOps,2 -bizitobs_application/10S/medium,Chronos-2,2196085.12,2196085.12,830.188125,1.8026079477842636,0.02545076847076416,0.02595753033955892,17.99670644890125,1481.9194040162913,0.05819061344630824,0.032599044279103956,0.025662633137848945,Web/CloudOps,2 -bizitobs_application/10S/short,Chronos-2,347597.08444444445,347597.08444444445,322.35493055555554,1.003902166736118,0.01409513579474555,0.014232810338338215,6.047178036464015,589.5736463279583,0.022737498945655292,0.012431941182046274,0.009567255598719189,Web/CloudOps,2 -bizitobs_l2c/5T/long,Chronos-2,96.63757936507936,96.63757936507936,6.570953621031746,0.6471512945825063,0.550500069975595,0.811466548859127,5.148691017725035,9.830441463387052,0.5519645916327948,0.36894922222762233,0.2977954916750903,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Chronos-2,81.7203762755102,81.7203762755102,5.994177163052721,0.5826103672409443,0.44157504104487144,0.6657834007626489,4.4808924084573665,9.03993231586997,0.47563337777660464,0.3153818669691863,0.24720505695342634,Web/CloudOps,7 -bizitobs_l2c/5T/short,Chronos-2,18.978057570684523,18.978057570684523,2.549083310081845,0.26651688823943137,0.14929786712825943,0.20394537789481026,2.299294813002909,4.356381247168861,0.14961576532832183,0.08754583877878035,0.06914039860435806,Web/CloudOps,7 -bizitobs_l2c/H/long,Chronos-2,85.68464781746032,85.68464781746032,5.357755921378968,0.5712635894958724,0.6297144659223869,0.7843813941592261,5.135881343364576,9.256600229968901,0.5654198582399772,0.3272671951136721,0.2672670630808577,Web/CloudOps,7 -bizitobs_l2c/H/medium,Chronos-2,76.57036830357143,76.57036830357143,4.936408923921131,0.4979993374385745,0.4964170399098629,0.7691191173735119,3.3091562434700794,8.750449605795774,0.5298530148006348,0.29890705831800585,0.2359357980831375,Web/CloudOps,7 -bizitobs_l2c/H/short,Chronos-2,47.4854484437004,47.4854484437004,4.190994989304316,0.41262127782541824,0.37390458726768583,0.5874816652328249,2.7231773555636156,6.890968614331399,0.3714408951824537,0.22590538684138434,0.17649751439409647,Web/CloudOps,7 -bizitobs_service/10S/long,Chronos-2,131188.08042328042,131188.08042328042,76.91575396825397,1.3255743262231172,0.07917331917576058,0.07955421730324073,18.17709193096694,362.1989514386816,0.2683285624352049,0.056981649474530625,0.0511030149757811,Web/CloudOps,2 -bizitobs_service/10S/medium,Chronos-2,37981.88444444445,37981.88444444445,39.59513888888889,0.9790487780830548,0.06515108623201885,0.0587854488312252,10.441729982856108,194.8894159374604,0.1466783495022143,0.029800231031514076,0.02211947431703989,Web/CloudOps,2 -bizitobs_service/10S/short,Chronos-2,1836.175238095238,1836.175238095238,17.853725198412697,0.71692567427202,0.05574358258928572,0.04660181439112103,5.53490701103776,42.85061537592241,0.03174510577022393,0.013226610396232215,0.010125705005055306,Web/CloudOps,2 -car_parts/M/short,Chronos-2,1.3728812440578502,1.3728812440578502,0.45334444113171746,0.8359773797258253,0.8484361781497719,1.903766687321476,15.376365499809188,1.1717001510872354,2.8096167687905766,1.0870734655617833,0.9660464513713963,Sales,1 -covid_deaths/D/short,Chronos-2,423913.4796992481,423913.4796992481,111.35696271929825,32.53917305471355,0.08699892892236405,0.3613865596608709,604.5420741472494,651.0863842066183,0.24497442060846983,0.04189859914843811,0.034780098757498136,Healthcare,1 -electricity/15T/long,Chronos-2,311064.08655255253,311064.08655255253,56.517862706456455,0.8554108103088559,0.12106029066856129,0.1287547336154514,7.247739954641148,557.7311956064073,0.8798893135132914,0.08916385493544576,0.07167304284461921,Energy,1 -electricity/15T/medium,Chronos-2,259282.1163963964,259282.1163963964,51.24800267454955,0.8041393284984164,0.11774989078055213,0.12681871816274282,7.066440269952036,509.1975219857186,0.877398605698191,0.08830546919418839,0.07142175805937691,Energy,1 -electricity/15T/short,Chronos-2,126343.6006644144,126343.6006644144,47.61598025760135,0.9223486782234586,0.14979325660587203,0.15631814595815297,10.501715858896683,355.4484500801972,0.7006589590333686,0.09386048287203685,0.07837870526782645,Energy,1 -electricity/D/short,Chronos-2,1642103353.1594956,1642103353.1594956,4487.857657657658,1.4148597876653912,0.2449856870525856,0.0928476342553491,11.019876609629192,40522.87444344855,0.6659254039539466,0.0737504055329091,0.05806908852292287,Energy,1 -electricity/H/long,Chronos-2,9137662.634666666,9137662.634666666,282.3887507507508,1.1925033326651895,0.29071461624112027,0.14178493337087086,9.865966042933266,3022.856700981154,1.1922137887853903,0.11137403977292432,0.08775034376303065,Energy,1 -electricity/H/medium,Chronos-2,6055900.268108108,6055900.268108108,244.40511542792794,1.0691861062780872,0.23166439194444644,0.12954365223377676,8.94221628330075,2460.8738830155658,0.9605601728525409,0.09539937074459606,0.07593499533151156,Energy,1 -electricity/H/short,Chronos-2,1843608.664054054,1843608.664054054,176.86290153434686,0.9779272228502356,0.1562493743003312,0.1299548374210392,10.392534661159486,1357.7955162888313,0.6409764491330397,0.08349191999006204,0.06793049481322216,Energy,1 -electricity/W/short,Chronos-2,75884883990.14053,75884883990.14053,31256.063063063062,1.3850606026911376,0.12462861973381388,0.09090743193755278,10.443927982377515,275472.1110931931,0.6285364514202042,0.07131602137530944,0.05590120453637864,Energy,1 -ett1/15T/long,Chronos-2,8.841597222222223,8.841597222222223,1.5811059957837301,1.0133727405348167,0.7827073009652279,0.38617210751488096,8.330647018479825,2.9734823393156757,0.5667486738823136,0.30136036610327815,0.24113485656719547,Energy,7 -ett1/15T/medium,Chronos-2,9.473058655753968,9.473058655753968,1.5767613777281746,0.9859937447971706,0.7630063625767908,0.37749449714781746,7.507395887777427,3.077833435349283,0.5866380266337106,0.30053224208922674,0.23303125605934719,Energy,7 -ett1/15T/short,Chronos-2,5.543573869977679,5.543573869977679,1.1087347848074778,0.6869282579853626,0.49065431432400236,0.23012150355747768,4.892709657054664,2.354479532715814,0.4450945883523978,0.20959700255565278,0.1652939685623463,Energy,7 -ett1/D/short,Chronos-2,41941.88888888889,41941.88888888889,134.4664806547619,1.6403424775100648,1.3372307671440973,0.46783844478546627,11.555169672110857,204.79718965085652,0.5379066118101579,0.353180671738464,0.2742107738380927,Energy,7 -ett1/H/long,Chronos-2,149.0723544973545,149.0723544973545,7.340656001984127,1.380466540478325,3420950115626.8896,0.46889803447420636,10.575395748311882,12.209519011711906,0.5846874214886253,0.3515281171779196,0.2751785632214682,Energy,7 -ett1/H/medium,Chronos-2,140.37652529761905,140.37652529761905,6.97626720610119,1.273777521133431,3687778958273.7476,0.4353119623093378,9.667693706636634,11.848059980335137,0.5674939873309317,0.33414666198916376,0.26059463857135295,Energy,7 -ett1/H/short,Chronos-2,88.84915364583334,88.84915364583334,4.728027925037202,0.7867823367050838,0.437381344749814,0.24990012759254093,5.607514495280123,9.42598290078193,0.43994918527277593,0.22067640642489356,0.17056401315691946,Energy,7 -ett1/W/short,Chronos-2,1482215.4285714286,1482215.4285714286,943.4799107142857,1.5900649002196825,0.734635625566755,0.5536167621612549,9.080652070692834,1217.4627011007067,0.48445746686206087,0.37543317521485003,0.27142729507349506,Energy,7 -ett2/15T/long,Chronos-2,11.688671875,11.688671875,2.136253410218254,0.8903270215343758,0.1369816222269947,0.16666164822048612,7.1919007730020015,3.4188699704727,0.1821977572892066,0.11384480360607892,0.09289448313644817,Energy,7 -ett2/15T/medium,Chronos-2,11.215238095238096,11.215238095238096,2.059312220982143,0.8487085901964821,0.1280561590255245,0.1561493694971478,6.484077334641768,3.348915958222615,0.17846978159682161,0.1097444686290088,0.08738895504218223,Energy,7 -ett2/15T/short,Chronos-2,6.982102167038691,6.982102167038691,1.678560529436384,0.7261654250803261,0.10393601476211456,0.12403215680803571,4.924883285917772,2.6423667737539183,0.12544255085333136,0.07968723973738098,0.06219244724710097,Energy,7 -ett2/D/short,Chronos-2,105162.81904761905,105162.81904761905,194.35518353174604,1.2907810501715817,0.41752886090959823,0.13291920253208706,14.44876356539086,324.28817284572534,0.1855986562984518,0.11123458679232107,0.09419598685725401,Energy,7 -ett2/H/long,Chronos-2,241.76742724867725,241.76742724867725,9.487003968253969,1.0480351820112475,0.1770615728432216,0.17451640134135252,8.920745325257933,15.548872217903048,0.21292983772830115,0.129917217607728,0.10505843968023981,Energy,7 -ett2/H/medium,Chronos-2,272.4251302083333,272.4251302083333,10.183489118303571,1.0901769049896204,0.17723347685298205,0.17325506664457777,7.853810714208072,16.505306122830117,0.222419600232226,0.13722905602637375,0.10911511025055304,Energy,7 -ett2/H/short,Chronos-2,107.96099330357143,107.96099330357143,6.392679268973215,0.736262066815625,0.11775483856223916,0.11017581394740514,5.0969814789571055,10.39042796537137,0.13177953696245825,0.08107696014277567,0.06424452613433201,Energy,7 -ett2/W/short,Chronos-2,3115530.0,3115530.0,1194.0851004464287,0.771557833167684,0.12665045261383057,0.13521315370287215,11.857089292788105,1765.086400151562,0.14806366082511788,0.10016541473190521,0.08963468571660917,Energy,7 -hierarchical_sales/D/short,Chronos-2,29.65272840016536,29.65272840016536,2.3113032567693264,0.7440117573590805,0.6363216025673376,1.077673092574411,6.7952369747894625,5.445431883713665,1.670598197403056,0.7090822430373245,0.5788581494406615,Sales,1 -hierarchical_sales/W/short,Chronos-2,452.33736096398303,452.33736096398303,8.796943276615465,0.709343444262644,0.5139163476084444,0.45117019394696767,6.659061572975727,21.268224208052327,0.9764345764539204,0.40387196873442194,0.34072688823986497,Sales,1 -hospital/M/short,Chronos-2,2539.3183398522383,2539.3183398522383,17.25815032866145,0.7399125269543689,0.19340434897106556,0.17136815090585614,4.875913348448958,50.391649505173355,0.18293338935881895,0.06265109328008678,0.050963799382019365,Healthcare,1 -jena_weather/10T/long,Chronos-2,1327.8632275132275,1327.8632275132275,9.587346023478837,0.6918561732061718,0.8754758130744825,0.6431797572544643,6.041743119123887,36.439857676906854,0.2229817375208641,0.058666614273958986,0.051384381593606226,Nature,21 -jena_weather/10T/medium,Chronos-2,1531.8712842712844,1531.8712842712844,9.632253336940837,0.6057200244714774,0.7589422149086217,0.6483931812545094,5.420471604944751,39.139127280399144,0.24019612843487564,0.05911296751871536,0.04960599787793978,Nature,21 -jena_weather/10T/short,Chronos-2,924.9469246031746,924.9469246031746,5.812009781125992,0.26495034895161235,0.36129278455129765,0.5461646185980903,2.2516787743464732,30.41294008482532,0.18890248966526502,0.03609986783755168,0.030310675322483785,Nature,21 -jena_weather/D/short,Chronos-2,384.9302579365079,384.9302579365079,9.841698443700396,1.0822749224001693,0.6937773263999049,0.45297648111979166,6.705876677746834,19.619639597518297,0.11814229719109687,0.05926311014132397,0.04674940096158798,Nature,21 -jena_weather/H/long,Chronos-2,1441.8238095238096,1441.8238095238096,12.078509424603174,1.0084402648660813,2.5280535480459307,0.6797292235036375,8.152702147613358,37.971355118349535,0.22862965001289043,0.07272601606704138,0.058572540522816546,Nature,21 -jena_weather/H/medium,Chronos-2,1267.605257936508,1267.605257936508,10.338327752976191,0.7856590893334519,2.5514887758105416,0.6837812635633681,5.938333913704131,35.60344446730552,0.2178637745444152,0.06326205625438838,0.05033272427534201,Nature,21 -jena_weather/H/short,Chronos-2,1080.4010025062657,1080.4010025062657,8.565387263732248,0.5226186910177899,1.7302937784127035,0.6081429770715853,4.334729695255024,32.86945394292801,0.20148765276208633,0.05250527671570877,0.041877829764344655,Nature,21 -kdd_cup_2018/D/short,Chronos-2,2817.109648247889,2817.109648247889,20.892149162178335,1.1977513994294262,0.5028700768626571,0.4639661489502294,8.735722325835347,53.0764509763783,1.188968823527226,0.4680063107717131,0.3665565381873796,Nature,1 -kdd_cup_2018/H/long,Chronos-2,4165.433668656734,4165.433668656734,24.78768872926688,1.046908916534262,1.0979405344428492,0.6230810785178198,7.358994805141847,64.54017096860477,1.5143732109381864,0.5816193420208982,0.44533304415352265,Nature,1 -kdd_cup_2018/H/medium,Chronos-2,4983.18968989833,4983.18968989833,25.004080862510037,1.0401058282304305,1.1041164935683345,0.5461257448671087,8.564919389661053,70.59171119825847,1.4776123629245475,0.5233806969524404,0.41721991436439515,Nature,1 -kdd_cup_2018/H/short,Chronos-2,4297.654353139412,4297.654353139412,22.612387310053403,0.9466823637236516,0.9366268895143116,0.4948653785455921,7.212414950429725,65.55649741360052,1.3722162231533168,0.47331822069926655,0.37364788063282156,Nature,1 -loop_seattle/5T/long,Chronos-2,98.44317165462677,98.44317165462677,5.780385778580438,0.9113433908564642,0.1925361980958319,0.13005866653587606,6.227033630065236,9.921853236902205,0.17542039436024412,0.10219840271992542,0.0795193568123974,Transport,1 -loop_seattle/5T/medium,Chronos-2,84.09893567788957,84.09893567788957,5.332555741663442,0.845525262307259,0.17424029087503629,0.12172835374641222,5.885399477836339,9.170547185304134,0.1631628740887507,0.0948771216664779,0.07417131365244178,Transport,1 -loop_seattle/5T/short,Chronos-2,40.922246375935245,40.922246375935245,3.4197557321396896,0.5407390822395204,0.09625281244362593,0.07281893089459776,4.034108370509722,6.397049818153306,0.10969528464665351,0.05864126263243339,0.046210203008454326,Transport,1 -loop_seattle/D/short,Chronos-2,18.49518188854489,18.49518188854489,2.945795036764706,0.8877604452918465,0.054490638000677244,0.05431893961840493,7.093401860268932,4.300602502969194,0.07685751850996846,0.052645297120206415,0.042448193010935185,Transport,1 -loop_seattle/H/long,Chronos-2,60.37068283453733,60.37068283453733,4.181222318971448,0.8735345852417211,0.12331930973458462,0.09723127566541967,6.758270331885551,7.769857323949863,0.1388577527389141,0.07472404070079071,0.05933858722158785,Transport,1 -loop_seattle/H/medium,Chronos-2,68.24554308565531,68.24554308565531,4.435131659571724,0.9076093962599161,0.1153957118928986,0.10154155026525413,7.288989500176872,8.261086071798024,0.14601375510291262,0.07839044677196205,0.06281535872065858,Transport,1 -loop_seattle/H/short,Chronos-2,58.21938014298517,58.21938014298517,4.065773720109581,0.8250143479307347,0.11093648274739583,0.09368553532503072,6.746109670181771,7.630162524021698,0.13506163588996406,0.07196832938585822,0.05766227430781137,Transport,1 -m4_daily/D/short,Chronos-2,366856.0178444692,366856.0178444692,187.08370984571968,3.419271586278344,0.03769884951040765,0.031085941330554773,25.388629343583204,605.686402228471,0.09356181072244804,0.028899263026935136,0.022945916854759427,Econ/Fin,1 -m4_hourly/H/short,Chronos-2,1487109.9774557166,1487109.9774557166,222.1364231078905,0.7884232659057172,0.0983312732739533,0.08467229821639745,5.530164495328659,1219.4711876283575,0.16648473415397755,0.030326524908683,0.023215974823091404,Econ/Fin,1 -m4_monthly/M/short,Chronos-2,1788827.632925926,1788827.632925926,541.8757770543981,0.903217352832543,0.1552360033105921,0.128430229222333,7.411150271333955,1337.4706101166955,0.27797230879386525,0.11262038932888294,0.08985216836466069,Econ/Fin,1 -m4_quarterly/Q/short,Chronos-2,1714437.684,1714437.684,548.0652265625,1.1542560705399063,0.1132428429921468,0.09991071987152099,8.788063496846688,1309.3653745230931,0.21916470652015863,0.09173646781153375,0.07320823429107878,Econ/Fin,1 -m4_weekly/W/short,Chronos-2,254358.4349689308,254358.4349689308,251.72085922434113,2.02628121593723,0.06053978670887347,0.05966463123727769,14.571456668314367,504.33960281632733,0.09188285971128889,0.045859639547161234,0.03674363864157218,Econ/Fin,1 -m4_yearly/A/short,Chronos-2,3443777.201590203,3443777.201590203,862.5352753837672,3.2361825352858733,0.16316434131240523,0.14555810283152684,27.968301187299257,1855.741685038681,0.29757678119657,0.13831152955528317,0.111543749102192,Econ/Fin,1 -m_dense/D/short,Chronos-2,6930.688888888889,6930.688888888889,39.78548611111111,0.6150854754190758,0.1107411875912134,0.08871422661675347,6.180143266910618,83.250759088965,0.14411937838338387,0.06887456150263639,0.057821470734883224,Transport,1 -m_dense/H/long,Chronos-2,36285.14765432099,36285.14765432099,79.24003086419754,0.6824050396489317,0.2785773710583145,0.19208964783468363,7.31249086395161,190.48660754583506,0.32976047670281394,0.1371762072325005,0.1133079726778582,Transport,1 -m_dense/H/medium,Chronos-2,39619.293333333335,39619.293333333335,80.905859375,0.6970900925375951,0.2804773785363389,0.19406134711371528,8.459217312139414,199.04595784223636,0.34631481986063417,0.14076597394321705,0.11721631317369591,Transport,1 -m_dense/H/short,Chronos-2,46501.24444444444,46501.24444444444,87.80433159722222,0.781713995408606,0.2992745273058887,0.20855792575412327,8.266028130014432,215.64147199563547,0.3824942369095682,0.15574300481641093,0.1283405502433509,Transport,1 -restaurant/D/short,Chronos-2,137.94775614084244,137.94775614084244,7.066059295320819,0.6776595370625726,0.6683491779618473,0.38976390664401794,4.546086017289033,11.745116267659611,0.5405374637986634,0.3251964206655548,0.2543246092294533,Sales,1 -saugeen/D/short,Chronos-2,1310.9584375,1310.9584375,13.425966796875,2.9834124400923168,0.27905431111653645,0.3306080627441406,26.012261534478686,36.207160030855775,1.172758540313552,0.4348702634943774,0.3493609624403107,Nature,1 -saugeen/M/short,Chronos-2,394.7450241815476,394.7450241815476,12.348471505301339,0.723536495658706,0.35381516956147696,0.3544542902991885,4.29725689632799,19.86819126597959,0.5965027058202548,0.3707381596566985,0.2872324741828958,Nature,1 -saugeen/W/short,Chronos-2,961.8529296875,961.8529296875,14.63895263671875,1.1875833278434742,0.3702629566192627,0.3817787170410156,7.879954986137159,31.013753879327474,0.9361759495693607,0.4418889579992454,0.34686226133847486,Nature,1 -solar/10T/long,Chronos-2,19.23235613721614,19.23235613721614,1.8166723494208739,0.7941239726053174,3.5173645623054024,1.441211769122567,3.818120671119489,4.385471027975917,0.9478493075369309,0.3926446252718286,0.29006620400278477,Energy,1 -solar/10T/medium,Chronos-2,19.42522395487724,19.42522395487724,1.8085002626631277,0.7899656500667791,3.620400844569428,1.441321532501106,3.6594001896293933,4.407405580937298,0.9780207642000411,0.4013133750603495,0.29022404279507985,Energy,1 -solar/10T/short,Chronos-2,18.993281915298052,18.993281915298052,1.7785098333428375,0.7784342503824689,4.95596794503629,1.4938188761690236,3.345156606542752,4.358128258243217,1.2677790383669734,0.5173683179187785,0.3859580534483757,Energy,1 -solar/D/short,Chronos-2,115518.37274939172,115518.37274939172,246.50973236009733,0.9610008143028058,1.0288868850745134,0.42358564762013384,4.821040670497025,339.8799387274744,0.4910107468669106,0.35612260096688486,0.266607200888623,Energy,1 -solar/H/long,Chronos-2,863.2689375506894,863.2689375506894,13.061734590429847,0.9951096739685439,4.7256056823511985,1.4152125152068127,4.769305578292617,29.381438656925727,1.0187080243859727,0.4528741425850578,0.3396900582695741,Energy,1 -solar/H/medium,Chronos-2,917.9939781021898,917.9939781021898,14.019875304136253,1.0693943523681286,3.7866529688614436,1.4422406050410583,4.416687057144223,30.298415438801246,1.09141964095334,0.5050286310040903,0.36297925188459823,Energy,1 -solar/H/short,Chronos-2,772.562908182866,772.562908182866,12.760016237274298,0.9844577475332202,3.8046338085005247,1.423929264060299,4.149593147517625,27.795015887436833,1.0245335712254053,0.47033846130585366,0.3462130094907792,Energy,1 -solar/W/short,Chronos-2,1365023.299270073,1365023.299270073,931.693772810219,1.0123458246669514,0.20600950978968266,0.18294669937913435,7.145382261082743,1168.3421156793386,0.2384878358046343,0.19018199252448056,0.13870243020271988,Energy,1 -sz_taxi/15T/long,Chronos-2,15.539048032407408,15.539048032407408,2.702750233707265,0.5097233936474667,6160711490300.354,0.4007208060674858,4.202062870264974,3.9419599227297337,0.3648607885668546,0.25016174717646755,0.20111065809505738,Transport,1 -sz_taxi/15T/medium,Chronos-2,16.26090077457265,16.26090077457265,2.7383766109107905,0.5346223799157326,9442351611960.014,0.4037479498447516,4.004753579868697,4.0324807221575965,0.3752896393628863,0.25485165126805776,0.2020288543113543,Transport,1 -sz_taxi/15T/short,Chronos-2,16.578431889833258,16.578431889833258,2.7196888533558456,0.5429398753552315,1163949985076.9463,0.39440400958497884,3.8454181564168928,4.071662054963951,0.38071781591647613,0.25430253941622627,0.2001496307577382,Transport,1 -sz_taxi/H/short,Chronos-2,7.197291186732104,7.197291186732104,1.8412136501736112,0.5603278951066386,0.9449468759390024,0.2947785467164129,3.896584220508918,2.6827767679648833,0.24991721105031078,0.1715204134364916,0.1353093467050623,Transport,1 -temperature_rain/D/short,Chronos-2,178.7010383967318,178.7010383967318,5.6377336581088375,1.3102429840566732,19.851612468792222,1.4950144504601448,18.279545675244645,13.36791077157279,1.5737565167737806,0.6637103011751919,0.5387614883878814,Nature,1 -us_births/D/short,Chronos-2,136457.62666666668,136457.62666666668,221.83609375,0.32657256102272725,0.021253010431925456,0.021217021942138672,2.5987014645137263,369.40171448799026,0.03462841321123049,0.0207953336929897,0.016516423276982047,Healthcare,1 -us_births/M/short,Chronos-2,43295656.0,43295656.0,5612.541666666667,0.6328374062505758,0.017351916680733364,0.01754515866438548,3.6498563216851756,6579.943464802718,0.020437259540189448,0.017432516150841005,0.012520092252886573,Healthcare,1 -us_births/W/short,Chronos-2,1955762.5714285714,1955762.5714285714,1076.337123325893,0.9789677198038242,0.014646285346576146,0.014642353568758284,5.710766410140946,1398.4858138102693,0.01898430779475114,0.014611170909530551,0.011395826425098727,Healthcare,1 \ No newline at end of file diff --git a/results/chronos-2/config.json b/results/chronos-2/config.json deleted file mode 100644 index 6b932868f97081b16db87e4983e027057e4430b1..0000000000000000000000000000000000000000 --- a/results/chronos-2/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "Chronos-2", - "model_type": "pretrained", - "model_dtype": "float32", - "model_link": "https://huggingface.co/amazon/chronos-2", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos-2.ipynb", - "org": "AWS", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/chronos_base/config.json b/results/chronos_base/config.json index 7812c79f62b1f61e2e804b43ce2a6a1e8c6918ff..ba962e3824e1e0deff839c439eb5ee6a97b1db84 100644 --- a/results/chronos_base/config.json +++ b/results/chronos_base/config.json @@ -3,8 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/amazon/chronos-t5-base", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb", - "org": "AWS AI Labs", - "testdata_leakage": "Yes", - "replication_code_available": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb" } \ No newline at end of file diff --git a/results/chronos_bolt_base/config.json b/results/chronos_bolt_base/config.json index 7da531493b3c1c7c1d12862d926cd17fb56feb9a..e0c0950188b890aac3103a0260ae4c446a67ca0c 100644 --- a/results/chronos_bolt_base/config.json +++ b/results/chronos_bolt_base/config.json @@ -3,8 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/amazon/chronos-bolt-base", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb", - "org": "AWS AI Labs", - "testdata_leakage": "Yes", - "replication_code_available": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb" } \ No newline at end of file diff --git a/results/chronos_bolt_small/config.json b/results/chronos_bolt_small/config.json index 6d31556a432f22497eca05a360237a92ea6dc3ac..68813b26c5e322842ccacb197a93d18367617e3c 100644 --- a/results/chronos_bolt_small/config.json +++ b/results/chronos_bolt_small/config.json @@ -3,8 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/amazon/chronos-bolt-small", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb", - "org": "AWS AI Labs", - "testdata_leakage": "Yes", - "replication_code_available": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb" } \ No newline at end of file diff --git a/results/chronos_large/config.json b/results/chronos_large/config.json index 8852f3690bc29f664f8f59bd0e9167443a39962a..9ac2a7a48c60de66f27c94067ff75c2f69cd771a 100644 --- a/results/chronos_large/config.json +++ b/results/chronos_large/config.json @@ -3,8 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/amazon/chronos-t5-large", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb", - "org": "AWS AI Labs", - "testdata_leakage": "Yes", - "replication_code_available": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb" } \ No newline at end of file diff --git a/results/crossformer/config.json b/results/crossformer/config.json index d3ed42e4a2ba98040931d3f2cb7488965de3131c..4841c673f38238dca63f1399a3bb85fa9cc14b21 100644 --- a/results/crossformer/config.json +++ b/results/crossformer/config.json @@ -1,8 +1,5 @@ { "model": "Crossformer", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "Shanghai Jiao Tong University", - "testdata_leakage": "No", - "replication_code_available": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/deepar/config.json b/results/deepar/config.json index a3cee120ec6e6ebd127d23b6d43c0e02b47a9d7c..3037ea9d5bb2bda278dea88d26279c6fd0274c0a 100644 --- a/results/deepar/config.json +++ b/results/deepar/config.json @@ -1,8 +1,5 @@ { "model": "DeepAR", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "Amazon Research", - "testdata_leakage": "No", - "replication_code_available": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/granite-flowstate-r1/all_results.csv b/results/granite-flowstate-r1/all_results.csv deleted file mode 100644 index cb041a676d63303a5beef5dd0138940ef91e724f..0000000000000000000000000000000000000000 --- a/results/granite-flowstate-r1/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[mean],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -loop_seattle/5T/short,granite-flowstate-r1,41.45070986516458,41.45070986516458,3.68334098949767,3.68334098949767,0.5813897556858786,0.09886124407789902,0.07844791512002147,7.606605768755193,6.4382225703345,0.11040130296320078,0.0631611659982082,0.05097894461293396,Transport,1 -loop_seattle/5T/medium,granite-flowstate-r1,67.86012803021599,67.86012803021599,4.981009349212804,4.981009349212804,0.790006680520265,0.16011444740374145,0.11607832008101991,9.180053929724078,8.237725901619694,0.14656607111642367,0.08862239157104679,0.07093507776893808,Transport,1 -loop_seattle/5T/long,granite-flowstate-r1,79.07871830343066,79.07871830343066,5.317850134777046,5.317850134777046,0.8394078446775898,0.16856190792014192,0.1223696944817752,9.882365247554311,8.892621565288307,0.15722337782738696,0.09402068387044483,0.07540215965130731,Transport,1 -loop_seattle/D/short,granite-flowstate-r1,17.877726654273193,17.877726654273193,2.863226093965418,2.863226093965418,0.8657208474408145,0.052983993051960006,0.052864978901495996,13.267261042436884,4.228206079920088,0.07556369093575535,0.051169675165805055,0.04187817938435069,Transport,1 -loop_seattle/H/short,granite-flowstate-r1,60.634158223917595,60.634158223917595,4.262669572461908,4.262669572461908,0.8643577085161969,0.11724167449361284,0.09828421490823457,11.285968967365898,7.786793834686879,0.13783416954716468,0.0754535862959419,0.061230476495910985,Transport,1 -loop_seattle/H/medium,granite-flowstate-r1,72.02407488591653,72.02407488591653,4.750261550446659,4.750261550446659,0.9729870998504898,0.11810839028550375,0.10874966881223508,12.312641247205047,8.486699881927988,0.15000147299718203,0.08396034260693121,0.06849444236187414,Transport,1 -loop_seattle/H/long,granite-flowstate-r1,59.35896462203302,59.35896462203302,4.379443065432039,4.379443065432039,0.9158718561598912,0.13197178015562935,0.10146476527772738,10.522260549778112,7.70447692072817,0.13768929359442317,0.07826649728731244,0.0629778257045242,Transport,1 -m_dense/D/short,granite-flowstate-r1,9380.957893518518,9380.957893518518,47.98970847800926,47.98970847800926,0.7204544873748164,0.13280157339493376,0.10461858396176939,12.421598608601009,96.85534519848927,0.16767092932086655,0.08307728398319504,0.06929722616598537,Transport,1 -m_dense/H/short,granite-flowstate-r1,44050.734954969616,44050.734954969616,91.06528459337022,91.06528459337022,0.8017953233026586,0.29868924246199435,0.21883891820907592,13.303363171566074,209.8826694964823,0.37227959326725935,0.16152713890345904,0.13404622255349455,Transport,1 -m_dense/H/medium,granite-flowstate-r1,33829.452361111114,33829.452361111114,80.4361105685764,80.4361105685764,0.693955926928377,0.27457574761027653,0.19742523829142253,9.060708922181885,183.92784552946603,0.3200112252815566,0.13994867512221887,0.11536569543388168,Transport,1 -m_dense/H/long,granite-flowstate-r1,35609.223827160495,35609.223827160495,86.48302662037037,86.48302662037037,0.7524482334974119,0.305302813893232,0.2079464363757475,9.190739877876966,188.70406415114778,0.32667465676036056,0.14971491559491554,0.12368215185537891,Transport,1 -sz_taxi/15T/short,granite-flowstate-r1,16.91059370122404,16.91059370122404,2.761892222077273,2.761892222077273,0.5522357752225293,1159830451333.647,0.4053544820592226,6.241993918260454,4.112249226545497,0.38451290486395934,0.2582487448418171,0.20408371708276796,Transport,1 -sz_taxi/15T/medium,granite-flowstate-r1,16.478180714142628,16.478180714142628,2.778212418515458,2.778212418515458,0.5430802287976548,9412111153326.55,0.4101003206693209,5.574341311249096,4.059332545399875,0.37778864604456086,0.2585590355745906,0.20685295517771227,Transport,1 -sz_taxi/15T/long,granite-flowstate-r1,15.807911825587606,15.807911825587606,2.741741734692174,2.741741734692174,0.5182382154261588,6083944685688.302,0.40882021042696093,4.5911431673393714,3.9759164761835235,0.368003730082805,0.25377071961504843,0.2048554791615285,Transport,1 -sz_taxi/H/short,granite-flowstate-r1,7.2828467768481655,7.2828467768481655,1.860691213200235,1.860691213200235,0.5677746285130757,1.3856488349091294,0.29898438188764787,5.797239248866384,2.6986750039321454,0.25139823098224423,0.17333486941597898,0.1373117471495384,Transport,1 -bitbrains_fast_storage/5T/short,granite-flowstate-r1,1918819.9666293731,1918819.9666293731,201.15327408481332,201.15327408481332,0.9500239314974467,2.9549999314576767,0.7987144012317283,18.480890669205756,1385.2147727444192,4.349231791755918,0.6315715309129752,0.5084926808592367,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,granite-flowstate-r1,3717408.5179616907,3717408.5179616907,373.47259886729313,373.47259886729313,1.2533617759094229,5.391664400833723,0.8392735475954459,25.492025953636965,1928.0582247332914,5.857973415045989,1.134712908224219,0.8411009141441634,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,granite-flowstate-r1,5366802.427352255,5366802.427352255,460.9371408326316,460.9371408326316,1.055861374072807,5.210343709327751,0.8511663347259798,20.30618758994284,2316.636015292919,6.122177309423143,1.2181192410233277,0.8658625652290532,Web/CloudOps,2 -bitbrains_fast_storage/H/short,granite-flowstate-r1,3126915.8156498084,3126915.8156498084,354.63466586440234,354.63466586440234,1.0791480950475973,4.232477095497576,0.6059894199039743,21.94082534645925,1768.3087444362786,5.040254286777927,1.010823986754041,0.7392010261162123,Web/CloudOps,2 -bitbrains_rnd/5T/short,granite-flowstate-r1,1555017.346583365,1555017.346583365,133.39699393172572,133.39699393172572,1.9823352369510236,1.5000166037202662,0.7469129171016201,59.27290993473621,1247.003346660852,5.102377727247753,0.5458219920111853,0.46325646047415603,Web/CloudOps,2 -bitbrains_rnd/5T/medium,granite-flowstate-r1,2182926.1864392064,2182926.1864392064,171.75717876721612,171.75717876721612,4.587454147018737,1.5104346407332898,0.8079425893529406,166.6164674370469,1477.4729054839572,6.216481684861087,0.722670007745468,0.6856400101909502,Web/CloudOps,2 -bitbrains_rnd/5T/long,granite-flowstate-r1,2318756.184251273,2318756.184251273,199.32612558014335,199.32612558014335,3.5305473032687122,2.7361658186500417,0.7742371981209178,122.28185363909965,1522.7462639098062,5.83329791657803,0.7635734860256999,0.7007215543298201,Web/CloudOps,2 -bitbrains_rnd/H/short,granite-flowstate-r1,1809842.6158095275,1809842.6158095275,172.9287249995723,172.9287249995723,5.951792336726248,2.2392674411692366,0.6207620792694599,218.06738371469257,1345.3039120620765,5.887708240380929,0.756820722849105,0.6796321068215749,Web/CloudOps,2 -bizitobs_application/10S/short,granite-flowstate-r1,564079.97,564079.97,401.3475629340278,401.3475629340278,1.205330814369782,0.017286223305596246,0.01741252316368951,16.64117361013491,751.0525747242998,0.02896509747081063,0.0154783721822987,0.012511931527917401,Web/CloudOps,2 -bizitobs_application/10S/medium,granite-flowstate-r1,4976418.133333334,4976418.133333334,1250.62625,1250.62625,2.6442742523282483,0.038269913991292315,0.03911675771077474,66.51261882590339,2230.7886796676494,0.08759650584716513,0.049108412024515206,0.04376368959632607,Web/CloudOps,2 -bizitobs_application/10S/long,granite-flowstate-r1,7334613.333333333,7334613.333333333,1639.5430555555556,1639.5430555555556,3.4178548412047256,0.04929450988769531,0.050589014689127604,90.30547709531061,2708.249126896071,0.10444634364324562,0.06323062221189224,0.05627945354160002,Web/CloudOps,2 -bizitobs_l2c/5T/short,granite-flowstate-r1,29.628033043089367,29.628033043089367,3.469860739935012,3.469860739935012,0.3444041553361347,0.16937794267706957,0.22942097740513936,2.4474254861738625,5.443163881704222,0.1869402734313385,0.11916905858234553,0.09478604541398604,Web/CloudOps,7 -bizitobs_l2c/5T/medium,granite-flowstate-r1,66.40562287414966,66.40562287414966,5.196483544589711,5.196483544589711,0.5180485897477924,0.3705120584781616,0.6288693038784727,3.875708227374398,8.148964527726799,0.4287553848580038,0.27341146160202573,0.2112244848105934,Web/CloudOps,7 -bizitobs_l2c/5T/long,granite-flowstate-r1,67.40641121031746,67.40641121031746,5.146017136346726,5.146017136346726,0.516907105116478,0.4151420280377074,0.711968005952381,3.9959435697250067,8.210140754598392,0.4609871291882226,0.28894116889642985,0.22559264906869128,Web/CloudOps,7 -bizitobs_l2c/H/short,granite-flowstate-r1,59.07940383184524,59.07940383184524,4.714354863242497,4.714354863242497,0.4689803455559716,0.37752821576966944,0.6080510578458271,4.146648688030747,7.686312759174274,0.41431196434265083,0.2541158140705373,0.19521926403440873,Web/CloudOps,7 -bizitobs_l2c/H/medium,granite-flowstate-r1,73.83500744047619,73.83500744047619,4.907634626116072,4.907634626116072,0.4991792206030149,0.46633405537656436,0.7648399716331845,4.198331474513947,8.592729917812859,0.5203028481308202,0.29716473088026096,0.23055849706914167,Web/CloudOps,7 -bizitobs_l2c/H/long,granite-flowstate-r1,80.70536334325396,80.70536334325396,5.234917922247024,5.234917922247024,0.5635045624417966,0.5975497320674077,0.7721160404265873,7.231993337079029,8.983616384466446,0.5487452170767906,0.31976389559439494,0.2543398283037386,Web/CloudOps,7 -bizitobs_service/10S/short,granite-flowstate-r1,3734.028429387814,3734.028429387814,22.569405986644604,22.569405986644604,0.835773152632602,0.10586309404600235,0.07311184978989697,11.514130953747195,61.106697091135715,0.045269795560122464,0.016720137784652367,0.013660603295762404,Web/CloudOps,2 -bizitobs_service/10S/medium,granite-flowstate-r1,20807.38006448413,20807.38006448413,44.4945500062004,44.4945500062004,1.1988481894407885,0.11600869678315662,0.08680625794425843,18.00375816549739,144.24763451954465,0.10856416388452853,0.0334876452908972,0.026123493939465815,Web/CloudOps,2 -bizitobs_service/10S/long,granite-flowstate-r1,134217.727989418,134217.727989418,83.09535838293651,83.09535838293651,1.4830680232875069,0.12991568267660797,0.10591665762442129,27.317462410337274,366.35737741912334,0.2714092733008839,0.06155970160139071,0.05487095550436146,Web/CloudOps,2 -car_parts/M/short,granite-flowstate-r1,1.5211299250954822,1.5211299250954822,0.5162025805692899,0.5162025805692899,0.9924370321830904,0.7730255192209781,1.8536850222398422,17.87637317262042,1.23334096060071,2.9574251068625497,1.2378008359175041,1.054545676681899,Sales,1 -covid_deaths/D/short,granite-flowstate-r1,349468.1256999922,349468.1256999922,111.75090890372906,111.75090890372906,32.63615367574188,0.08952274402292126,0.3616466770793561,836.2838766375996,591.1582915767926,0.22242617362552547,0.042046821334996425,0.03612361491123092,Healthcare,1 -electricity/15T/short,granite-flowstate-r1,144724.20222686068,144724.20222686068,58.33758063256204,58.33758063256204,1.1053910631471215,0.19516474044690446,0.18062286785593978,17.62328372598712,380.4263427088885,0.7498953240654795,0.1149948729158524,0.09519503098242073,Energy,1 -electricity/15T/medium,granite-flowstate-r1,255531.68970834097,255531.68970834097,55.74208467170784,55.74208467170784,0.8762298879762337,0.13892117221714623,0.13829312089971593,10.266415539743985,505.5014240418527,0.8710298479353842,0.0960492240496222,0.07723075012397557,Energy,1 -electricity/15T/long,granite-flowstate-r1,338479.9556867628,338479.9556867628,61.35500133617504,61.35500133617504,0.9261343937402899,0.14641354581144864,0.1405701778514965,10.405953200713588,581.7903021594317,0.9178455315030511,0.09679503697938885,0.07719727479217263,Energy,1 -electricity/D/short,granite-flowstate-r1,1153071526.724901,1153071526.724901,4236.171493806306,4236.171493806306,1.4361974143535554,0.5123566384423204,0.10186529419014045,15.486124143825196,33956.90690750412,0.5580247672297972,0.06961436794628582,0.054861973992420855,Energy,1 -electricity/H/short,granite-flowstate-r1,1579957.5263886042,1579957.5263886042,172.9016443198436,172.9016443198436,0.9357391751830545,0.2106907078077149,0.12681146515113814,14.449524888648456,1256.9636137886428,0.5933765905528198,0.08162192372319259,0.0673880474837622,Energy,1 -electricity/H/medium,granite-flowstate-r1,5879730.607631044,5879730.607631044,248.33276843646624,248.33276843646624,1.1007946007359248,0.23596200307816834,0.13461733400284706,12.794364474293266,2424.815582189921,0.9464854248246886,0.09693246264081054,0.07729819344574272,Energy,1 -electricity/H/long,granite-flowstate-r1,8650005.133044858,8650005.133044858,280.31987386800864,280.31987386800864,1.217306687320673,0.30779311248211627,0.14736278884475296,14.600614397315248,2941.0891066142244,1.1599647254855574,0.11055808027311882,0.08814531063716587,Energy,1 -electricity/W/short,granite-flowstate-r1,31090345422.7027,31090345422.7027,23474.4601245777,23474.4601245777,1.3523889878950264,0.2253331336012585,0.08986290126770466,13.743660238931474,176324.54571812373,0.40231442986284055,0.05356097191626985,0.043029118455147515,Energy,1 -ett1/15T/short,granite-flowstate-r1,4.753783319109962,4.753783319109962,1.0830024310520716,1.0830024310520716,0.6936313679579977,0.4810389677210178,0.24103622266224453,7.737355839952342,2.180317251940635,0.4121707138332119,0.20473253820861395,0.16085393208643303,Energy,7 -ett1/15T/medium,granite-flowstate-r1,9.024871094718812,9.024871094718812,1.5885238695901538,1.5885238695901538,1.0171980184840024,0.7342526954922842,0.3997857569134425,9.24734334671116,3.004142322646983,0.5725924631855541,0.30277420229419616,0.23524821206151525,Energy,7 -ett1/15T/long,granite-flowstate-r1,8.657997329954117,8.657997329954117,1.5935708472842263,1.5935708472842263,1.038601150239987,0.7779727449578085,0.4055966331845238,9.075673533341416,2.9424475067457223,0.5608333875391712,0.30373617014990417,0.23614085107506394,Energy,7 -ett1/D/short,granite-flowstate-r1,42729.857936507935,42729.857936507935,136.88030443948412,136.88030443948412,1.650824058207011,1.3853217230902777,0.48698438311380055,13.300092362358745,206.71201691364712,0.5429359691313466,0.3595206599763899,0.27793536790099593,Energy,7 -ett1/H/short,granite-flowstate-r1,97.65726245698475,97.65726245698475,4.863809758140928,4.863809758140928,0.8147611850097799,0.4603510743095761,0.2567979029246739,7.5767020413291135,9.882168914615088,0.4612412622581233,0.2270139047016764,0.17589358721837078,Energy,7 -ett1/H/medium,granite-flowstate-r1,133.07732747395832,133.07732747395832,6.712583705357143,6.712583705357143,1.2441574581694812,3601580795892.649,0.4400715192159017,9.779893131019977,11.535914678687526,0.5525430288560677,0.3215168831701089,0.24921015888892628,Energy,7 -ett1/H/long,granite-flowstate-r1,152.69128017526455,152.69128017526455,7.173261047040344,7.173261047040344,1.3534500436894037,3178053082302.1973,0.4754653769195395,10.244868988012207,12.356831316128927,0.5917418740490133,0.3435119268382309,0.2671987839058532,Energy,7 -ett1/W/short,granite-flowstate-r1,1153703.642857143,1153703.642857143,807.1321149553571,807.1321149553571,1.437918932475465,0.600034236907959,0.5177733557564872,8.476227584543862,1074.1059737554497,0.42741240344900644,0.3211771329674151,0.23954378242734561,Energy,7 -ett2/15T/short,granite-flowstate-r1,8.079884579068139,8.079884579068139,1.7689632597423735,1.7689632597423735,0.7606725100303628,0.110214520996241,0.1303801133519127,9.521636790309456,2.8425137781668077,0.13494424628729973,0.08397898213524545,0.06688484847874661,Energy,7 -ett2/15T/medium,granite-flowstate-r1,11.302273879278275,11.302273879278275,2.0768988424634176,2.0768988424634176,0.8578578323142202,0.13124028227417137,0.1556886987837534,8.670731350583093,3.361885464925638,0.17916095447987504,0.11068169420284639,0.08859935043973656,Energy,7 -ett2/15T/long,granite-flowstate-r1,11.622918914310516,11.622918914310516,2.150206337580605,2.150206337580605,0.8841918897640062,0.13563125675853127,0.15981333778018042,9.220829384385736,3.409240225374345,0.18168457377743186,0.11458838220588266,0.09245678467426278,Energy,7 -ett2/D/short,granite-flowstate-r1,116055.11746031746,116055.11746031746,214.21402529761906,214.21402529761906,1.3853478707387081,0.3905816396077474,0.1446110407511393,11.591489052704377,340.6686329269507,0.1949736357924987,0.12260033156319076,0.10014086620359766,Energy,7 -ett2/H/short,granite-flowstate-r1,103.29599296933128,103.29599296933128,6.279765065511068,6.279765065511068,0.7210159931254069,0.11709959820781349,0.10748744692121233,7.587828395192321,10.16346363054108,0.1289009896129199,0.0796448888790804,0.0634817218422438,Energy,7 -ett2/H/medium,granite-flowstate-r1,237.22610212053573,237.22610212053573,9.6669189453125,9.6669189453125,1.0347466818960263,0.1728648029064636,0.1682235036577497,9.427088771711766,15.402146023218185,0.20755380940581772,0.13026794118769514,0.10288442402605545,Energy,7 -ett2/H/long,granite-flowstate-r1,254.48660714285714,254.48660714285714,9.941204737103174,9.941204737103174,1.093888267848659,0.1896096961045579,0.18104914912471065,9.979924869094882,15.952636369668092,0.2184590832099773,0.13613715141150856,0.10948928808631372,Energy,7 -ett2/W/short,granite-flowstate-r1,4486741.428571428,4486741.428571428,1501.7201450892858,1501.7201450892858,1.0041806776522244,0.14903204781668528,0.16555353573390416,6.453682831422532,2118.1929630162185,0.17768391571973718,0.1259712973999872,0.1001393739432319,Energy,7 -hierarchical_sales/D/short,granite-flowstate-r1,29.501266210776016,29.501266210776016,2.314163122220491,2.314163122220491,0.7496909429630705,0.6271814765890572,1.0608916214449733,10.94785936133736,5.431506808499463,1.666326138538244,0.7099596180866424,0.585107546801234,Sales,1 -hierarchical_sales/W/short,granite-flowstate-r1,463.85805802426097,463.85805802426097,8.872117171853276,8.872117171853276,0.7165200568267713,0.5035296427934495,0.460279900138661,8.515515407059791,21.53736423112775,0.9887909265594902,0.4073232390350769,0.3476428653865981,Sales,1 -hospital/M/short,granite-flowstate-r1,2218.0071900528133,2218.0071900528133,16.69859941292721,16.69859941292721,0.7395816302454122,0.18764167907496007,0.17011142160414822,6.891172982290973,47.095723691783455,0.1709684132955381,0.06061979352032028,0.047927039262772995,Healthcare,1 -jena_weather/10T/short,granite-flowstate-r1,731.8024675444951,731.8024675444951,5.569103058557662,5.569103058557662,0.28313389941336975,0.39085432959340793,0.5456442548553385,3.53112237892989,27.05184776580881,0.1680258838319551,0.0345911108056769,0.028579948265808106,Nature,21 -jena_weather/10T/medium,granite-flowstate-r1,1424.3475896309637,1424.3475896309637,9.879849864084484,9.879849864084484,0.6245233803473645,0.8979574166511548,0.6541079604780519,7.442667774706929,37.740529800613075,0.23161298652513693,0.0606324698018257,0.04986644977298401,Nature,21 -jena_weather/10T/long,granite-flowstate-r1,1388.1151193285746,1388.1151193285746,9.899603319924974,9.899603319924974,0.664336558808387,0.8928145231395589,0.6483050265009441,7.4333228167736864,37.2574169706996,0.22798451095367808,0.060577367005969426,0.050057395784607016,Nature,21 -jena_weather/D/short,granite-flowstate-r1,425.2829799107143,425.2829799107143,10.116009230840774,10.116009230840774,1.1081753325628592,0.6378101406968789,0.4615920172797309,11.584663121393367,20.62239025696862,0.12418049508094667,0.06091490942010182,0.04899783681871478,Nature,21 -jena_weather/H/short,granite-flowstate-r1,1119.1113282492884,1119.1113282492884,8.509681834712463,8.509681834712463,0.5290198272121043,1.3226758994139345,0.6105376595293569,6.630339600149357,33.45312135286166,0.2050654990884911,0.05216380660306613,0.042893424293060305,Nature,21 -jena_weather/H/medium,granite-flowstate-r1,1391.652802579365,1391.652802579365,10.950601438492063,10.950601438492063,0.8596039049496134,2.682871867311553,0.6940502348400298,7.809706127473915,37.30486298834731,0.22827506842806194,0.06700867105398674,0.05391206451330069,Nature,21 -jena_weather/H/long,granite-flowstate-r1,1422.4071800595239,1422.4071800595239,12.70726402323082,12.70726402323082,1.044518269791028,2.954945765763497,0.6665930531012318,13.488051825052747,37.71481380120448,0.2270849614677194,0.07651180717173131,0.06572870330627677,Nature,21 -kdd_cup_2018/D/short,granite-flowstate-r1,3032.32181562313,3032.32181562313,21.635173881566782,21.635173881566782,1.2123058616308304,0.5086835775851601,0.47670414527209726,13.047530834988663,55.0665217316577,1.2335485596689217,0.48465086845044814,0.3827459373829025,Nature,1 -kdd_cup_2018/H/short,granite-flowstate-r1,4398.889456482666,4398.889456482666,22.99881595337505,22.99881595337505,0.9621035710690519,0.900165149616715,0.5072937672239056,10.875513545292073,66.32412424210867,1.3882840391902038,0.4814068707758709,0.3849187984595621,Nature,1 -kdd_cup_2018/H/medium,granite-flowstate-r1,5368.844173032959,5368.844173032959,26.429856858879496,26.429856858879496,1.0867170392472882,0.9654808078084267,0.5897087234461115,11.904954974859622,73.27239707443015,1.533723976012515,0.5532247717495267,0.4374087327717977,Nature,1 -kdd_cup_2018/H/long,granite-flowstate-r1,4298.344292679033,4298.344292679033,25.00750598429966,25.00750598429966,1.0536505519582111,0.9858010004321838,0.634165778231531,10.313944573374185,65.56175937754442,1.5383438105776532,0.5867771459181955,0.4548619458932433,Nature,1 -m4_daily/D/short,granite-flowstate-r1,399766.270814983,399766.270814983,181.88649192009964,181.88649192009964,3.394312899049464,0.040879217896014605,0.0322031014377918,36.098650959773316,632.2707258880353,0.09766835355992211,0.028096436341688256,0.023188241902595382,Econ/Fin,1 -m4_hourly/H/short,granite-flowstate-r1,657845.4228326293,657845.4228326293,199.4017492212151,199.4017492212151,0.7743540721458968,0.09569012737216581,0.09158436994813683,11.529113065344992,811.0767058870754,0.11072988396912965,0.027222742798855275,0.021717714386104512,Econ/Fin,1 -m4_monthly/M/short,granite-flowstate-r1,1810024.2804412616,1810024.2804412616,546.8177242657697,546.8177242657697,0.9275168286784129,0.15474473074906403,0.1276774451771268,10.92075979718353,1345.3714284320376,0.27961436910551773,0.11364749522329542,0.09145717317865883,Econ/Fin,1 -m4_quarterly/Q/short,granite-flowstate-r1,1813208.9696484476,1813208.9696484476,560.6868153514862,560.6868153514862,1.163979704198937,0.11897007710703959,0.10137886070956786,13.974069106984153,1346.5544807576289,0.2253895077271329,0.09384909939185122,0.0758432968911375,Econ/Fin,1 -m4_weekly/W/short,granite-flowstate-r1,247913.7640347118,247913.7640347118,258.61758800152666,258.61758800152666,2.132835719352128,0.0643054270146617,0.06372861717098245,19.953150903923188,497.90939339875064,0.09071136978426826,0.047116113833053246,0.03788654032486178,Econ/Fin,1 -m4_yearly/A/short,granite-flowstate-r1,3416111.6546458313,3416111.6546458313,837.9894702568552,837.9894702568552,3.0439671869605203,0.16826883908582263,0.1376340912930601,43.546291680992425,1848.2726137249967,0.2963790796483007,0.13437549531675713,0.10971602196381898,Econ/Fin,1 -restaurant/D/short,granite-flowstate-r1,140.0810135071536,140.0810135071536,7.133117464863086,7.133117464863086,0.6849189931973065,0.6533557068481046,0.39503787828001685,6.633627656124336,11.83558251659603,0.5447009301829102,0.3282825930001077,0.2575497402716079,Sales,1 -saugeen/D/short,granite-flowstate-r1,1068.6747916666666,1068.6747916666666,12.19817362467448,12.19817362467448,2.7099982698220106,0.27274765650431315,0.31110912958780923,38.49821158533616,32.69059179131921,1.058856071113682,0.39510175531344033,0.3271089749626672,Nature,1 -saugeen/M/short,granite-flowstate-r1,411.3457496279762,411.3457496279762,12.68285406203497,12.68285406203497,0.7430905682283409,0.35811719440278555,0.3497826712472098,7.33920900430693,20.28166042581268,0.6089162903943062,0.3807773271481169,0.2939252156333414,Nature,1 -saugeen/W/short,granite-flowstate-r1,938.9776306152344,938.9776306152344,14.244345378875732,14.244345378875732,1.1556695439234392,0.3624628633260727,0.37230038046836855,15.578419024490469,30.642741891273932,0.9249766442040995,0.42997740979532445,0.35385577225602904,Nature,1 -solar/10T/short,granite-flowstate-r1,21.057992838653046,21.057992838653046,2.281677351438796,2.281677351438796,1.0011767062116412,4.745119596668192,1.5327916560671915,10.133787458420612,4.588898869952687,1.334910172653622,0.6637396886413464,0.5046082467519232,Energy,1 -solar/10T/medium,granite-flowstate-r1,19.156864655479293,19.156864655479293,2.073804189525487,2.073804189525487,0.9060669155644001,3.622893415958009,1.46119343049493,8.632422131956845,4.376855567125707,0.9712416424977848,0.4601853902563728,0.3543395366674974,Energy,1 -solar/10T/long,granite-flowstate-r1,19.169525794951973,19.169525794951973,2.0671651464309138,2.0671651464309138,0.9037365413832188,3.2422036952935214,1.4646145538305329,8.773815996371814,4.378301702138852,0.9462998804344512,0.4467846813640622,0.3488510572743121,Energy,1 -solar/D/short,granite-flowstate-r1,121347.74087591241,121347.74087591241,256.78397795962593,256.78397795962593,1.0015340795661285,1.0439979386155622,0.4368907422624945,6.607136502845791,348.35002637564475,0.5032471599899597,0.3709654021950753,0.280654910395402,Energy,1 -solar/H/short,granite-flowstate-r1,713.4490955072872,713.4490955072872,11.475249673572511,11.475249673572511,0.8859552202543093,4.375803574353946,1.4186280522033246,12.32426860953147,26.710467901317028,0.9845567424808795,0.4229815246782049,0.3448414114976219,Energy,1 -solar/H/medium,granite-flowstate-r1,712.8135530717761,712.8135530717761,11.776983330719663,11.776983330719663,0.9016590559568123,4.737373116344047,1.4157643709778862,8.538134026085128,26.69856837120253,0.9617447526390759,0.42423442945556517,0.33839230208400595,Energy,1 -solar/H/long,granite-flowstate-r1,714.8083510746147,714.8083510746147,13.339576066536141,13.339576066536141,1.0169086041890907,5.111570059698401,1.429740653304868,6.824690001781491,26.73590004235157,0.9269825135404268,0.46250747991028823,0.34884154125412525,Energy,1 -solar/W/short,granite-flowstate-r1,890704.2919708029,890704.2919708029,743.0374693487682,743.0374693487682,0.8112240251586813,0.1576858123723608,0.14842241088839342,3.8728657066847503,943.7713133862477,0.19264731378202396,0.1516725190510744,0.11438148962827827,Energy,1 -temperature_rain/D/short,granite-flowstate-r1,181.547905769168,181.547905769168,5.781678077448807,5.781678077448807,1.359014791131396,26.571249788589398,1.4947792542153324,24.059845175681293,13.473971417854797,1.5862426759433996,0.680656372245432,0.5524565883478724,Nature,1 -us_births/D/short,granite-flowstate-r1,231954.86,231954.86,322.3644075520833,322.3644075520833,0.4743718687466878,0.030893956820170085,0.030358022848765057,6.6010378952088224,481.6169224601644,0.04514767838473024,0.0302190473717197,0.02474334418060943,Healthcare,1 -us_births/M/short,granite-flowstate-r1,58419845.333333336,58419845.333333336,6172.6171875,6172.6171875,0.6981300595380177,0.01890546331803004,0.019177773346503574,4.204686400388867,7643.28759980503,0.023739999173717906,0.019172106900002678,0.01431911954281851,Healthcare,1 -us_births/W/short,granite-flowstate-r1,2029664.857142857,2029664.857142857,1096.3985770089287,1096.3985770089287,0.996539615709116,0.014860334140913827,0.014883742800780706,8.46915905257393,1424.6630679367165,0.01933966145272204,0.014883503176163443,0.011571570051207547,Healthcare,1 diff --git a/results/granite-flowstate-r1/config.json b/results/granite-flowstate-r1/config.json deleted file mode 100644 index 1d544257217dad8282a3465819a893576c6a2d69..0000000000000000000000000000000000000000 --- a/results/granite-flowstate-r1/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "granite-flowstate-r1", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-flowstate-r1", - "code_link": "https://github.com/ibm-granite/granite-tsfm/blob/main/notebooks/hfdemo/flowstate_gift_eval.ipynb", - "org": "IBM Research", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/iTransformer/config.json b/results/iTransformer/config.json index 3b04f0f3dc45e29b6fd69ee106e70492eb2d4528..234e955ce7c114a40c89fd6f3d35b54c4c2a68e1 100644 --- a/results/iTransformer/config.json +++ b/results/iTransformer/config.json @@ -1,8 +1,5 @@ { "model": "i_transformer", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "Tsinghua University", - "testdata_leakage": "No", - "replication_code_available": "Yes" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/naive/all_results.csv b/results/naive/all_results.csv index 9c0a8a37448fd3c8119c87f371a267074a8b75ca..135fbcae0566f049b6967b2e6bc0625684c40645 100644 --- a/results/naive/all_results.csv +++ b/results/naive/all_results.csv @@ -1,98 +1,98 @@ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,Naive,4070174.893125562,4070174.893125562,1007.6444123429384,3.965954931783283,0.17520083696448324,0.16353434447586868,52.037766411775884,2017.4674453694568,0.323510256896443,0.1615804524853844,0.13714565740983498,Econ/Fin,1 -ett1/W/short,Naive,1571455.142857143,1571455.142857143,970.0256696428571,1.7689207877683926,0.6150428227015904,0.6251754760742188,12.799615221235317,1253.5769393448265,0.4988281842249676,0.38599636628003803,0.3118251468740881,Energy,7 -bizitobs_l2c/H/short,Naive,263.3054935515873,263.3054935515873,11.038767981150794,1.0860940042784581,1.2889937073778477,1.012692318579292,9.95825254038738,16.2266907763594,0.8746601901190734,0.5950178649574238,0.48632480596516664,Web/CloudOps,7 -bizitobs_l2c/H/medium,Naive,538.6467633928571,538.6467633928571,15.811579241071428,1.4850898274863054,1.1477882042734406,1.7614684262279354,15.525790979764977,23.20876479679298,1.4053259605351764,0.9574151394565906,0.8639523074274909,Web/CloudOps,7 -bizitobs_l2c/H/long,Naive,326.6440476190476,326.6440476190476,13.937010168650794,1.4278952141944932,2.5975948022295894,0.9060760013640873,17.133983554777487,18.07329653436383,1.103969115064981,0.8513128057896183,0.8178194151135174,Web/CloudOps,7 -ett2/D/short,Naive,122878.94603174603,122878.94603174603,211.56051587301587,1.3901143157307532,0.4713712661985367,0.14168287004743305,32.70861808020462,350.5409334610525,0.20062382493032205,0.12108166507005774,0.15331100213050436,Energy,7 -solar/W/short,Naive,2813150.8321167883,2813150.8321167883,1341.8134124087592,1.4703506784640206,0.3086572076282362,0.2534797557079009,11.583228430599936,1677.245012547895,0.3423676402558515,0.2738976644635694,0.20974762813186765,Energy,1 -jena_weather/D/short,Naive,633.5864087301587,633.5864087301587,13.136656746031745,1.5734140297468562,1.7707135350571066,0.4854136849221913,31.00640730511845,25.17114238031637,0.15157141745411598,0.07910414448099358,0.2105908566035542,Nature,21 -saugeen/M/short,Naive,1026.8541666666667,1026.8541666666667,21.004798525855655,1.2302953456028718,0.7310131617954799,0.6058716092790876,19.163894291336852,32.04456532185554,0.9620739837576904,0.6306270655515779,0.7678932982488066,Nature,1 -kdd_cup_2018/H/short,Naive,5594.245732646461,5594.245732646461,31.058507069381246,1.2800082967549335,1.348935586862045,0.6193935963094522,14.046664157776483,74.79469053780797,1.565588323464133,0.6501107988069305,0.5618630829214255,Nature,1 -kdd_cup_2018/H/medium,Naive,5746.392704032155,5746.392704032155,32.48218551304779,1.4458736272627521,1.5174890734649364,0.7064537834436008,35.987336722040006,75.80496490357446,1.5867351909559555,0.6799109649120403,0.9996707248005586,Nature,1 -kdd_cup_2018/H/long,Naive,4181.4676662897455,4181.4676662897455,25.868315472284554,1.180184135046071,1.0975976860084167,0.7152840645066674,43.77688578853556,64.66426885297433,1.5172850161804052,0.6069751990727064,1.2460511734345185,Nature,1 -loop_seattle/H/short,Naive,193.11455617565585,193.11455617565585,7.998571198519228,1.591891706430253,0.30128818640632,0.16068070405310833,28.110449663719383,13.896566344808196,0.24598335775575475,0.14158284513176175,0.1820636511546452,Transport,1 -loop_seattle/H/medium,Naive,216.44939692982456,216.44939692982456,8.814659825690145,1.776697703849278,0.32173781891971426,0.17238567340484715,87.84401294486503,14.712219306747183,0.2600368222627069,0.15579812145323396,0.45764837206742065,Transport,1 -loop_seattle/H/long,Naive,210.4300653594771,210.4300653594771,8.603209924320605,1.7512023647663921,0.32551775113142845,0.17357919100514382,108.10293428245187,14.506207821463097,0.25924535980099905,0.15375088236182935,0.5564661074073203,Transport,1 -ett1/H/short,Naive,345.0893973214286,345.0893973214286,10.155936104910714,1.8250258188592174,0.9246851603190104,0.5119235084170387,15.402812126847282,18.57658196013003,0.8670450800238758,0.47401908767171513,0.4067901701225723,Energy,7 -ett1/H/medium,Naive,471.78404017857144,471.78404017857144,11.986143275669642,2.0328290422336406,9728700156311.344,0.649679928734189,46.87255743203719,21.720590235501692,1.0403647838029844,0.5741078498521575,0.8414627329278955,Energy,7 -ett1/H/long,Naive,443.52450396825395,443.52450396825395,12.352705439814814,2.2728552413599847,5652510468487.882,0.749044170076885,57.27242142453815,21.06002146172349,1.008518814962206,0.5915443094102009,0.9868526334206741,Energy,7 -ett1/D/short,Naive,57976.62857142857,57976.62857142857,154.97894345238095,1.7783697877928102,1.1623038155691965,0.5164825923859127,23.47582377746543,240.7833643992636,0.6324254934604606,0.4070573356817528,0.40843357791503654,Energy,7 -covid_deaths/D/short,Naive,4435194.482205514,4435194.482205514,353.7093984962406,46.91239825526407,0.132724402667158,0.17781104711702989,1472.7784117419058,2105.990142950701,0.792389040211866,0.13308488253209907,0.12672327831223676,Healthcare,1 -m4_quarterly/Q/short,Naive,2269820.328,2269820.328,655.560453125,1.4770341883914722,0.13182427295049032,0.11610267241795857,13.675610580733625,1506.5922899046045,0.2521770186622125,0.10972927561734654,0.0943391723614522,Econ/Fin,1 -ett2/15T/short,Naive,19.819986979166668,19.819986979166668,3.024351283482143,1.2443102434383637,0.16288884327408776,0.1946806408110119,13.571980290549968,4.451964395541216,0.2113505875232857,0.14357671442317277,0.12107340753866416,Energy,7 -ett2/15T/medium,Naive,18.141490575396826,18.141490575396826,2.8998893229166667,1.216851599992199,0.18442374644793325,0.21048138330853175,34.48033209514033,4.259282871023809,0.226984894057983,0.15454035119858,0.2420306513801469,Energy,7 -ett2/15T/long,Naive,16.585460069444444,16.585460069444444,2.8831569320436508,1.300946032338565,0.2003668146589512,0.24919358803902203,42.00999676184563,4.07252502379598,0.2170322303229923,0.153648651801133,0.28312587034305786,Energy,7 -bizitobs_l2c/5T/short,Naive,21.968577938988094,21.968577938988094,2.758282761346726,0.2830657721671069,0.16774414213623878,0.15846396092072984,3.0909789998825157,4.687064959971015,0.16097278253260314,0.09473059471069084,0.08008460690704661,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Naive,157.30076530612246,157.30076530612246,8.568459024234693,0.8255719817212336,0.6241842431613175,0.8702898646468468,6.3292605433256774,12.541960185956677,0.6598915432932841,0.4508269493215748,0.35883413736823033,Web/CloudOps,7 -bizitobs_l2c/5T/long,Naive,352.5859523809524,352.5859523809524,13.23037326388889,1.2484065615509454,0.986276036425049,1.2111292430391314,9.782056549262665,18.77727222950534,1.054315763614314,0.7428656793238004,0.5919795411117497,Web/CloudOps,7 -solar/10T/short,Naive,36.5841692328163,36.5841692328163,3.299997772439553,1.4492625864683233,2.6560337692441602,1.410252805214504,17.739652658708565,6.04848487084297,1.7595035939179897,0.9599689946351024,0.9431423497567353,Energy,1 -solar/10T/medium,Naive,80.77481613581067,80.77481613581067,5.939547217153285,2.564335860697831,2.8615413042870976,1.6386301084254773,33.25251391362155,8.987481078467463,1.994357743628945,1.3180091154305162,1.373706540018747,Energy,1 -solar/10T/long,Naive,94.56340575577859,94.56340575577859,4.626759277739761,2.017857283966539,1.0,2.0,40.636930571439166,9.724371740929005,2.10176738342858,1.0,1.5490163371145615,Energy,1 -temperature_rain/D/short,Naive,310.64911445557556,310.64911445557556,7.890313322375261,2.0115357653222166,97.02872693676676,1.3339967533230714,47.47823470729249,17.625240833973745,2.0749568489172328,0.9288984940737532,1.2679455200160903,Nature,1 -m4_hourly/H/short,Naive,57543048.45088567,57543048.45088567,1218.0648148148148,11.607687602745548,0.37716947546327745,0.43002987675812704,75.79187144378025,7585.713443762931,1.0356173223815475,0.16629273322245952,0.13648849999564483,Econ/Fin,1 -m4_daily/D/short,Naive,497624.29605596676,497624.29605596676,180.83018393659805,3.278424296610481,0.04217648847004251,0.03045251709590828,32.19152086932198,705.4249046184624,0.10896865245350737,0.02793326594720746,0.024356147648337902,Econ/Fin,1 -ett2/H/short,Naive,171.92237723214285,171.92237723214285,8.963274274553571,1.0872035620086857,0.18046948507834626,0.15999321710495723,14.11974224626444,13.111917374363783,0.1662955949497229,0.11367925724568231,0.1127608016960007,Energy,7 -ett2/H/medium,Naive,411.36316964285714,411.36316964285714,13.140455264136905,1.4275550147417344,0.2418973113685871,0.21611560639880953,43.67294503723677,20.282089873651017,0.27331420397137396,0.17707608499481128,0.29476933710790065,Energy,7 -ett2/H/long,Naive,294.4490740740741,294.4490740740741,11.495035548941798,1.3228699351602262,0.21793927625643253,0.22216192982184194,53.48074459500154,17.159518468595618,0.23498639848662972,0.1574156646100037,0.3493770970075422,Energy,7 -car_parts/M/short,Naive,2.9952172180151453,2.9952172180151453,0.6895841636774279,1.2125433213650758,0.8496029665824577,1.6621891716563262,25.79123698119671,1.7306695866095139,4.149968135842565,1.6535520866517999,2.4999765507853837,Sales,1 -loop_seattle/5T/short,Naive,101.24308777170408,101.24308777170408,5.645426243328335,0.8927905562283007,0.13816935201670486,0.12006684510946519,14.599096828070424,10.061962421501288,0.17254044173284333,0.09680659666474113,0.11567081973538601,Transport,1 -loop_seattle/5T/medium,Naive,392.4867645446336,392.4867645446336,12.876183142253613,1.9871339455955137,0.36263277508526026,0.29879087623304873,44.143153465916726,19.81127872058322,0.3524833622181743,0.22909375969775447,0.3268672871228811,Transport,1 -loop_seattle/5T/long,Naive,213.5793191721133,213.5793191721133,9.076179516397202,1.408835030757464,0.3416879652137262,0.18775294267697942,54.0115563888563,14.614353190343843,0.2583847545162609,0.16046871084512437,0.3605186433614601,Transport,1 -loop_seattle/D/short,Naive,58.65786974329205,58.65786974329205,5.940060831075852,1.7323957056549062,0.11095823557630288,0.10706747427075271,21.844122635807963,7.658842585096788,0.13687375626545378,0.10615682844516516,0.1032547596878987,Transport,1 -sz_taxi/15T/short,Naive,33.12582751449939,33.12582751449939,3.9447246546855923,0.7855968930284677,1313272123878.9246,0.5283226274017595,22.646180528246813,5.755504106027498,0.5381642491850911,0.36884862610167457,0.6611611222089492,Transport,1 -sz_taxi/15T/medium,Naive,30.582425213675215,30.582425213675215,3.9741160857371796,0.7792049328037046,9997834048909.348,0.5282687717013889,70.06940102207474,5.530137901867838,0.5146716380849939,0.3698578357504329,1.9355403237762074,Transport,1 -sz_taxi/15T/long,Naive,28.46968482905983,28.46968482905983,3.853752114494302,0.742344566525504,9279995729984.182,0.5353563966234864,85.04158178037052,5.335699094688514,0.49386280363180085,0.35669643096266895,2.4019918478793696,Transport,1 -m_dense/H/short,Naive,358130.1511111111,358130.1511111111,343.1718142361111,2.7139864906441935,1.04015513186624,0.7294788180763577,28.457280672803915,598.4397639788913,1.0614830196673901,0.608701285519745,0.5387397922383297,Transport,1 -m_dense/H/medium,Naive,360975.25333333336,360975.25333333336,348.6905208333333,2.7352300662596214,1.184018235880775,0.7301190185546875,88.57658717624187,600.812161439275,1.0453372563528116,0.6066774538833799,1.2024808459166658,Transport,1 -m_dense/H/long,Naive,395605.0488888889,395605.0488888889,362.1829012345679,2.855573013916035,1.0900783741369702,0.7696336082175926,108.34227578348323,628.9714213610098,1.088842509259061,0.6269921424055026,1.4425473283951151,Transport,1 -electricity/15T/short,Naive,603056.6183220721,603056.6183220721,127.26604527378942,2.456280689463761,0.3533350538350351,0.2991550499122379,20.028534644922413,776.567201420503,1.5307670327684602,0.2508664622193399,0.22064410727117006,Energy,1 -electricity/15T/medium,Naive,2925108.410720721,2925108.410720721,223.189305884009,3.4575185664171464,0.4035447466155227,0.41189880728913403,54.65956633387852,1710.294831519034,2.9470102785062213,0.38457765665388643,0.4298202929628236,Energy,1 -electricity/15T/long,Naive,6142069.7442042045,6142069.7442042045,249.06663475975975,3.412327584432877,0.5159208992945213,0.384422579940197,66.52365216918754,2478.3199438741167,3.9098536420593453,0.3929331608043482,0.4779696439404405,Energy,1 -saugeen/D/short,Naive,994.7777083333333,994.7777083333333,15.3725,3.413049305768806,0.47135218302408854,0.4242374928792318,58.71112805300772,31.540096834558597,1.0215912513857213,0.49791893773514423,0.5850141736335629,Nature,1 -bitbrains_rnd/H/short,Naive,4185060.895273006,4185060.895273006,250.85095744000682,5.871568507205478,3.2411984858834244,0.3079789245627823,242.30099047281863,2045.742138020578,8.953168487020333,1.0978465200232173,1.4590277821900182,Web/CloudOps,2 -bitbrains_rnd/5T/short,Naive,1851989.8842618705,1851989.8842618705,130.16222649958368,2.0564165206768776,1.6385894817054072,0.3454797128234705,132.3593826217692,1360.878350280388,5.568321339995986,0.53258625455368,0.8683346368401388,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Naive,2446023.529571928,2446023.529571928,157.91522160865162,4.774544594207593,0.575097733397328,0.3963432651932816,389.568567896354,1563.9768315329763,6.580448103895253,0.6644298685628833,2.0082074202220164,Web/CloudOps,2 -bitbrains_rnd/5T/long,Naive,4798684.04501758,4798684.04501758,287.5256742664699,3.9868856753563806,5.680248975320888,0.42959668137427853,405.57989889313706,2190.589885171932,8.391656316532812,1.101446079412625,2.3614946872229625,Web/CloudOps,2 -m_dense/D/short,Naive,39929.671111111114,39929.671111111114,118.38299768518519,1.6693368812171987,0.2532523956185716,0.2557375985604745,23.291872240434824,199.82410042612756,0.3459250756939904,0.20493848013227253,0.22686565739428607,Transport,1 -saugeen/W/short,Naive,1800.4892578125,1800.4892578125,24.552409362792968,1.990658004828632,0.824161148071289,0.557866096496582,23.291068065474015,42.43217243805106,1.2808504084686672,0.7411348925661494,0.7340352210434381,Nature,1 -jena_weather/H/short,Naive,2411.4258563074354,2411.4258563074354,12.323681110327904,0.6477717456282629,1.0957840912703458,0.4789195589483279,38.34991731121663,49.10627104869026,0.3010182802235531,0.0755433718470736,0.3077983594115,Nature,21 -jena_weather/H/medium,Naive,2461.295238095238,2461.295238095238,13.411072358630953,0.8727345201332312,2.037994607709956,0.5234983085857982,118.95415019975488,49.611442612518715,0.3035812160888766,0.08206473025764742,0.9274165841787126,Nature,21 -jena_weather/H/long,Naive,3726.665343915344,3726.665343915344,17.702639302248677,1.4657949793485425,2.372808282131346,0.5753933143958292,144.34046179305136,61.046419583095485,0.36756708078564854,0.1065895018080746,1.1409491476417122,Nature,21 -bitbrains_fast_storage/H/short,Naive,3129276.3703227686,3129276.3703227686,280.24068195254864,1.4078564206723017,4.944441147552041,0.34317742724255007,60.99672161140896,1768.9760796355524,5.042156288732694,0.7987769496334254,1.2058405212636514,Web/CloudOps,2 -kdd_cup_2018/D/short,Naive,3183.23772462431,3183.23772462431,23.884613481510574,1.4970343738270926,0.5893064466306873,0.5721879846611051,24.580612052672397,56.42018898075679,1.263872161836989,0.5350407118596068,0.674506129145155,Nature,1 -sz_taxi/H/short,Naive,14.13685542701656,14.13685542701656,2.687046148838141,0.8346056944917577,0.9688676361344818,0.39366466163570046,25.403498760775808,3.7599009863314965,0.3502579788038407,0.250314930224525,0.4858205976799266,Transport,1 -ett1/15T/short,Naive,18.248468889508928,18.248468889508928,2.6184166317894344,1.9694015637556788,0.9318440512861078,0.5606311979747954,12.695925468598913,4.271822665971626,0.8075522991563836,0.4949897354107341,0.3848114505420264,Energy,7 -ett1/15T/medium,Naive,25.974303075396826,25.974303075396826,2.7894202628968254,1.8231059347573026,1.1089171131632836,0.6168787629867231,32.362039726312574,5.096499099911314,0.9713977675690157,0.5316662601268594,0.6174724527459637,Energy,7 -ett1/15T/long,Naive,28.417566964285715,28.417566964285715,3.0355567956349208,2.0446378289055525,1.1703466395952207,0.6797395967709925,39.70402516501359,5.330812974048678,1.0160582236557134,0.5785801266324228,0.7155861341837335,Energy,7 -m4_monthly/M/short,Naive,2501230.9708518516,2501230.9708518516,671.266107421875,1.205128987689887,0.1901403370256777,0.15256483493027864,12.668739500059264,1581.528049340843,0.3286958215348495,0.13951214127343117,0.1293608518757588,Econ/Fin,1 -electricity/W/short,Naive,317380739662.4144,317380739662.4144,56604.72342342342,2.0896995064573947,0.285636731125347,0.1196364761812733,15.938528356933686,563365.5471027798,1.2854142672489681,0.12915329923232946,0.0993032024444414,Energy,1 -bitbrains_fast_storage/5T/short,Naive,2299344.812804004,2299344.812804004,173.28187846869332,1.0948146284753992,2.4327068429200516,0.37465512436513404,70.19326241539363,1516.3590646031053,4.760992442912181,0.5440622430011399,0.8929464680662709,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Naive,3704568.4175853366,3704568.4175853366,280.36127147074325,1.271487396616441,4.117625747562423,0.4499728768046956,196.7313435217375,1924.7255434438794,5.847847822156081,0.8518149802561188,2.0491239073790615,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Naive,3499892.2950397297,3499892.2950397297,289.35216809290046,1.1903598806207039,1.795788841111804,0.5099463825492213,231.56717491055915,1870.7999078040734,4.9439656728737384,0.7646714009634807,2.122607238439539,Web/CloudOps,2 -hospital/M/short,Naive,5851.3624511082135,5851.3624511082135,24.06573229030856,0.9676000297092351,0.24346126737101395,0.21603344482527978,11.846825178636562,76.49419880689132,0.27769170470586313,0.08736419661785731,0.08634250116644743,Healthcare,1 -us_births/W/short,Naive,5288246.857142857,5288246.857142857,1720.3214285714287,1.5634204337929731,0.023174324205943515,0.023229679891041348,13.279190377688378,2299.6188504060533,0.031217100406456557,0.023353194707727513,0.01929679613291714,Healthcare,1 -us_births/M/short,Naive,261017514.66666666,261017514.66666666,13253.958333333334,1.49946000119159,0.040852939089139305,0.04129129648208618,18.13215937344219,16156.036477634812,0.05018053914906859,0.041166704218987014,0.039028587852434786,Healthcare,1 -jena_weather/10T/short,Naive,1652.7815476190476,1652.7815476190476,10.021488250248016,0.3638272449122593,0.4180473438261545,0.4286353601518544,51.18596442950847,40.65441609000242,0.2525148769682579,0.06224600217963847,0.28772334069999134,Nature,21 -jena_weather/10T/medium,Naive,3627.5171717171715,3627.5171717171715,15.930331213924964,0.7603204747977048,0.9488791837688123,0.5798423298817329,161.65362364190366,60.22887323964455,0.36962356543938746,0.09776417032562872,0.8818873317406091,Nature,21 -jena_weather/10T/long,Naive,2737.9084656084656,2737.9084656084656,13.276913855820105,0.7531930861787588,0.8439116329840955,0.4988705728488919,197.68405969526736,52.325027143886565,0.3201858133426328,0.08124371249566584,1.070325463540521,Nature,21 -electricity/D/short,Naive,4108036968.189694,4108036968.189694,6964.321117117117,1.987041399842847,0.6052061423957089,0.13582226488305205,24.595303547998157,64093.9698270414,1.0532768308357283,0.11444692995228782,0.10405987098536745,Energy,1 -bizitobs_application/10S/short,Naive,5848946.915555555,5848946.915555555,1048.0411805555555,3.763641918540148,0.05442916870117188,0.05418765597873264,47.88283566865831,2418.4596162755242,0.09327032054038906,0.04041875919370658,0.03821885679668207,Web/CloudOps,2 -bizitobs_application/10S/medium,Naive,21648810.666666668,21648810.666666668,1831.66875,6.51945902607138,0.08660387674967449,0.08887063344319661,105.36258094385984,4652.828243839082,0.18270287104380797,0.07192424088925747,0.08853450662390008,Web/CloudOps,2 -bizitobs_application/10S/long,Naive,20331799.893333334,20331799.893333334,2235.4125,7.207407834346513,0.09553615146213108,0.09904331631130642,110.06515727907836,4509.079716897156,0.17389718137400212,0.08621092492587615,0.09735421334116763,Web/CloudOps,2 -us_births/D/short,Naive,2855120.2133333334,2855120.2133333334,1266.88,1.8648391912491094,0.12472813924153646,0.1233361307779948,22.730582398368462,1689.7100974230264,0.15839661042681866,0.11875971986174946,0.11952686561898423,Healthcare,1 -restaurant/D/short,Naive,269.8283833006444,269.8283833006444,10.66129634818343,1.0060757450046092,0.9434227416570351,0.5561225187815214,20.74904381007386,16.426453765211907,0.7559834619860213,0.49065755991884735,0.6770164746085788,Sales,1 -hierarchical_sales/D/short,Naive,58.96940884663084,58.96940884663084,3.4495245969408845,1.1348033996861053,1.219942090035255,1.1912773542022284,28.8672586949405,7.679154175209066,2.3558794594517027,1.058275945161004,1.7364572065613715,Sales,1 -solar/D/short,Naive,181869.82773722627,181869.82773722627,295.1889902676399,1.155861190610263,1.1535581565541362,0.49778203291324513,22.3811319807538,426.46198861941525,0.6160923187357285,0.4264475482750424,0.5590651164917229,Energy,1 -solar/H/short,Naive,3225.9262389550518,3225.9262389550518,27.129435080666138,2.0928584989912538,1.0,2.0,33.34841823228013,56.79723795181463,2.093565080988042,1.0,1.329536973647586,Energy,1 -solar/H/medium,Naive,3336.7024330900244,3336.7024330900244,27.760555428832117,2.120411643361188,1.0,2.0,103.26192446836347,57.76419680987544,2.080801191386882,1.0,2.9763472520215295,Energy,1 -solar/H/long,Naive,3580.4882400648826,3580.4882400648826,28.841864355231145,2.203647240050593,1.0,2.0,126.30882847556744,59.83718108387863,2.074664118341773,1.0,3.4587368807623573,Energy,1 -hierarchical_sales/W/short,Naive,1045.4721927966102,1045.4721927966102,13.569120762711865,1.0250133777288337,0.9503837463030324,0.5798107884238987,15.297190661198302,32.33376242871544,1.4844588487218926,0.6229649713667368,0.8322267526416594,Sales,1 -ett2/W/short,Naive,4688738.857142857,4688738.857142857,1323.111607142857,0.7785209533197451,0.12217171703066144,0.1269862311226981,16.565488704288857,2165.3495923621335,0.1816396339666693,0.11098877930205708,0.13367758870767996,Energy,7 -bizitobs_service/10S/short,Naive,24411.641058201058,24411.641058201058,51.05920717592593,2.18868692523134,0.29019553694144756,0.09446033861271289,26.76123260084578,156.2422511941026,0.11574925802661973,0.03782629410977125,0.036596222506683165,Web/CloudOps,2 -bizitobs_service/10S/medium,Naive,120879.21777777778,120879.21777777778,96.6510615079365,3.7066999349383667,0.1335696265811012,0.13655030265687004,61.109367709322896,347.67688703417974,0.2616698182259184,0.0727418664837433,0.09526178896511943,Web/CloudOps,2 -bizitobs_service/10S/long,Naive,209141.57037037038,209141.57037037038,126.71164021164022,3.859549432045509,0.15100740947420635,0.15907528831845238,66.98322651223562,457.31998684768894,0.33879726640926433,0.09387203393810047,0.11879424998333914,Web/CloudOps,2 -m4_weekly/W/short,Naive,453525.1459181487,453525.1459181487,347.99148275123207,2.777295047362158,0.08937289522218837,0.09161286714732429,26.631225199626535,673.442756229621,0.1226908336142798,0.06339865521526268,0.06087039452311706,Econ/Fin,1 -electricity/H/short,Naive,14602393.667747747,14602393.667747747,705.7568060247747,3.922376436778821,0.5214915893589874,0.39891619637702774,34.02521326369052,3821.3078478117604,1.8039302069745988,0.3331676148246089,0.2943305500709037,Energy,1 -electricity/H/medium,Naive,70374987.66846848,70374987.66846848,969.0920326576577,4.089287698069245,0.5908175828889569,0.3929812789135593,100.07077757479067,8388.98013279734,3.2744952546433783,0.378268539443073,0.5591833066588809,Energy,1 -electricity/H/long,Naive,62374735.49069069,62374735.49069069,944.6613333333333,4.015252993124353,0.5441184704539119,0.3889218944666957,122.42396003880515,7897.767753656136,3.1148773103603298,0.3725741557559165,0.6680188049702744,Energy,1 +bitbrains_fast_storage/5T/short,Naive,2300000.0,2300000.0,173.0,1.09,2.43,0.375,70.2,1520.0,4.76,0.544,0.893,Web/CloudOps,2.0 +bitbrains_fast_storage/H/short,Naive,3130000.0,3130000.0,280.0,1.41,4.94,0.343,61.0,1770.0,5.04,0.799,1.21,Web/CloudOps,2.0 +bitbrains_rnd/5T/short,Naive,1850000.0,1850000.0,130.0,2.06,1.64,0.345,132.0,1360.0,5.57,0.533,0.868,Web/CloudOps,2.0 +bitbrains_rnd/H/short,Naive,4190000.0,4190000.0,251.0,5.87,3.24,0.308,242.0,2050.0,8.95,1.1,1.46,Web/CloudOps,2.0 +bizitobs_application/10S/short,Naive,5850000.0,5850000.0,1050.0,3.76,0.0544,0.0542,47.9,2420.0,0.0933,0.0404,0.0382,Web/CloudOps,2.0 +bizitobs_l2c/5T/short,Naive,22.0,22.0,2.76,0.283,0.168,0.158,3.09,4.69,0.161,0.0947,0.0801,Web/CloudOps,7.0 +bizitobs_l2c/H/short,Naive,263.0,263.0,11.0,1.09,1.29,1.01,9.96,16.2,0.875,0.595,0.486,Web/CloudOps,7.0 +bizitobs_service/10S/short,Naive,24400.0,24400.0,51.1,2.19,0.29,0.0945,26.8,156.0,0.116,0.0378,0.0366,Web/CloudOps,2.0 +car_parts/M/short,Naive,3.0,3.0,0.69,1.21,0.85,1.66,25.8,1.73,4.15,1.65,2.5,Sales,1.0 +covid_deaths/D/short,Naive,4440000.0,4440000.0,354.0,46.9,0.133,0.178,1470.0,2110.0,0.792,0.133,0.127,Healthcare,1.0 +electricity/15T/short,Naive,603000.0,603000.0,127.0,2.46,0.353,0.299,20.0,777.0,1.53,0.251,0.221,Energy,1.0 +electricity/D/short,Naive,4110000000.0,4110000000.0,6960.0,1.99,0.605,0.136,24.6,64100.0,1.05,0.114,0.104,Energy,1.0 +electricity/H/short,Naive,14600000.0,14600000.0,706.0,3.92,0.521,0.399,34.0,3820.0,1.8,0.333,0.294,Energy,1.0 +electricity/W/short,Naive,317000000000.0,317000000000.0,56600.0,2.09,0.286,0.12,15.9,563000.0,1.29,0.129,0.0993,Energy,1.0 +ett1/15T/short,Naive,18.2,18.2,2.62,1.97,0.932,0.561,12.7,4.27,0.808,0.495,0.385,Energy,7.0 +ett1/D/short,Naive,58000.0,58000.0,155.0,1.78,1.16,0.516,23.5,241.0,0.632,0.407,0.408,Energy,7.0 +ett1/H/short,Naive,345.0,345.0,10.2,1.83,0.925,0.512,15.4,18.6,0.867,0.474,0.407,Energy,7.0 +ett1/W/short,Naive,1570000.0,1570000.0,970.0,1.77,0.615,0.625,12.8,1250.0,0.499,0.386,0.312,Energy,7.0 +ett2/15T/short,Naive,19.8,19.8,3.02,1.24,0.163,0.195,13.6,4.45,0.211,0.144,0.121,Energy,7.0 +ett2/D/short,Naive,123000.0,123000.0,212.0,1.39,0.471,0.142,32.7,351.0,0.201,0.121,0.153,Energy,7.0 +ett2/H/short,Naive,172.0,172.0,8.96,1.09,0.18,0.16,14.1,13.1,0.166,0.114,0.113,Energy,7.0 +ett2/W/short,Naive,4690000.0,4690000.0,1320.0,0.779,0.122,0.127,16.6,2170.0,0.182,0.111,0.134,Energy,7.0 +hierarchical_sales/D/short,Naive,59.0,59.0,3.45,1.13,1.22,1.19,28.9,7.68,2.36,1.06,1.74,Sales,1.0 +hierarchical_sales/W/short,Naive,1050.0,1050.0,13.6,1.03,0.95,0.58,15.3,32.3,1.48,0.623,0.832,Sales,1.0 +hospital/M/short,Naive,5850.0,5850.0,24.1,0.968,0.243,0.216,11.8,76.5,0.278,0.0874,0.0863,Healthcare,1.0 +jena_weather/10T/short,Naive,1650.0,1650.0,10.0,0.364,0.418,0.429,51.2,40.7,0.253,0.0622,0.288,Nature,21.0 +jena_weather/D/short,Naive,634.0,634.0,13.1,1.57,1.77,0.485,31.0,25.2,0.152,0.0791,0.211,Nature,21.0 +jena_weather/H/short,Naive,2410.0,2410.0,12.3,0.648,1.1,0.479,38.3,49.1,0.301,0.0755,0.308,Nature,21.0 +kdd_cup_2018/D/short,Naive,3180.0,3180.0,23.9,1.5,0.589,0.572,24.6,56.4,1.26,0.535,0.675,Nature,1.0 +kdd_cup_2018/H/short,Naive,5590.0,5590.0,31.1,1.28,1.35,0.619,14.0,74.8,1.57,0.65,0.562,Nature,1.0 +loop_seattle/5T/short,Naive,101.0,101.0,5.65,0.893,0.138,0.12,14.6,10.1,0.173,0.0968,0.116,Transport,1.0 +loop_seattle/D/short,Naive,58.7,58.7,5.94,1.73,0.111,0.107,21.8,7.66,0.137,0.106,0.103,Transport,1.0 +loop_seattle/H/short,Naive,193.0,193.0,8.0,1.59,0.301,0.161,28.1,13.9,0.246,0.142,0.182,Transport,1.0 +m_dense/D/short,Naive,39900.0,39900.0,118.0,1.67,0.253,0.256,23.3,200.0,0.346,0.205,0.227,Transport,1.0 +m_dense/H/short,Naive,358000.0,358000.0,343.0,2.71,1.04,0.729,28.5,598.0,1.06,0.609,0.539,Transport,1.0 +m4_daily/D/short,Naive,498000.0,498000.0,181.0,3.28,0.0422,0.0305,32.2,705.0,0.109,0.0279,0.0244,Econ/Fin,1.0 +m4_hourly/H/short,Naive,57500000.0,57500000.0,1220.0,11.6,0.377,0.43,75.8,7590.0,1.04,0.166,0.136,Econ/Fin,1.0 +m4_monthly/M/short,Naive,2500000.0,2500000.0,671.0,1.21,0.19,0.153,12.7,1580.0,0.329,0.14,0.129,Econ/Fin,1.0 +m4_quarterly/Q/short,Naive,2270000.0,2270000.0,656.0,1.48,0.132,0.116,13.7,1510.0,0.252,0.11,0.0943,Econ/Fin,1.0 +m4_weekly/W/short,Naive,454000.0,454000.0,348.0,2.78,0.0894,0.0916,26.6,673.0,0.123,0.0634,0.0609,Econ/Fin,1.0 +m4_yearly/A/short,Naive,4070000.0,4070000.0,1010.0,3.97,0.175,0.164,52.0,2020.0,0.324,0.162,0.137,Econ/Fin,1.0 +restaurant/D/short,Naive,274.0,274.0,10.7,1.01,0.944,0.556,20.7,16.6,0.757,0.49,0.674,Sales,1.0 +saugeen/D/short,Naive,995.0,995.0,15.4,3.41,0.471,0.424,58.7,31.5,1.02,0.498,0.585,Nature,1.0 +saugeen/M/short,Naive,1030.0,1030.0,21.0,1.23,0.731,0.606,19.2,32.0,0.962,0.631,0.768,Nature,1.0 +saugeen/W/short,Naive,1800.0,1800.0,24.6,1.99,0.824,0.558,23.3,42.4,1.28,0.741,0.734,Nature,1.0 +solar/10T/short,Naive,36.6,36.6,3.3,1.45,2.66,1.41,17.7,6.05,1.76,0.96,0.943,Energy,1.0 +solar/D/short,Naive,182000.0,182000.0,295.0,1.16,1.15,0.498,22.4,426.0,0.616,0.426,0.559,Energy,1.0 +solar/H/short,Naive,3230.0,3230.0,27.1,2.09,1.0,2.0,33.3,56.8,2.09,1.0,1.33,Energy,1.0 +solar/W/short,Naive,2810000.0,2810000.0,1340.0,1.47,0.309,0.253,11.6,1680.0,0.342,0.274,0.21,Energy,1.0 +sz_taxi/15T/short,Naive,33.1,33.1,3.94,0.786,1310000000000.0,0.528,22.6,5.76,0.538,0.369,0.661,Transport,1.0 +sz_taxi/H/short,Naive,14.1,14.1,2.69,0.835,0.969,0.394,25.4,3.76,0.35,0.25,0.486,Transport,1.0 +temperature_rain/D/short,Naive,311.0,311.0,7.89,2.01,97.0,1.33,47.5,17.6,2.07,0.929,1.27,Nature,1.0 +us_births/D/short,Naive,2860000.0,2860000.0,1270.0,1.86,0.125,0.123,22.7,1690.0,0.158,0.119,0.12,Healthcare,1.0 +us_births/M/short,Naive,261000000.0,261000000.0,13300.0,1.5,0.0409,0.0413,18.1,16200.0,0.0502,0.0412,0.039,Healthcare,1.0 +us_births/W/short,Naive,5290000.0,5290000.0,1720.0,1.56,0.0232,0.0232,13.3,2300.0,0.0312,0.0234,0.0193,Healthcare,1.0 +bitbrains_fast_storage/5T/medium,Naive,3700000.0,3700000.0,280.0,1.27,4.12,0.45,197.0,1920.0,5.85,0.852,2.05,Web/CloudOps,2.0 +bitbrains_rnd/5T/medium,Naive,2450000.0,2450000.0,158.0,4.77,0.575,0.396,390.0,1560.0,6.58,0.664,2.01,Web/CloudOps,2.0 +bizitobs_application/10S/medium,Naive,21600000.0,21600000.0,1830.0,6.52,0.0866,0.0889,105.0,4650.0,0.183,0.0719,0.0885,Web/CloudOps,2.0 +bizitobs_l2c/5T/medium,Naive,157.0,157.0,8.57,0.826,0.624,0.87,6.33,12.5,0.66,0.451,0.359,Web/CloudOps,7.0 +bizitobs_l2c/H/medium,Naive,559.0,559.0,16.5,1.58,1.0,2.0,17.9,23.6,1.43,1.0,0.877,Web/CloudOps,7.0 +bizitobs_service/10S/medium,Naive,121000.0,121000.0,96.7,3.71,0.134,0.137,61.1,348.0,0.262,0.0727,0.0953,Web/CloudOps,2.0 +electricity/15T/medium,Naive,2930000.0,2930000.0,223.0,3.46,0.404,0.412,54.7,1710.0,2.95,0.385,0.43,Energy,1.0 +electricity/H/medium,Naive,70400000.0,70400000.0,969.0,4.09,0.591,0.393,100.0,8390.0,3.27,0.378,0.559,Energy,1.0 +ett1/15T/medium,Naive,26.0,26.0,2.79,1.82,1.11,0.617,32.4,5.1,0.971,0.532,0.617,Energy,7.0 +ett1/H/medium,Naive,472.0,472.0,12.0,2.03,9730000000000.0,0.65,46.9,21.7,1.04,0.574,0.841,Energy,7.0 +ett2/15T/medium,Naive,18.1,18.1,2.9,1.22,0.184,0.21,34.5,4.26,0.227,0.155,0.242,Energy,7.0 +ett2/H/medium,Naive,411.0,411.0,13.1,1.43,0.242,0.216,43.7,20.3,0.273,0.177,0.295,Energy,7.0 +jena_weather/10T/medium,Naive,3630.0,3630.0,15.9,0.76,0.949,0.58,162.0,60.2,0.37,0.0978,0.882,Nature,21.0 +jena_weather/H/medium,Naive,2460.0,2460.0,13.4,0.873,2.04,0.523,119.0,49.6,0.304,0.0821,0.927,Nature,21.0 +kdd_cup_2018/H/medium,Naive,5750.0,5750.0,32.5,1.45,1.52,0.706,36.0,75.8,1.59,0.68,1.0,Nature,1.0 +loop_seattle/5T/medium,Naive,392.0,392.0,12.9,1.99,0.363,0.299,44.1,19.8,0.352,0.229,0.327,Transport,1.0 +loop_seattle/H/medium,Naive,216.0,216.0,8.81,1.78,0.322,0.172,87.8,14.7,0.26,0.156,0.458,Transport,1.0 +m_dense/H/medium,Naive,361000.0,361000.0,349.0,2.74,1.18,0.73,88.6,601.0,1.05,0.607,1.2,Transport,1.0 +solar/10T/medium,Naive,80.8,80.8,5.94,2.56,2.86,1.64,33.3,8.99,1.99,1.32,1.37,Energy,1.0 +solar/H/medium,Naive,3340.0,3340.0,27.8,2.12,1.0,2.0,103.0,57.8,2.08,1.0,2.98,Energy,1.0 +sz_taxi/15T/medium,Naive,30.6,30.6,3.97,0.779,10000000000000.0,0.528,70.1,5.53,0.515,0.37,1.94,Transport,1.0 +bitbrains_fast_storage/5T/long,Naive,3500000.0,3500000.0,289.0,1.19,1.8,0.51,232.0,1870.0,4.94,0.765,2.12,Web/CloudOps,2.0 +bitbrains_rnd/5T/long,Naive,4800000.0,4800000.0,288.0,3.99,5.68,0.43,406.0,2190.0,8.39,1.1,2.36,Web/CloudOps,2.0 +bizitobs_application/10S/long,Naive,1370000000.0,1370000000.0,28300.0,38600.0,559.0,1.04,605000.0,37100.0,1.43,1.09,0.983,Web/CloudOps,2.0 +bizitobs_l2c/5T/long,Naive,353.0,353.0,13.2,1.25,0.986,1.21,9.78,18.8,1.05,0.743,0.592,Web/CloudOps,7.0 +bizitobs_l2c/H/long,Naive,944.0,944.0,27.7,4.15,6.13,1.12,25.6,30.7,1.88,1.69,1.27,Web/CloudOps,7.0 +bizitobs_service/10S/long,Naive,209000.0,209000.0,127.0,3.86,0.151,0.159,67.0,457.0,0.339,0.0939,0.119,Web/CloudOps,2.0 +electricity/15T/long,Naive,6140000.0,6140000.0,249.0,3.41,0.516,0.384,66.5,2480.0,3.91,0.393,0.478,Energy,1.0 +electricity/H/long,Naive,62400000.0,62400000.0,945.0,4.02,0.544,0.389,122.0,7900.0,3.11,0.373,0.668,Energy,1.0 +ett1/15T/long,Naive,28.4,28.4,3.04,2.04,1.17,0.68,39.7,5.33,1.02,0.579,0.716,Energy,7.0 +ett1/H/long,Naive,444.0,444.0,12.4,2.27,5650000000000.0,0.749,57.3,21.1,1.01,0.592,0.987,Energy,7.0 +ett2/15T/long,Naive,16.6,16.6,2.88,1.3,0.2,0.249,42.0,4.07,0.217,0.154,0.283,Energy,7.0 +ett2/H/long,Naive,294.0,294.0,11.5,1.32,0.218,0.222,53.5,17.2,0.235,0.157,0.349,Energy,7.0 +jena_weather/10T/long,Naive,2740.0,2740.0,13.3,0.753,0.844,0.499,198.0,52.3,0.32,0.0812,1.07,Nature,21.0 +jena_weather/H/long,Naive,3730.0,3730.0,17.7,1.47,2.37,0.575,144.0,61.0,0.368,0.107,1.14,Nature,21.0 +kdd_cup_2018/H/long,Naive,4180.0,4180.0,25.9,1.18,1.1,0.715,43.8,64.7,1.52,0.607,1.25,Nature,1.0 +loop_seattle/5T/long,Naive,214.0,214.0,9.08,1.41,0.342,0.188,54.0,14.6,0.258,0.16,0.361,Transport,1.0 +loop_seattle/H/long,Naive,210.0,210.0,8.6,1.75,0.326,0.174,108.0,14.5,0.259,0.154,0.556,Transport,1.0 +m_dense/H/long,Naive,396000.0,396000.0,362.0,2.86,1.09,0.77,108.0,629.0,1.09,0.627,1.44,Transport,1.0 +solar/10T/long,Naive,94.6,94.6,4.63,2.02,1.0,2.0,40.6,9.72,2.1,1.0,1.55,Energy,1.0 +solar/H/long,Naive,3580.0,3580.0,28.8,2.2,1.0,2.0,126.0,59.8,2.07,1.0,3.46,Energy,1.0 +sz_taxi/15T/long,Naive,28.5,28.5,3.85,0.742,9280000000000.0,0.535,85.0,5.34,0.494,0.357,2.4,Transport,1.0 diff --git a/results/naive/config.json b/results/naive/config.json index 2d911c8b174722d6d294885d59526966dc288ce9..e0cc6841f66fcb202023d04e4deeeb2fef50b12a 100644 --- a/results/naive/config.json +++ b/results/naive/config.json @@ -1,9 +1,5 @@ { "model": "Naive", "model_type": "statistical", - "model_dtype": "float32", - "testdata_leakage": "No", - "model_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/naive.ipynb", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/naive.ipynb", - "replication_code_available": "Yes" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/seasonal_naive/all_results.csv b/results/seasonal_naive/all_results.csv index 7f8f9cc8f419a62b733cabf20886baeccdf5d23e..e1e69e07f06592482f6b27958c2a21b0837a8622 100644 --- a/results/seasonal_naive/all_results.csv +++ b/results/seasonal_naive/all_results.csv @@ -1,98 +1,98 @@ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/H/short,Seasonal_Naive,4198734.112364942,4198734.112364942,323.91137144521844,1.2985124569638047,4.122732886534151,0.31268171558395724,34.57780677446709,2049.0812849579547,5.840547085975178,0.9232525964178199,1.0222395520114795,Web/CloudOps,2 -solar/H/short,Seasonal_Naive,1009.7531694199001,1009.7531694199001,12.29965959124688,0.9519362458316395,3.8670855712527974,0.6200499536903793,16.15202026601857,31.77661356123242,1.171296544389828,0.4533695432534923,0.5916559631947232,Energy,1 -solar/H/medium,Seasonal_Naive,993.4109489051095,993.4109489051095,12.224457306873479,0.9349685908749716,5.243116882202785,0.568037363492515,29.963349764914504,31.518422373353484,1.1353671382460326,0.4403534842165714,0.9459874672135314,Energy,1 -solar/H/long,Seasonal_Naive,1160.9764801297647,1160.9764801297647,13.995331508515815,1.0711966477927781,5.3301362264546865,0.7115105305381197,36.28762062617156,34.073104938202576,1.1813766446766685,0.48524364916713286,1.0779764182077312,Energy,1 -m_dense/D/short,Seasonal_Naive,39929.671111111114,39929.671111111114,118.38299768518519,1.6693368812171987,0.2532523956185716,0.2557375985604745,23.291872240434824,199.82410042612756,0.3459250756939904,0.20493848013227253,0.22686565739428607,Transport,1 -ett2/H/short,Seasonal_Naive,151.78024553571427,151.78024553571427,7.9655081612723215,0.9231678277377344,0.1445715792364967,0.13716111864362443,9.954323499249957,12.319912562015782,0.1562507702523094,0.10102480674150278,0.08895289185095483,Energy,7 -ett2/H/medium,Seasonal_Naive,338.06856398809526,338.06856398809526,11.509515671502976,1.2384249730532626,0.20117125149055434,0.19754707699730284,25.271323860489556,18.3866409109466,0.24777181028131637,0.15509812516606425,0.1862050814081793,Energy,7 -ett2/H/long,Seasonal_Naive,258.848330026455,258.848330026455,10.011796254960318,1.1284016697207502,0.19021284402568,0.18827246610449735,30.708493118791996,16.08876409257265,0.2203232414204263,0.1371038440641976,0.20790633225768584,Energy,7 -ett2/D/short,Seasonal_Naive,122878.94603174603,122878.94603174603,211.56051587301587,1.3901143157307532,0.4713712661985367,0.14168287004743305,32.70861808020462,350.5409334610525,0.20062382493032205,0.12108166507005774,0.15331100213050436,Energy,7 -electricity/15T/short,Seasonal_Naive,366722.87168918917,366722.87168918917,94.21436127533784,1.7170601412372184,0.30235384997180187,0.23004003312303262,26.97516311288429,605.5764788110492,1.193710612408421,0.18571507783204827,0.16489261500071847,Energy,1 -electricity/15T/medium,Seasonal_Naive,373592.9609594595,373592.9609594595,70.78214646677928,1.1507882041044137,0.21527551573127818,0.162019429539667,12.499327683031675,611.2225134592635,1.0531979612070823,0.12196476848793535,0.11275485280084273,Energy,1 -electricity/15T/long,Seasonal_Naive,432425.0750750751,432425.0750750751,73.83208042417418,1.1635317075319347,0.19999027586553747,0.1567721679648986,14.199582505532762,657.5903550654275,1.0374294291948993,0.11647916132088339,0.11255332426055248,Energy,1 -bitbrains_rnd/H/short,Seasonal_Naive,3710112.576957504,3710112.576957504,254.47180959890883,6.0373618687533925,4.792930625938438,0.36119250186151963,222.24992066836998,1926.1652517262125,8.429841528920612,1.1136931405932016,1.2432170953390527,Web/CloudOps,2 -loop_seattle/H/short,Seasonal_Naive,125.56935646488512,125.56935646488512,6.351615330169464,1.2928412613913995,0.1710389719335216,0.14238265652237242,14.224911524098493,11.205773354163698,0.1983535851600468,0.11243005123144872,0.10423508438727377,Transport,1 -loop_seattle/H/medium,Seasonal_Naive,153.75436661506708,153.75436661506708,7.21920260497291,1.4805126192799751,0.20086050437084785,0.16072294158463138,25.49530583293511,12.399772845301122,0.21916459102759323,0.1275985944423001,0.16208090743299983,Transport,1 -loop_seattle/H/long,Seasonal_Naive,167.0609584623323,167.0609584623323,7.511452824002408,1.5460268753677606,0.23476947145360338,0.16369716459260514,30.628061299733826,12.925206321847721,0.23099074580000042,0.13423972094936726,0.18709450197084374,Transport,1 -solar/W/short,Seasonal_Naive,2813150.8321167883,2813150.8321167883,1341.8134124087592,1.4703506784640206,0.3086572076282362,0.2534797557079009,11.583228430599936,1677.245012547895,0.3423676402558515,0.2738976644635694,0.20974762813186765,Energy,1 -bizitobs_application/10S/short,Seasonal_Naive,3938171.448888889,3938171.448888889,1054.6502083333332,2.2423302386755415,0.0340565554300944,0.03481890360514323,26.67150527524178,1984.4826653031992,0.07653356419683467,0.04067364297805646,0.03483868247005456,Web/CloudOps,2 -bizitobs_application/10S/medium,Seasonal_Naive,5143417.173333333,5143417.173333333,1276.9247916666666,2.6914183385868085,0.03906738917032877,0.03992526690165202,28.10250665325578,2267.9103098079813,0.08905416300729038,0.05014107835453231,0.04268912394422428,Web/CloudOps,2 -bizitobs_application/10S/long,Seasonal_Naive,5953087.1466666665,5953087.1466666665,1470.5443055555556,3.2063450561622804,0.04643516540527344,0.04741138034396701,21.30987046900455,2439.894904840507,0.09409699394104339,0.05671301593170149,0.04573090228155046,Web/CloudOps,2 -bizitobs_service/10S/short,Seasonal_Naive,73535.24306878306,73535.24306878306,55.138688409391534,1.225305042620374,0.08240041742879878,0.07553190362516535,16.350613928029045,271.1738244535838,0.2008942442703866,0.04084850431410673,0.03998233993351736,Web/CloudOps,2 -bizitobs_service/10S/medium,Seasonal_Naive,96480.73142857142,96480.73142857142,63.760550595238094,1.3205790727650968,0.0796314687577505,0.07705034528459821,17.448776648119132,310.6134759288003,0.23377502162466005,0.04798769290234763,0.04755831292581585,Web/CloudOps,2 -bizitobs_service/10S/long,Seasonal_Naive,130947.25079365079,130947.25079365079,75.32497354497355,1.367193469742303,0.08491984049479166,0.08144445348668981,17.936180702599792,361.8663438255218,0.26808215608217806,0.055803148480992645,0.05345299584740217,Web/CloudOps,2 -bizitobs_l2c/5T/short,Seasonal_Naive,174.3096912202381,174.3096912202381,9.685235305059523,0.9860210805885202,0.42617953698061745,0.5068802606491816,7.245192912077175,13.202639555037399,0.45343208257188783,0.3326301832497065,0.26206838623738704,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Seasonal_Naive,294.66615646258504,294.66615646258504,12.864580676020408,1.2435997989335994,0.9360730119589105,1.1668089007914848,7.637297246345627,17.16584272509174,0.9031757619892758,0.6768661254104057,0.5203917750796179,Web/CloudOps,7 -bizitobs_l2c/5T/long,Seasonal_Naive,416.33936507936505,416.33936507936505,15.353609871031747,1.4542830442631411,1.0210265852224123,1.298652753589897,8.166517613570452,20.404395729336485,1.1456763155758307,0.862082240570442,0.6484921267253538,Web/CloudOps,7 -ett2/W/short,Seasonal_Naive,4688738.857142857,4688738.857142857,1323.111607142857,0.7785209533197451,0.12217171703066144,0.1269862311226981,16.565488704288857,2165.3495923621335,0.1816396339666693,0.11098877930205708,0.13367758870767996,Energy,7 -loop_seattle/5T/short,Seasonal_Naive,76.27992181453173,76.27992181453173,4.806023281794214,0.7623397118337274,0.13014939133965933,0.10246611478035911,9.641956549599044,8.733837748351622,0.14976603568936095,0.08241268902447142,0.08083202568463041,Transport,1 -loop_seattle/5T/medium,Seasonal_Naive,149.91421294504644,149.91421294504644,7.272513948013416,1.1532851467900647,0.22437167928437823,0.16228546394529234,12.161708217874148,12.243945971174751,0.217844961125775,0.12939296873911593,0.11727937998585615,Transport,1 -loop_seattle/5T/long,Seasonal_Naive,172.4441060371517,172.4441060371517,7.891544207229676,1.2506857542807588,0.23036973349395495,0.1761461930635284,12.853493858763652,13.131797517367975,0.2321728668857043,0.13952411620149754,0.12716656087996273,Transport,1 -loop_seattle/D/short,Seasonal_Naive,58.65786974329205,58.65786974329205,5.940060831075852,1.7323957056549062,0.11095823557630288,0.10706747427075271,21.844122635807963,7.658842585096788,0.13687375626545378,0.10615682844516516,0.1032547596878987,Transport,1 -bitbrains_rnd/5T/short,Seasonal_Naive,3407833.9580161185,3407833.9580161185,210.5518366560826,1.9708126376716957,3.046095137251105,0.38408369415249266,74.37164472655921,1846.0319493486884,7.553429809324837,0.8615173317922609,1.1015904440824371,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Seasonal_Naive,3267017.897300195,3267017.897300195,204.8128727866788,4.542392314855541,2.750306364851926,0.3918429569214854,178.58186300111112,1807.489390646649,7.605029622998441,0.8617522032352579,1.169291300089368,Web/CloudOps,2 -bitbrains_rnd/5T/long,Seasonal_Naive,3851065.095876048,3851065.095876048,237.4491928177016,3.501257904408163,3.179405138911208,0.40348581728614835,137.13909859136817,1962.413079826989,7.517562382830161,0.909614361068727,1.175220506770106,Web/CloudOps,2 -electricity/W/short,Seasonal_Naive,317380739662.4144,317380739662.4144,56604.72342342342,2.0896995064573947,0.285636731125347,0.1196364761812733,15.938528356933686,563365.5471027798,1.2854142672489681,0.12915329923232946,0.0993032024444414,Energy,1 -solar/D/short,Seasonal_Naive,181869.82773722627,181869.82773722627,295.1889902676399,1.155861190610263,1.1535581565541362,0.49778203291324513,22.3811319807538,426.46198861941525,0.6160923187357285,0.4264475482750424,0.5590651164917229,Energy,1 -ett1/H/short,Seasonal_Naive,141.21642485119048,141.21642485119048,5.822230747767857,0.9773176243434534,0.5700293404715402,0.28737351553780693,9.542335004679739,11.883451722929264,0.5546493091237623,0.271747328730894,0.24036844107833774,Energy,7 -ett1/H/medium,Seasonal_Naive,204.51871279761906,204.51871279761906,8.163152785528274,1.5678007461285395,2345799664153.64,0.45697086879185267,22.421278808802057,14.301003908733787,0.6849841868181539,0.3909956677413786,0.4349118298516912,Energy,7 -ett1/H/long,Seasonal_Naive,199.49484126984126,199.49484126984126,7.749353091931217,1.4787012870437326,2646635889605.55,0.47721551158440806,25.35082572229849,14.124264273576916,0.6763804250290198,0.37109973563904386,0.47106248989542887,Energy,7 -temperature_rain/D/short,Seasonal_Naive,310.64911445557556,310.64911445557556,7.890313322375261,2.0115357653222166,97.02872693676676,1.3339967533230714,47.47823470729249,17.625240833973745,2.0749568489172328,0.9288984940737532,1.2679455200160903,Nature,1 -saugeen/W/short,Seasonal_Naive,1800.4892578125,1800.4892578125,24.552409362792968,1.990658004828632,0.824161148071289,0.557866096496582,23.291068065474015,42.43217243805106,1.2808504084686672,0.7411348925661494,0.7340352210434381,Nature,1 -covid_deaths/D/short,Seasonal_Naive,4435194.482205514,4435194.482205514,353.7093984962406,46.91239825526407,0.132724402667158,0.17781104711702989,1472.7784117419058,2105.990142950701,0.792389040211866,0.13308488253209907,0.12672327831223676,Healthcare,1 -us_births/D/short,Seasonal_Naive,2855120.2133333334,2855120.2133333334,1266.88,1.8648391912491094,0.12472813924153646,0.1233361307779948,22.730582398368462,1689.7100974230264,0.15839661042681866,0.11875971986174946,0.11952686561898423,Healthcare,1 -m4_weekly/W/short,Seasonal_Naive,453525.1459181487,453525.1459181487,347.99148275123207,2.777295047362158,0.08937289522218837,0.09161286714732429,26.631225199626535,673.442756229621,0.1226908336142798,0.06339865521526268,0.06087039452311706,Econ/Fin,1 -car_parts/M/short,Seasonal_Naive,2.5046831406935035,2.5046831406935035,0.6672313006509898,1.2014638390969912,0.8541762945076465,1.6763314144736843,20.368070639282948,1.5826190763078472,3.7949581992256025,1.5999522140809175,1.7217438941460104,Sales,1 -sz_taxi/H/short,Seasonal_Naive,12.029984433426817,12.029984433426817,2.4300792726696048,0.7381672994945998,1.552363729884482,0.38466275859082866,8.057108467274903,3.468426795166191,0.3231053592422672,0.22637687999566908,0.21382494112013678,Transport,1 -ett1/15T/short,Seasonal_Naive,8.776201520647321,8.776201520647321,1.4945350283668155,0.9341549587802876,0.6328330752819759,0.30603092738560267,8.439802226867357,2.962465446321243,0.5600292824430536,0.2825293306160358,0.24136394938567346,Energy,7 -ett1/15T/medium,Seasonal_Naive,12.883658234126985,12.883658234126985,1.8389095052083333,1.188305978388322,0.9080556087099148,0.42800251405065676,12.40964763458674,3.5893813163450585,0.6841396278696877,0.3504979699009272,0.32155116949786394,Energy,7 -ett1/15T/long,Seasonal_Naive,13.023148561507936,13.023148561507936,1.8557530381944445,1.190885705867336,0.9840622608053838,0.4204419926758006,13.893107103740524,3.608759975602137,0.6878332194845934,0.35370836394199906,0.3401793468576481,Energy,7 -saugeen/M/short,Seasonal_Naive,713.7325613839286,713.7325613839286,16.662892659505207,0.9763741730697157,0.4971456073579334,0.4422340847197033,10.586458554994822,26.71577364374703,0.8020876713555654,0.500270025848136,0.4450846286544009,Nature,1 -us_births/M/short,Seasonal_Naive,74172117.33333333,74172117.33333333,6704.75,0.7605310792868888,0.020606640726327896,0.020953829089800518,5.646371868808973,8612.323573422756,0.02674981830097076,0.020824907787592,0.016825596814792485,Healthcare,1 -m4_quarterly/Q/short,Seasonal_Naive,2487884.428,2487884.428,708.8467662760416,1.602247174616002,0.14192582162221273,0.12521366802851358,12.540135359281015,1577.302896719587,0.26401272904907985,0.11864846608179633,0.09805397548466407,Econ/Fin,1 -ett1/W/short,Seasonal_Naive,1571455.142857143,1571455.142857143,970.0256696428571,1.7689207877683926,0.6150428227015904,0.6251754760742188,12.799615221235317,1253.5769393448265,0.4988281842249676,0.38599636628003803,0.3118251468740881,Energy,7 -saugeen/D/short,Seasonal_Naive,994.7777083333333,994.7777083333333,15.3725,3.413049305768806,0.47135218302408854,0.4242374928792318,58.71112805300772,31.540096834558597,1.0215912513857213,0.49791893773514423,0.5850141736335629,Nature,1 -electricity/H/short,Seasonal_Naive,2887718.36009009,2887718.36009009,247.69792300112613,1.3577726051465433,0.3631860614314555,0.16452154722835458,18.287059022037134,1699.3287969342748,0.8022045515456071,0.11693110926995137,0.10566499518474065,Energy,1 -electricity/H/medium,Seasonal_Naive,8353587.978378379,8353587.978378379,308.2015061936937,1.3924537851702354,0.3866618745283321,0.15667597765115268,20.69150990156008,2890.2574242406813,1.128162669425473,0.1203011991361897,0.12738701179892217,Energy,1 -electricity/H/long,Seasonal_Naive,13526606.943327328,13526606.943327328,357.2064234234234,1.5240786163496851,0.36051114996741657,0.16771684918985819,24.628137038027603,3677.8535782882013,1.4505443840804357,0.14088210974822626,0.15341839756227935,Energy,1 -ett1/D/short,Seasonal_Naive,57976.62857142857,57976.62857142857,154.97894345238095,1.7783697877928102,1.1623038155691965,0.5164825923859127,23.47582377746543,240.7833643992636,0.6324254934604606,0.4070573356817528,0.40843357791503654,Energy,7 -bizitobs_l2c/H/short,Seasonal_Naive,281.8430679563492,281.8430679563492,12.53165302579365,1.2140641267600039,1.3605904339028776,1.138373051002047,7.486930567002142,16.788182389894065,0.9049260260934667,0.675488192208351,0.5211675771895117,Web/CloudOps,7 -bizitobs_l2c/H/medium,Seasonal_Naive,456.3732886904762,456.3732886904762,15.667392113095238,1.5102861226255864,1.6912910179529201,1.4024095456148358,18.533653530822832,21.362895138311103,1.293555748999092,0.9486843898499615,0.9042051354742671,Web/CloudOps,7 -bizitobs_l2c/H/long,Seasonal_Naive,309.27222222222224,309.27222222222224,13.635487971230159,1.4260541525613812,2.4383105011700468,0.9168538411458333,22.036198194504,17.586137217201003,1.0742120179443015,0.8328949597245134,0.9410651242377539,Web/CloudOps,7 -hospital/M/short,Seasonal_Naive,3464.0771403737504,3464.0771403737504,20.005975662755322,0.9205278266364826,0.23307140471364352,0.21025354682958836,6.599875528051222,58.85641120875236,0.21366243998041978,0.07262633732682543,0.06248817651596496,Healthcare,1 -m4_daily/D/short,Seasonal_Naive,497624.29605596676,497624.29605596676,180.83018393659805,3.278424296610481,0.04217648847004251,0.03045251709590828,32.19152086932198,705.4249046184624,0.10896865245350737,0.02793326594720746,0.024356147648337902,Econ/Fin,1 -bitbrains_fast_storage/5T/short,Seasonal_Naive,5404398.959598898,5404398.959598898,346.43736615760264,1.1360241737534567,5.932190463200192,0.4761820977820179,31.06038796757978,2324.736320445589,7.299097101583847,1.087727650212227,1.210386425305063,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Seasonal_Naive,5191990.375993042,5191990.375993042,332.75293038959035,1.2202713595882653,5.768419487236171,0.4516151765437165,36.13464652226377,2278.593947150971,6.92299777327362,1.0109953109538277,1.1979128978088514,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Seasonal_Naive,6473012.871295428,6473012.871295428,404.5292260549034,1.1366422318753115,6.462525817893773,0.5033018991430218,32.16149564789198,2544.2116404291974,6.723591850916995,1.0690499817466734,1.1773077032683998,Web/CloudOps,2 -ett2/15T/short,Seasonal_Naive,14.617721121651785,14.617721121651785,2.534710984002976,1.067314800011479,0.15340707960692476,0.17031889997991706,8.528224043296179,3.8233128464267456,0.181506262086668,0.12033174753313121,0.09638438775638664,Energy,7 -ett2/15T/medium,Seasonal_Naive,14.948670634920635,14.948670634920635,2.4787872023809525,1.0512391501712754,0.1650384327009311,0.1899925877490716,13.312453415228259,3.8663510749698657,0.20604484738813625,0.1320990569451144,0.12412877798161984,Energy,7 -ett2/15T/long,Seasonal_Naive,13.674988839285714,13.674988839285714,2.362206101190476,1.0126577751121104,0.1609817984920091,0.1894470412273871,15.633896791936913,3.697970908388236,0.19707156352079597,0.12588623903557683,0.13282444765695306,Energy,7 -jena_weather/10T/short,Seasonal_Naive,1986.203373015873,1986.203373015873,10.556470114087302,0.7428874675800381,0.8774809708937492,0.39362146904346257,20.65580527060839,44.566841631597285,0.2768159431081858,0.06556891005829443,0.15520092375681044,Nature,21 -jena_weather/10T/medium,Seasonal_Naive,2572.8219336219336,2572.8219336219336,12.447531114718615,0.7160100391723763,0.8990536291309329,0.4023956803535614,27.512189615220795,50.72299215959104,0.311286135757249,0.07639028565640756,0.21183609008512827,Nature,21 -jena_weather/10T/long,Seasonal_Naive,2044.5376984126983,2044.5376984126983,11.441627397486773,0.7614923703210493,0.7718160693153647,0.4043349656668781,32.10701219708694,45.21656442513847,0.2766879110680792,0.0700132799578619,0.2371949131202996,Nature,21 -jena_weather/D/short,Seasonal_Naive,633.5864087301587,633.5864087301587,13.136656746031745,1.5734140297468562,1.7707135350571066,0.4854136849221913,31.00640730511845,25.17114238031637,0.15157141745411598,0.07910414448099358,0.2105908566035542,Nature,21 -us_births/W/short,Seasonal_Naive,5288246.857142857,5288246.857142857,1720.3214285714287,1.5634204337929731,0.023174324205943515,0.023229679891041348,13.279190377688378,2299.6188504060533,0.031217100406456557,0.023353194707727513,0.01929679613291714,Healthcare,1 -m_dense/H/short,Seasonal_Naive,145691.3111111111,145691.3111111111,180.18139756944444,1.4875444172240992,0.5399732053067908,0.3908195287297072,14.354127446705878,381.69531187992226,0.6770323708995337,0.31959690096169935,0.27479786933969136,Transport,1 -m_dense/H/medium,Seasonal_Naive,158402.70222222223,158402.70222222223,189.1006423611111,1.569946511898904,0.5428334697930458,0.397995368109809,22.89829990230752,397.99836962256796,0.6924668813918432,0.32901122738058514,0.3773835994986822,Transport,1 -m_dense/H/long,Seasonal_Naive,137584.05530864198,137584.05530864198,175.8238888888889,1.4779178744488217,0.4978060829089136,0.38833718102796383,27.64819460133164,370.92324719359664,0.6421229733186591,0.3043771431636812,0.4189507904607682,Transport,1 -kdd_cup_2018/D/short,Seasonal_Naive,3183.23772462431,3183.23772462431,23.884613481510574,1.4970343738270926,0.5893064466306873,0.5721879846611051,24.580612052672397,56.42018898075679,1.263872161836989,0.5350407118596068,0.674506129145155,Nature,1 -electricity/D/short,Seasonal_Naive,4108036968.189694,4108036968.189694,6964.321117117117,1.987041399842847,0.6052061423957089,0.13582226488305205,24.595303547998157,64093.9698270414,1.0532768308357283,0.11444692995228782,0.10405987098536745,Energy,1 -restaurant/D/short,Seasonal_Naive,269.8283833006444,269.8283833006444,10.66129634818343,1.0060757450046092,0.9434227416570351,0.5561225187815214,20.74904381007386,16.426453765211907,0.7559834619860213,0.49065755991884735,0.6770164746085788,Sales,1 -jena_weather/H/short,Seasonal_Naive,1941.9557226399331,1941.9557226399331,11.035693889672096,0.7229102185557323,1.1349936220434893,0.40077710759656054,17.382581140637207,44.06762669624872,0.2701317147149199,0.06764809310095782,0.15420788787522166,Nature,21 -jena_weather/H/medium,Seasonal_Naive,1951.8111111111111,1951.8111111111111,12.670400080605159,0.8886133410507664,1.7017065286708326,0.45474790719102887,42.286346185217376,44.1793063674738,0.27034101099927343,0.077532425227887,0.3428055217231077,Nature,21 -jena_weather/H/long,Seasonal_Naive,2714.1007936507935,2714.1007936507935,15.67983630952381,1.2677178473920059,1.8661278391921836,0.5020780815368567,51.127769997760254,52.097032484113655,0.31368185519406616,0.09440998667650671,0.41914340130808575,Nature,21 -m4_monthly/M/short,Seasonal_Naive,2652970.8577407408,2652970.8577407408,700.2370131655092,1.259717038974429,0.19222692431343927,0.15988281532570167,10.588031340884587,1628.794295711015,0.33851937015163663,0.14553329004027185,0.12192063664829357,Econ/Fin,1 -hierarchical_sales/D/short,Seasonal_Naive,58.96940884663084,58.96940884663084,3.4495245969408845,1.1348033996861053,1.219942090035255,1.1912773542022284,28.8672586949405,7.679154175209066,2.3558794594517027,1.058275945161004,1.7364572065613715,Sales,1 -m4_hourly/H/short,Seasonal_Naive,3614355.993558776,3614355.993558776,353.8562298711755,1.1932101877276544,0.1561203125786282,0.1391227341312525,9.229226853352637,1901.1459685039379,0.25954838815818215,0.04830918594592099,0.03757255147212195,Econ/Fin,1 -m4_yearly/A/short,Seasonal_Naive,4070174.893125562,4070174.893125562,1007.6444123429384,3.965954931783283,0.17520083696448324,0.16353434447586868,52.037766411775884,2017.4674453694568,0.323510256896443,0.1615804524853844,0.13714565740983498,Econ/Fin,1 -hierarchical_sales/W/short,Seasonal_Naive,1045.4721927966102,1045.4721927966102,13.569120762711865,1.0250133777288337,0.9503837463030324,0.5798107884238987,15.297190661198302,32.33376242871544,1.4844588487218926,0.6229649713667368,0.8322267526416594,Sales,1 -kdd_cup_2018/H/short,Seasonal_Naive,5615.976023835748,5615.976023835748,32.23957254610716,1.340425278558367,1.4218929148819215,0.6653979848821596,11.487358350395635,74.9398160114885,1.5686260624453057,0.6748326381021144,0.547536580422314,Nature,1 -kdd_cup_2018/H/medium,Seasonal_Naive,5785.5976861948375,5785.5976861948375,31.3213270934715,1.4289997072062277,1.6378374272295781,0.6550327560864638,21.848163370875977,76.06311646386071,1.592138770600474,0.6556120959870002,0.7587892026846513,Nature,1 -kdd_cup_2018/H/long,Seasonal_Naive,4863.8870949314805,4863.8870949314805,29.603438898808303,1.3355336658089667,1.3504101339311876,0.8098963463700346,25.694140506460055,69.74157364822995,1.6364191009705393,0.694616286015704,0.9362833347539716,Nature,1 -sz_taxi/15T/short,Seasonal_Naive,31.23822759271978,31.23822759271978,3.8183243114578564,0.7644167735543712,1312692316403.7466,0.538321787329966,7.055930253561324,5.58911688844667,0.5226063327326282,0.3570296534180041,0.3087068505119876,Transport,1 -sz_taxi/15T/medium,Seasonal_Naive,28.930445379273504,28.930445379273504,3.6711446647970085,0.7134840540400068,18917890105834.12,0.5172677325387286,10.816848306969383,5.378702945810774,0.5005780877467839,0.34166128798341444,0.379172158612487,Transport,1 -sz_taxi/15T/long,Seasonal_Naive,28.48211582977208,28.48211582977208,3.6789685719373217,0.6909431344941248,3284113398767.079,0.5217545990251068,12.685376303466118,5.336863857151696,0.4939706119705816,0.3405187776085254,0.4279704063043717,Transport,1 -solar/10T/short,Seasonal_Naive,37.917036857512166,37.917036857512166,2.5271684706936206,1.1057028063509833,4.40491207343912,0.925039670208932,18.785745549933484,6.157681126650857,1.7912687729072838,0.7351530344494263,0.8595386998005952,Energy,1 -solar/10T/medium,Seasonal_Naive,29.228987641008626,29.228987641008626,2.118083017100752,0.9270005404761702,3.219568147534267,0.6587355148950821,15.807425762449464,5.406383970918883,1.1996980737201242,0.4700110331163171,0.6550635536405177,Energy,1 -solar/10T/long,Seasonal_Naive,26.90307906275345,26.90307906275345,1.995840144718167,0.8709317964102903,2.310040346896705,0.6597688060895561,18.202191205836492,5.18681781661487,1.1210476934837867,0.4313689182665159,0.6738494837655387,Energy,1 +bitbrains_fast_storage/5T/short,Seasonal_Naive,5400000.0,5400000.0,346.0,1.14,5.93,0.476,31.1,2320.0,7.3,1.09,1.21,Web/CloudOps,2.0 +bitbrains_fast_storage/H/short,Seasonal_Naive,4200000.0,4200000.0,324.0,1.3,4.12,0.313,35.8,2050.0,5.84,0.923,1.08,Web/CloudOps,2.0 +bitbrains_rnd/5T/short,Seasonal_Naive,3410000.0,3410000.0,211.0,1.97,3.05,0.384,74.4,1850.0,7.55,0.862,1.1,Web/CloudOps,2.0 +bitbrains_rnd/H/short,Seasonal_Naive,3710000.0,3710000.0,254.0,6.04,4.79,0.361,223.0,1930.0,8.43,1.11,1.3,Web/CloudOps,2.0 +bizitobs_application/10S/short,Seasonal_Naive,3940000.0,3940000.0,1050.0,2.24,0.0341,0.0348,26.7,1980.0,0.0765,0.0407,0.0348,Web/CloudOps,2.0 +bizitobs_l2c/5T/short,Seasonal_Naive,174.0,174.0,9.69,0.986,0.426,0.507,7.25,13.2,0.453,0.333,0.262,Web/CloudOps,7.0 +bizitobs_l2c/H/short,Seasonal_Naive,282.0,282.0,12.5,1.21,1.36,1.14,8.5,16.8,0.905,0.675,0.536,Web/CloudOps,7.0 +bizitobs_service/10S/short,Seasonal_Naive,73500.0,73500.0,55.1,1.23,0.0824,0.0755,16.4,271.0,0.201,0.0408,0.04,Web/CloudOps,2.0 +car_parts/M/short,Seasonal_Naive,2.5,2.5,0.667,1.2,0.854,1.68,20.4,1.58,3.79,1.6,1.72,Sales,1.0 +covid_deaths/D/short,Seasonal_Naive,4440000.0,4440000.0,354.0,46.9,0.133,0.178,1390.0,2110.0,0.792,0.133,0.125,Healthcare,1.0 +electricity/15T/short,Seasonal_Naive,367000.0,367000.0,94.2,1.72,0.302,0.23,27.0,606.0,1.19,0.186,0.165,Energy,1.0 +electricity/D/short,Seasonal_Naive,4110000000.0,4110000000.0,6960.0,1.99,0.605,0.136,33.8,64100.0,1.05,0.114,0.122,Energy,1.0 +electricity/H/short,Seasonal_Naive,2890000.0,2890000.0,248.0,1.36,0.363,0.165,18.0,1700.0,0.802,0.117,0.109,Energy,1.0 +electricity/W/short,Seasonal_Naive,317000000000.0,317000000000.0,56600.0,2.09,0.286,0.12,18.0,563000.0,1.29,0.129,0.0989,Energy,1.0 +ett1/15T/short,Seasonal_Naive,8.78,8.78,1.49,0.934,0.633,0.306,8.44,2.96,0.56,0.283,0.241,Energy,7.0 +ett1/D/short,Seasonal_Naive,58000.0,58000.0,155.0,1.78,1.16,0.516,34.2,241.0,0.632,0.407,0.515,Energy,7.0 +ett1/H/short,Seasonal_Naive,141.0,141.0,5.82,0.977,0.57,0.287,10.2,11.9,0.555,0.272,0.25,Energy,7.0 +ett1/W/short,Seasonal_Naive,1570000.0,1570000.0,970.0,1.77,0.615,0.625,17.8,1250.0,0.499,0.386,0.338,Energy,7.0 +ett2/15T/short,Seasonal_Naive,14.6,14.6,2.53,1.07,0.153,0.17,8.53,3.82,0.182,0.12,0.0964,Energy,7.0 +ett2/D/short,Seasonal_Naive,123000.0,123000.0,212.0,1.39,0.471,0.142,48.0,351.0,0.201,0.121,0.205,Energy,7.0 +ett2/H/short,Seasonal_Naive,152.0,152.0,7.97,0.923,0.145,0.137,11.5,12.3,0.156,0.101,0.0941,Energy,7.0 +ett2/W/short,Seasonal_Naive,4690000.0,4690000.0,1320.0,0.779,0.122,0.127,23.0,2170.0,0.182,0.111,0.169,Energy,7.0 +hierarchical_sales/D/short,Seasonal_Naive,59.0,59.0,3.45,1.13,1.22,1.19,40.8,7.68,2.36,1.06,2.36,Sales,1.0 +hierarchical_sales/W/short,Seasonal_Naive,1050.0,1050.0,13.6,1.03,0.95,0.58,20.6,32.3,1.48,0.623,1.03,Sales,1.0 +hospital/M/short,Seasonal_Naive,3460.0,3460.0,20.0,0.921,0.233,0.21,6.6,58.9,0.214,0.0726,0.0625,Healthcare,1.0 +jena_weather/10T/short,Seasonal_Naive,1990.0,1990.0,10.6,0.743,0.877,0.394,20.7,44.6,0.277,0.0656,0.155,Nature,21.0 +jena_weather/D/short,Seasonal_Naive,634.0,634.0,13.1,1.57,1.77,0.485,45.4,25.2,0.152,0.0791,0.297,Nature,21.0 +jena_weather/H/short,Seasonal_Naive,1940.0,1940.0,11.0,0.723,1.13,0.401,19.8,44.1,0.27,0.0676,0.173,Nature,21.0 +kdd_cup_2018/D/short,Seasonal_Naive,3180.0,3180.0,23.9,1.5,0.589,0.572,35.5,56.4,1.26,0.535,0.888,Nature,1.0 +kdd_cup_2018/H/short,Seasonal_Naive,5620.0,5620.0,32.2,1.34,1.42,0.665,12.1,74.9,1.57,0.675,0.559,Nature,1.0 +loop_seattle/5T/short,Seasonal_Naive,76.3,76.3,4.81,0.762,0.13,0.102,9.64,8.73,0.15,0.0824,0.0808,Transport,1.0 +loop_seattle/D/short,Seasonal_Naive,58.7,58.7,5.94,1.73,0.111,0.107,32.0,7.66,0.137,0.106,0.131,Transport,1.0 +loop_seattle/H/short,Seasonal_Naive,126.0,126.0,6.35,1.29,0.171,0.142,14.4,11.2,0.198,0.112,0.108,Transport,1.0 +m_dense/D/short,Seasonal_Naive,39900.0,39900.0,118.0,1.67,0.253,0.256,34.1,200.0,0.346,0.205,0.294,Transport,1.0 +m_dense/H/short,Seasonal_Naive,146000.0,146000.0,180.0,1.49,0.54,0.391,14.1,382.0,0.677,0.32,0.281,Transport,1.0 +m4_daily/D/short,Seasonal_Naive,498000.0,498000.0,181.0,3.28,0.0422,0.0305,36.5,705.0,0.109,0.0279,0.0265,Econ/Fin,1.0 +m4_hourly/H/short,Seasonal_Naive,3610000.0,3610000.0,354.0,1.19,0.156,0.139,10.0,1900.0,0.26,0.0483,0.0395,Econ/Fin,1.0 +m4_monthly/M/short,Seasonal_Naive,2650000.0,2650000.0,700.0,1.26,0.192,0.16,11.1,1630.0,0.339,0.146,0.126,Econ/Fin,1.0 +m4_quarterly/Q/short,Seasonal_Naive,2490000.0,2490000.0,709.0,1.6,0.142,0.125,12.6,1580.0,0.264,0.119,0.0992,Econ/Fin,1.0 +m4_weekly/W/short,Seasonal_Naive,454000.0,454000.0,348.0,2.78,0.0894,0.0916,31.2,673.0,0.123,0.0634,0.0727,Econ/Fin,1.0 +m4_yearly/A/short,Seasonal_Naive,4070000.0,4070000.0,1010.0,3.97,0.175,0.164,47.0,2020.0,0.324,0.162,0.138,Econ/Fin,1.0 +restaurant/D/short,Seasonal_Naive,274.0,274.0,10.7,1.01,0.944,0.556,30.2,16.6,0.757,0.49,0.907,Sales,1.0 +saugeen/D/short,Seasonal_Naive,995.0,995.0,15.4,3.41,0.471,0.424,77.2,31.5,1.02,0.498,0.754,Nature,1.0 +saugeen/M/short,Seasonal_Naive,714.0,714.0,16.7,0.976,0.497,0.442,10.6,26.7,0.802,0.5,0.445,Nature,1.0 +saugeen/W/short,Seasonal_Naive,1800.0,1800.0,24.6,1.99,0.824,0.558,30.4,42.4,1.28,0.741,0.855,Nature,1.0 +solar/10T/short,Seasonal_Naive,37.9,37.9,2.53,1.11,4.4,0.925,18.8,6.16,1.79,0.735,0.86,Energy,1.0 +solar/D/short,Seasonal_Naive,182000.0,182000.0,295.0,1.16,1.15,0.498,32.8,426.0,0.616,0.426,0.757,Energy,1.0 +solar/H/short,Seasonal_Naive,1010.0,1010.0,12.3,0.952,3.87,0.62,16.6,31.8,1.17,0.453,0.628,Energy,1.0 +solar/W/short,Seasonal_Naive,2810000.0,2810000.0,1340.0,1.47,0.309,0.253,16.1,1680.0,0.342,0.274,0.236,Energy,1.0 +sz_taxi/15T/short,Seasonal_Naive,31.2,31.2,3.82,0.764,1310000000000.0,0.538,7.06,5.59,0.523,0.357,0.309,Transport,1.0 +sz_taxi/H/short,Seasonal_Naive,12.0,12.0,2.43,0.738,1.55,0.385,9.27,3.47,0.323,0.226,0.229,Transport,1.0 +temperature_rain/D/short,Seasonal_Naive,311.0,311.0,7.89,2.01,97.0,1.33,61.8,17.6,2.07,0.929,1.63,Nature,1.0 +us_births/D/short,Seasonal_Naive,2860000.0,2860000.0,1270.0,1.86,0.125,0.123,32.8,1690.0,0.158,0.119,0.144,Healthcare,1.0 +us_births/M/short,Seasonal_Naive,74200000.0,74200000.0,6700.0,0.761,0.0206,0.021,5.65,8610.0,0.0267,0.0208,0.0168,Healthcare,1.0 +us_births/W/short,Seasonal_Naive,5290000.0,5290000.0,1720.0,1.56,0.0232,0.0232,17.0,2300.0,0.0312,0.0234,0.0216,Healthcare,1.0 +bitbrains_fast_storage/5T/medium,Seasonal_Naive,5190000.0,5190000.0,333.0,1.22,5.77,0.452,39.1,2280.0,6.92,1.01,1.27,Web/CloudOps,2.0 +bitbrains_rnd/5T/medium,Seasonal_Naive,3270000.0,3270000.0,205.0,4.54,2.75,0.392,183.0,1810.0,7.61,0.862,1.26,Web/CloudOps,2.0 +bizitobs_application/10S/medium,Seasonal_Naive,5140000.0,5140000.0,1280.0,2.69,0.0391,0.0399,21.6,2270.0,0.0891,0.0501,0.0419,Web/CloudOps,2.0 +bizitobs_l2c/5T/medium,Seasonal_Naive,295.0,295.0,12.9,1.24,0.936,1.17,8.51,17.2,0.903,0.677,0.53,Web/CloudOps,7.0 +bizitobs_l2c/H/medium,Seasonal_Naive,447.0,447.0,15.6,1.65,1.6,1.48,47.1,21.1,1.28,0.946,1.42,Web/CloudOps,7.0 +bizitobs_service/10S/medium,Seasonal_Naive,96500.0,96500.0,63.8,1.32,0.0796,0.0771,17.4,311.0,0.234,0.048,0.0486,Web/CloudOps,2.0 +electricity/15T/medium,Seasonal_Naive,374000.0,374000.0,70.8,1.15,0.215,0.162,14.7,611.0,1.05,0.122,0.124,Energy,1.0 +electricity/H/medium,Seasonal_Naive,8350000.0,8350000.0,308.0,1.39,0.387,0.157,28.7,2890.0,1.13,0.12,0.156,Energy,1.0 +ett1/15T/medium,Seasonal_Naive,12.9,12.9,1.84,1.19,0.908,0.428,14.9,3.59,0.684,0.35,0.352,Energy,7.0 +ett1/H/medium,Seasonal_Naive,205.0,205.0,8.16,1.57,2350000000000.0,0.457,29.9,14.3,0.685,0.391,0.54,Energy,7.0 +ett2/15T/medium,Seasonal_Naive,14.9,14.9,2.48,1.05,0.165,0.19,17.5,3.87,0.206,0.132,0.143,Energy,7.0 +ett2/H/medium,Seasonal_Naive,338.0,338.0,11.5,1.24,0.201,0.198,36.6,18.4,0.248,0.155,0.241,Energy,7.0 +jena_weather/10T/medium,Seasonal_Naive,2570.0,2570.0,12.4,0.716,0.899,0.402,37.5,50.7,0.311,0.0764,0.277,Nature,21.0 +jena_weather/H/medium,Seasonal_Naive,1950.0,1950.0,12.7,0.889,1.7,0.455,61.3,44.2,0.27,0.0775,0.486,Nature,21.0 +kdd_cup_2018/H/medium,Seasonal_Naive,5790.0,5790.0,31.3,1.43,1.64,0.655,30.7,76.1,1.59,0.656,0.949,Nature,1.0 +loop_seattle/5T/medium,Seasonal_Naive,150.0,150.0,7.27,1.15,0.224,0.162,12.3,12.2,0.218,0.129,0.123,Transport,1.0 +loop_seattle/H/medium,Seasonal_Naive,154.0,154.0,7.22,1.48,0.201,0.161,35.8,12.4,0.219,0.128,0.206,Transport,1.0 +m_dense/H/medium,Seasonal_Naive,158000.0,158000.0,189.0,1.57,0.543,0.398,32.6,398.0,0.692,0.329,0.479,Transport,1.0 +solar/10T/medium,Seasonal_Naive,29.2,29.2,2.12,0.927,3.22,0.659,19.5,5.41,1.2,0.47,0.771,Energy,1.0 +solar/H/medium,Seasonal_Naive,993.0,993.0,12.2,0.935,5.24,0.568,43.1,31.5,1.14,0.44,1.27,Energy,1.0 +sz_taxi/15T/medium,Seasonal_Naive,28.9,28.9,3.67,0.713,18900000000000.0,0.517,14.3,5.38,0.501,0.342,0.454,Transport,1.0 +bitbrains_fast_storage/5T/long,Seasonal_Naive,6470000.0,6470000.0,405.0,1.14,6.46,0.503,37.2,2540.0,6.72,1.07,1.29,Web/CloudOps,2.0 +bitbrains_rnd/5T/long,Seasonal_Naive,3850000.0,3850000.0,237.0,3.5,3.18,0.403,145.0,1960.0,7.52,0.91,1.29,Web/CloudOps,2.0 +bizitobs_application/10S/long,Seasonal_Naive,1230000000.0,1230000000.0,26000.0,36400.0,538.0,1.03,1310000.0,35000.0,1.35,1.0,0.973,Web/CloudOps,2.0 +bizitobs_l2c/5T/long,Seasonal_Naive,416.0,416.0,15.4,1.45,1.02,1.3,10.4,20.4,1.15,0.862,0.674,Web/CloudOps,7.0 +bizitobs_l2c/H/long,Seasonal_Naive,907.0,907.0,27.0,4.04,5.97,1.11,59.4,30.1,1.84,1.65,1.82,Web/CloudOps,7.0 +bizitobs_service/10S/long,Seasonal_Naive,131000.0,131000.0,75.3,1.37,0.0849,0.0814,19.3,362.0,0.268,0.0558,0.0559,Web/CloudOps,2.0 +electricity/15T/long,Seasonal_Naive,432000.0,432000.0,73.8,1.16,0.2,0.157,17.8,658.0,1.04,0.116,0.129,Energy,1.0 +electricity/H/long,Seasonal_Naive,13500000.0,13500000.0,357.0,1.52,0.361,0.168,34.9,3680.0,1.45,0.141,0.19,Energy,1.0 +ett1/15T/long,Seasonal_Naive,13.0,13.0,1.86,1.19,0.984,0.42,18.7,3.61,0.688,0.354,0.396,Energy,7.0 +ett1/H/long,Seasonal_Naive,199.0,199.0,7.75,1.48,2650000000000.0,0.477,36.6,14.1,0.676,0.371,0.616,Energy,7.0 +ett2/15T/long,Seasonal_Naive,13.7,13.7,2.36,1.01,0.161,0.189,22.2,3.7,0.197,0.126,0.165,Energy,7.0 +ett2/H/long,Seasonal_Naive,259.0,259.0,10.0,1.13,0.19,0.188,45.0,16.1,0.22,0.137,0.287,Energy,7.0 +jena_weather/10T/long,Seasonal_Naive,2040.0,2040.0,11.4,0.761,0.772,0.404,42.1,45.2,0.277,0.07,0.304,Nature,21.0 +jena_weather/H/long,Seasonal_Naive,2710.0,2710.0,15.7,1.27,1.87,0.502,74.9,52.1,0.314,0.0944,0.598,Nature,21.0 +kdd_cup_2018/H/long,Seasonal_Naive,4860.0,4860.0,29.6,1.34,1.35,0.81,37.3,69.7,1.64,0.695,1.25,Nature,1.0 +loop_seattle/5T/long,Seasonal_Naive,172.0,172.0,7.89,1.25,0.23,0.176,13.7,13.1,0.232,0.14,0.137,Transport,1.0 +loop_seattle/H/long,Seasonal_Naive,167.0,167.0,7.51,1.55,0.235,0.164,44.0,12.9,0.231,0.134,0.245,Transport,1.0 +m_dense/H/long,Seasonal_Naive,138000.0,138000.0,176.0,1.48,0.498,0.388,39.9,371.0,0.642,0.304,0.552,Transport,1.0 +solar/10T/long,Seasonal_Naive,26.9,26.9,2.0,0.871,2.31,0.66,21.6,5.19,1.12,0.431,0.786,Energy,1.0 +solar/H/long,Seasonal_Naive,1160.0,1160.0,14.0,1.07,5.33,0.712,52.7,34.1,1.18,0.485,1.47,Energy,1.0 +sz_taxi/15T/long,Seasonal_Naive,28.5,28.5,3.68,0.691,3280000000000.0,0.522,18.0,5.34,0.494,0.341,0.554,Transport,1.0 diff --git a/results/seasonal_naive/config.json b/results/seasonal_naive/config.json index ab0b24afde5c5a424144bd07e3ee968d6ddb4e9d..b1bbd11a136f62b9b129ec1ad6e61bb5af33681e 100644 --- a/results/seasonal_naive/config.json +++ b/results/seasonal_naive/config.json @@ -1,9 +1,5 @@ { "model": "Seasonal_Naive", "model_type": "statistical", - "model_dtype": "float32", - "model_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/naive.ipynb", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/naive.ipynb", - "testdata_leakage": "No", - "replication_code_available": "Yes" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/sundial_base_128m/all_results.csv b/results/sundial_base_128m/all_results.csv deleted file mode 100644 index 874fce4bc2b976be3ecceaabe372fa01a47e4e0e..0000000000000000000000000000000000000000 --- a/results/sundial_base_128m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -loop_seattle/5T/short,sundial_base_128m,37.38912407685113,37.79311926559275,3.4166639455946854,0.5418677171663021,0.09302349149627213,0.07274604222599813,7.18608638094916,6.114664674113466,0.10485300271420192,0.0585882453174582,0.04965148377952988,Transport,1 -loop_seattle/5T/medium,sundial_base_128m,73.50833688080495,74.1760066756966,5.1912426631836945,0.8196109744635476,0.16633652882925373,0.11945720078160273,10.00461001995478,8.57370030271673,0.1525437419054575,0.09236287169151089,0.07743672868191195,Transport,1 -loop_seattle/5T/long,sundial_base_128m,89.1548085081986,89.1674085540649,5.676028121775026,0.8928347181609574,0.1759994486189944,0.12873686830817854,11.336979324492034,9.44218240176489,0.16693971589687603,0.10035333800527821,0.08449760538340932,Transport,1 -loop_seattle/D/short,sundial_base_128m,18.429789086687308,18.52360197368421,2.9816349812951497,0.8995514672181137,0.05507054127160733,0.05483099245557598,17.340802301170243,4.292993021970489,0.07672152690732546,0.05328580486260833,0.047140832678401315,Transport,1 -loop_seattle/H/short,sundial_base_128m,60.18822434617891,59.60882590570854,4.346736920183586,0.8804201037957281,0.11585792996093326,0.0996091810639487,14.29746495628067,7.758107007909785,0.13732638336324776,0.07694166374242092,0.06656832037283406,Transport,1 -loop_seattle/H/medium,sundial_base_128m,68.30358617131063,68.25909442724458,4.956817998581012,1.0140278217947962,0.13040064761513157,0.11203929479892286,15.993568057047899,8.264598367211235,0.1460758345241723,0.0876111933762954,0.07529871485222338,Transport,1 -loop_seattle/H/long,sundial_base_128m,61.90195218438252,61.99824991400069,4.742837332301342,0.9871012881124152,0.1341950817127838,0.10877739716363519,14.785401125459641,7.867779368054402,0.14060775076603935,0.08476085288459285,0.07223909651791788,Transport,1 -m_dense/D/short,sundial_base_128m,7680.636296296297,7703.6059259259255,46.08437789351852,0.681301001844435,0.11990517064144737,0.09835581461588541,9.91669473889226,87.63923947808023,0.1517164870782257,0.07977887490612931,0.0667818156593893,Transport,1 -m_dense/H/short,sundial_base_128m,37860.537777777776,38285.991111111114,88.01905381944445,0.7911527641202473,0.2899245252902517,0.21493893093532987,12.412157782362964,194.5778450332354,0.34513261139769036,0.15612386853328483,0.1332013739663772,Transport,1 -m_dense/H/medium,sundial_base_128m,35793.24888888889,35781.76222222222,88.92399305555556,0.7587427424857901,0.31726641572233644,0.2202114529079861,10.309195756556583,189.19103807762377,0.32916850450703183,0.15471651356382551,0.12820447024260126,Transport,1 -m_dense/H/long,sundial_base_128m,36111.89728395062,36283.50419753086,91.45063271604938,0.7708725770495878,0.31554077114423046,0.22397265022183643,9.930669545560853,190.03130606284486,0.32897228252995997,0.15831456409323705,0.13037685659946585,Transport,1 -sz_taxi/15T/short,sundial_base_128m,16.603792407137135,16.572001362894916,2.7728853889437386,0.5537644512663443,1009814854269.8967,0.40059981852660687,8.565742344193046,4.074775135775854,0.3810089022864843,0.2592766430058536,0.22330872874467816,Transport,1 -sz_taxi/15T/medium,sundial_base_128m,17.097115384615385,16.801088408119657,2.8827486478365385,0.5632305341689838,10381046999526.62,0.4233590150490785,7.808679476853401,4.1348658242578304,0.384818282074604,0.26828787909035945,0.22793119415486818,Transport,1 -sz_taxi/15T/long,sundial_base_128m,16.49652221331909,16.136390446937323,2.842568108974359,0.5372017514900559,6770085977715.236,0.42102996739227205,6.8094228015750105,4.061591093810293,0.3759336366610085,0.26310304065120377,0.22107241287489293,Transport,1 -sz_taxi/H/short,sundial_base_128m,7.454074435763889,7.3442643646501065,1.9091939029530582,0.581415842067849,1.1498924320579593,0.30563041491386217,9.275420074651208,2.7302150896520754,0.25433638345203247,0.17785319348140846,0.1539936485059681,Transport,1 -bitbrains_fast_storage/5T/short,sundial_base_128m,1667311.4097309958,1711803.8958942452,181.00176684311495,0.7400817080132996,2.7738987359024674,0.7436391409035517,15.030792252691075,1291.2441325059315,4.054187281222474,0.5683007903315925,0.4616509497480503,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,sundial_base_128m,2907838.320472667,2877131.708869361,312.31870573567153,1.107871728301013,5.429366768086938,0.8071652781288694,23.570398005670967,1705.238493722408,5.1809858111874965,0.9489105418050499,0.7275862010337261,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,sundial_base_128m,3992984.6565324483,4074699.162123445,402.33512084912155,1.0110702542831842,5.992031531062318,0.8165969330820964,18.612015056705424,1998.2453944729732,5.280765984355254,1.0632516038160884,0.8112800068833039,Web/CloudOps,2 -bitbrains_fast_storage/H/short,sundial_base_128m,3002624.164511734,2752943.8191741854,339.0138164151865,1.1499014166635186,4.92548112469647,0.6058981282372182,26.78099747530234,1732.80817302774,4.939066039798932,0.9662994748883101,0.7644337983354259,Web/CloudOps,2 -bitbrains_rnd/5T/short,sundial_base_128m,1401344.1824515034,1540852.046103743,126.1733071134422,1.7146683452862685,1.251046626626142,0.6860893218903396,54.78645586026391,1183.7838411008588,4.843701719118544,0.5162647464455313,0.43335403099156355,Web/CloudOps,2 -bitbrains_rnd/5T/medium,sundial_base_128m,2428639.0456661023,2380889.5423860047,192.43698043853752,4.562192427864367,2.3730152582257915,0.7679559803279247,162.52809085259724,1558.4091393681258,6.557020809910089,0.8096803679188941,0.7302818091340633,Web/CloudOps,2 -bitbrains_rnd/5T/long,sundial_base_128m,2565960.3198869433,2498272.9553300245,213.09548285627994,3.5224942814874347,3.1000865366677197,0.7565202479385937,118.99906619854494,1601.8615170753505,6.136370845058673,0.8163208206664636,0.7151753520857796,Web/CloudOps,2 -bitbrains_rnd/H/short,sundial_base_128m,1869378.5397041899,1716389.989173522,185.33467243510506,5.979679667159194,3.1037654265433465,0.628305893623993,215.97912400831697,1367.252185847289,5.983764780277694,0.8111152404515893,0.7254302314523267,Web/CloudOps,2 -bizitobs_application/10S/short,sundial_base_128m,774527.2177777778,768104.0355555556,471.55090277777776,1.4287244579270495,0.020714524586995443,0.020905992719862196,35.81248179934669,880.0722798598862,0.033940870084758545,0.018185833477313076,0.01629924027964623,Web/CloudOps,2 -bizitobs_application/10S/medium,sundial_base_128m,5238368.0,5198286.08,1243.2520833333333,2.856535846734542,0.04106525739034017,0.041892293294270834,78.73858005059354,2288.7481294367017,0.08987240285474259,0.04881885020298451,0.045652348909626786,Web/CloudOps,2 -bizitobs_application/10S/long,sundial_base_128m,7611294.72,7586994.062222222,1659.1941666666667,3.705243579739328,0.053602634006076386,0.05475409613715278,116.51264873478618,2758.8575026630137,0.10639810640911743,0.06398848701971185,0.06144815995612539,Web/CloudOps,2 -bizitobs_l2c/5T/short,sundial_base_128m,13.797112165178572,13.8958984375,2.3654639834449407,0.24810980377841865,0.14426095549885387,0.18704002017066593,2.571659952682548,3.7144464143635956,0.12756912481367788,0.08123960786712507,0.06698296806393049,Web/CloudOps,7 -bizitobs_l2c/5T/medium,sundial_base_128m,66.03551232993198,65.80960884353742,5.367345610119048,0.5301663187479796,0.38707447388300453,0.6539383868781887,6.148770555055706,8.126223743531307,0.4275588702051335,0.2824013093276875,0.23410610413208455,Web/CloudOps,7 -bizitobs_l2c/5T/long,sundial_base_128m,100.55199404761905,100.30006944444445,6.6023542906746036,0.6350142264434739,0.5411881561194837,0.7743142361111112,8.052863430563646,10.027561719960593,0.5630325993440862,0.3707123228839833,0.3100932988678433,Web/CloudOps,7 -bizitobs_l2c/H/short,sundial_base_128m,58.68516322544643,58.96220858134921,4.811042906746032,0.47643937627355415,0.43927799654807403,0.6333610292464967,6.710387912500496,7.660624205993035,0.41292726390068957,0.25932753396744096,0.22284932806442695,Web/CloudOps,7 -bizitobs_l2c/H/medium,sundial_base_128m,80.02613467261905,81.70883556547619,5.414339192708334,0.550211607426368,0.5420053790474771,0.7827382405598958,7.306465746157197,8.945732763313414,0.5416777066064108,0.3278464620274622,0.2763385372212042,Web/CloudOps,7 -bizitobs_l2c/H/long,sundial_base_128m,97.68327132936508,99.13186383928571,6.222073800223214,0.6649538160803763,0.7405663072226014,0.8124540298704117,10.485610887419583,9.883484776604105,0.6037117756486757,0.3800622257284957,0.3248376979085573,Web/CloudOps,7 -bizitobs_service/10S/short,sundial_base_128m,9121.87343915344,8756.308994708994,24.69885582010582,0.8388123634083924,0.08373284556878306,0.060550466063161376,17.905196376640344,95.50849930322138,0.0707557496138025,0.018297701994107993,0.016323560876514015,Web/CloudOps,2 -bizitobs_service/10S/medium,sundial_base_128m,97587.21015873016,98052.56126984127,62.335679563492064,1.2720799283158783,0.11873136005704366,0.08927653963603671,25.27081604263379,312.3895167234812,0.2351117117793331,0.046915301386613345,0.043992180562493655,Web/CloudOps,2 -bizitobs_service/10S/long,sundial_base_128m,131941.80402116402,132003.10857142857,80.13044312169312,1.4572878609645141,0.14303743231233465,0.09928294994212963,33.667228398243935,363.2379440823384,0.2690982814013858,0.0593631939704274,0.05684489042419871,Web/CloudOps,2 -car_parts/M/short,sundial_base_128m,1.3770329361499123,1.3562324800639574,0.551516586999043,0.9567785611036184,0.70865921627573,1.8251849331988177,22.831523029638067,1.1734704666713656,2.8138618039615704,1.3224802008097474,1.1888484578014162,Sales,1 -covid_deaths/D/short,sundial_base_128m,4750637.297243108,4477442.309774436,402.08527568922307,60.37512053436367,0.18997728960451823,0.49796751399984335,1672.7840554639606,2179.5956728813508,0.8200834790539754,0.15128654175005096,0.13106394889424736,Healthcare,1 -electricity/15T/short,sundial_base_128m,124519.82138513513,123738.63013513513,48.25886278856982,0.8949515637448004,0.15680433652720963,0.1562521980904244,15.645124261570613,352.873662073461,0.6955835443448973,0.09512773106183055,0.08361455153603606,Energy,1 -electricity/15T/medium,sundial_base_128m,272671.48163963965,271096.7338378378,55.858921030405405,0.8541111431489306,0.13563336248916688,0.13590715782921595,12.673007115851096,522.179549235356,0.8997679459171861,0.09625054583289519,0.08223440436805927,Energy,1 -electricity/15T/long,sundial_base_128m,351951.0873393393,350408.16377777775,61.53619725975976,0.9062450146418705,0.1447556375475504,0.1396683082007789,12.487065442346577,593.2546563992055,0.9359319264724014,0.09708089271255115,0.0822060533974962,Energy,1 -electricity/D/short,sundial_base_128m,1541255668.3762162,1539757168.2525406,4528.206414414414,1.4558630961996544,0.36211926647797205,0.0978611301731419,20.78940248735936,39258.82917734832,0.6451529423256734,0.07441346960502421,0.0637861337580781,Energy,1 -electricity/H/short,sundial_base_128m,1351264.2378378378,1379962.9265765767,167.6371364724099,0.9319139633483812,0.19978903100319842,0.125901819177576,18.20285203101193,1162.438917895404,0.548754183518862,0.07913670003315797,0.0693888009177783,Energy,1 -electricity/H/medium,sundial_base_128m,6083808.971531532,6075464.317837838,234.6254349662162,0.9934519730818583,0.23161005280010538,0.12755011756140908,17.08206870586526,2466.537851226194,0.962771006297108,0.09158203917812302,0.07987482442231447,Energy,1 -electricity/H/long,sundial_base_128m,9736594.810234234,9653101.306618618,268.749015015015,1.0753938178294977,0.2753369705480924,0.13579960820195194,18.99597491045824,3120.351712585335,1.230665792525617,0.10599453203302554,0.09298005290378641,Energy,1 -electricity/W/short,sundial_base_128m,156600593037.14594,165368612203.47388,38040.983783783784,1.6144820919972853,0.2493729220592092,0.10377194859960058,23.430973726850713,395727.9280479784,0.9029205411614254,0.08679697142882138,0.07201883556331198,Energy,1 -ett1/15T/short,sundial_base_128m,4.865275355747768,5.149080403645834,1.1326773507254464,0.7097059923931616,0.5011566507454214,0.24652840750558036,8.353473729986172,2.2057369189791807,0.416976091831895,0.21412316715929222,0.177346765109939,Energy,7 -ett1/15T/medium,sundial_base_128m,9.479339037698413,9.522857142857143,1.6631180245535715,1.0667045190646924,0.7683557793146981,0.41591959635416664,11.184306913045495,3.0788535265092447,0.5868325263463925,0.3169919398755988,0.25999396052771984,Energy,7 -ett1/15T/long,sundial_base_128m,8.588688616071428,8.424805307539682,1.6301932973710318,1.0882545863507582,0.8187500465963538,0.42221966455853177,11.252389295554746,2.930646450200267,0.5585841110632471,0.3107164543205254,0.2526629588207522,Energy,7 -ett1/D/short,sundial_base_128m,68630.59047619048,68294.18412698412,166.50721726190477,1.9033811160400669,1.3687090192522322,0.4968551393539187,24.00443651496063,261.97440805580703,0.6880844725384713,0.43733672923921607,0.37294448291981835,Energy,7 -ett1/H/short,sundial_base_128m,95.23248697916667,95.23712797619048,4.891598656063988,0.8294853844046632,0.46782146635509675,0.26254708426339285,10.191291541613097,9.758713387489493,0.4554790783442203,0.2283109216374914,0.19024286697535567,Energy,7 -ett1/H/medium,sundial_base_128m,130.83780691964284,130.98899739583334,6.7376825241815474,1.287646484989819,3319469932639.093,0.45304470970517113,18.002215579451374,11.438435510140486,0.5478739462199896,0.32271902128820656,0.2691033786548276,Energy,7 -ett1/H/long,sundial_base_128m,138.46879133597884,138.9707671957672,7.078614831349206,1.4054141627728294,2813768040816.4478,0.4885259396184689,19.88461175593543,11.767276292157792,0.5635093754804101,0.3389795330579862,0.2827083326570119,Energy,7 -ett1/W/short,sundial_base_128m,3683896.8571428573,3667887.4285714286,1210.601841517857,1.8430188542279329,0.7876313073294503,0.569650377546038,19.434605080896997,1919.3480291866967,0.7637543913302695,0.481727367080771,0.40394643549239956,Energy,7 -ett2/15T/short,sundial_base_128m,7.541792224702381,7.563260904947916,1.7109636579241072,0.7473446713848775,0.10887933783997097,0.12903707595098587,9.339846044863958,2.746232369028954,0.1303734201583106,0.08122553151939337,0.06865643859780353,Energy,7 -ett2/15T/medium,sundial_base_128m,11.871760912698413,11.802404513888888,2.167898685515873,0.9071844187960826,0.13902399172318064,0.16579804920014882,10.156929834549702,3.4455421797880246,0.18361917964455493,0.11553124513234761,0.09575432167607961,Energy,7 -ett2/15T/long,sundial_base_128m,11.858053075396825,11.674272073412698,2.192200210813492,0.9212435305526607,0.14248933317189483,0.16945643833705357,10.670394862380515,3.443552391847237,0.18351314024341298,0.11682631740440864,0.09767482003230302,Energy,7 -ett2/D/short,sundial_base_128m,123608.8634920635,121005.04126984127,214.6015625,1.5070957175381101,0.4611456976996528,0.15564621213882687,16.407948600654866,351.5805220601157,0.20121880891418592,0.1228221362899896,0.10264152652572658,Energy,7 -ett2/H/short,sundial_base_128m,116.91236979166666,113.46793154761905,6.692149135044643,0.7705117287530697,0.12544639634027052,0.11536166781470888,10.325377422224458,10.812602359823774,0.13713388294356296,0.08487507129051912,0.07206166053668694,Energy,7 -ett2/H/medium,sundial_base_128m,249.36175595238095,247.17736235119048,9.998749069940477,1.1150264396546699,0.18467223795585141,0.19034965151832217,12.937642203068034,15.791192353726204,0.21279647190226167,0.1347395736723634,0.11361835317008559,Energy,7 -ett2/H/long,sundial_base_128m,265.9783399470899,262.0705687830688,10.183081183862434,1.1394240030514897,0.19260719409961238,0.2007253818411045,12.836588365728401,16.308842385254998,0.22333704425394743,0.1394494593348947,0.1168439160691367,Energy,7 -ett2/W/short,sundial_base_128m,5142077.142857143,4744008.571428572,1413.6238839285713,0.9364579000863359,0.1392535056386675,0.15048468112945557,10.745230062350288,2267.614857698975,0.1902181219063579,0.11858137168660238,0.09822798552528894,Energy,7 -hierarchical_sales/D/short,sundial_base_128m,27.632022529971064,27.589404195948738,2.4153553250310047,0.7895142034181349,0.7173977498846464,0.9863113308701943,13.2086148556627,5.256617023330791,1.6126718892360312,0.7410042652728633,0.6491728654993807,Sales,1 -hierarchical_sales/W/short,sundial_base_128m,488.66101694915255,457.8660288665254,9.61986477092161,0.7509810161145285,0.684453321534199,0.4777563062764831,12.34329558035152,22.10567838699262,1.0148825074383763,0.4416526970588593,0.390210413147802,Sales,1 -hospital/M/short,sundial_base_128m,3398.8683181225556,3304.418948283355,19.89609476857888,0.8373906478027656,0.21264946849280883,0.18976043608332654,11.769844117897033,58.29981404878197,0.21164186303998,0.07222744416506434,0.060909019611888625,Healthcare,1 -jena_weather/10T/short,sundial_base_128m,714.0433531746032,760.1094246031746,6.064236886160714,0.2972308027391687,0.38476208500555925,0.5452390640500993,3.2718569348065434,26.721589645352374,0.16597456243755457,0.03766651150087034,0.03083460709230938,Nature,21 -jena_weather/10T/medium,sundial_base_128m,1463.590909090909,1480.0989898989899,10.606262400793652,0.638583318706897,0.9757086962325224,0.6651314343208874,8.155205430508937,38.25690668481848,0.23478198231038327,0.06509045103614167,0.05419848017485068,Nature,21 -jena_weather/10T/long,sundial_base_128m,1526.9175925925927,1558.02791005291,11.005598958333334,0.6789694776833581,1.0108914967660652,0.6608337079406416,8.533353195159828,39.075792923401984,0.23911147728185272,0.06734514717224567,0.05596186914190761,Nature,21 -jena_weather/D/short,sundial_base_128m,335.6586061507937,334.7327628968254,9.400784350198412,0.9311451482969424,0.7852221390519247,0.4243041023375496,12.1090083589597,18.32098813248875,0.11032229282421843,0.05660808665777112,0.04789903735717624,Nature,21 -jena_weather/H/short,sundial_base_128m,1175.9592731829573,1197.6470342522973,9.223842483813701,0.538848642518242,1.7519873990653612,0.6039663265422671,7.101045333338838,34.292262584772054,0.21020936201829357,0.05654156062425354,0.049646022452524335,Nature,21 -jena_weather/H/medium,sundial_base_128m,1254.0208333333333,1268.9168650793652,11.169922650049603,0.8685233040993711,3.727921892923987,0.6717536078559028,9.29711649459636,35.41215657557915,0.2166932665415237,0.06835073771600188,0.058378717407223495,Nature,21 -jena_weather/H/long,sundial_base_128m,1318.643253968254,1302.2162698412699,12.913236813822751,1.0881605488875692,3.741977550866237,0.6666673125413359,16.21339153163842,36.31312784611447,0.21864526188016492,0.0777519925257821,0.06572756237882615,Nature,21 -kdd_cup_2018/D/short,sundial_base_128m,2643.5459804508278,2724.824522907108,20.464195175876057,1.1735723109796423,0.6069453145779307,0.4520561319215872,15.39534679028357,51.41542551074364,1.1517600905129683,0.4584196873585527,0.39564856635318785,Nature,1 -kdd_cup_2018/H/short,sundial_base_128m,4073.6266219234467,4104.7201669204405,19.309099624056028,0.8005728901664165,0.7581389280063812,0.43962308808986017,12.435813682322742,63.82496864020731,1.3359721898774635,0.40417442670017184,0.35148524836753353,Nature,1 -kdd_cup_2018/H/medium,sundial_base_128m,4404.977002365145,4450.69979332406,20.754435195464538,0.8408122037406998,0.8275329345817771,0.46384470316113263,12.144618647993806,66.3700007711703,1.3892443178118112,0.4344279166743208,0.377101024069857,Nature,1 -kdd_cup_2018/H/long,sundial_base_128m,3238.069802731411,3268.87309073652,18.44123979822007,0.7747499424036353,0.7683727548419232,0.4688291406922856,11.506386680401688,56.90404030234946,1.3351987287102756,0.43270600476866583,0.3747144138264249,Nature,1 -m4_daily/D/short,sundial_base_128m,558540.8372030146,569714.9005373619,198.68158458379804,3.7149470831998554,0.03920724474206129,0.033353801738769055,56.307273731262825,747.3558972825562,0.11544583260675516,0.030690814161374223,0.02663964652326942,Econ/Fin,1 -m4_hourly/H/short,sundial_base_128m,1133904.4895330113,1233038.6344605475,210.26280696457326,0.8692680665321981,0.10755592383048385,0.0946229698193438,11.960330569851838,1064.849514970548,0.1453754629157268,0.028705514222148454,0.022980994079973973,Econ/Fin,1 -m4_monthly/M/short,sundial_base_128m,2240086.612148148,2205808.108740741,651.5672013888889,1.0992059769425646,0.19644105455610486,0.15443810484144424,16.191359550291462,1496.6918895177284,0.3110639567967747,0.1354180330651334,0.11625780995502473,Econ/Fin,1 -m4_quarterly/Q/short,sundial_base_128m,2086986.472,2065333.488,646.3814778645833,1.4574114966025733,0.1290917205810547,0.11715818134943644,22.36748309124086,1444.6406030566911,0.24180739765731493,0.10819287698657507,0.0930524818903939,Econ/Fin,1 -m4_weekly/W/short,sundial_base_128m,348123.5226055282,332966.698950075,291.571110992072,2.39610125632833,0.0751076570179418,0.07227586509585253,25.402798188491037,590.0199340747126,0.10749248824941905,0.05311973784637697,0.04345332274527163,Econ/Fin,1 -m4_yearly/A/short,sundial_base_128m,4997100.187690433,4880675.977133571,1120.7643876411016,4.329944223633436,0.2032089580984097,0.19126191826505415,87.54221907098058,2235.4194657134112,0.35845987326478984,0.17971976468644776,0.16001316958189987,Econ/Fin,1 -restaurant/D/short,sundial_base_128m,142.67255066778742,143.40200336228637,7.351215910152237,0.7043214521479607,0.7502271075581395,0.4040559494518773,9.366935736069946,11.944561551927615,0.5497163978907575,0.3383199887813349,0.2856917953281462,Sales,1 -saugeen/D/short,sundial_base_128m,1033.1979166666667,1043.4395833333333,12.527005208333334,2.783229391019545,0.3125386047363281,0.3236430867513021,75.86281309813374,32.14339615950167,1.0411322602659936,0.40575268345005383,0.37915594576518236,Nature,1 -saugeen/M/short,sundial_base_128m,376.7884579613095,385.25225539434524,12.846944173177084,0.7528023615422877,0.4215218226114909,0.3763252894083659,10.33956776215654,19.411039589916598,0.5827776410626605,0.38570380455032605,0.3319812578629679,Nature,1 -saugeen/W/short,sundial_base_128m,1053.62548828125,1092.30908203125,14.770535278320313,1.1982786164986492,0.3316983222961426,0.371807599067688,27.377672970743134,32.45959778372569,0.9798199500794474,0.44586088945028773,0.40576197921446977,Nature,1 -solar/10T/short,sundial_base_128m,17.38408416970803,18.219959892031632,1.918618128421533,0.8373387888683936,4.711767718096245,1.5110303447194342,7.535477779227135,4.1694225223294445,1.2128845602252192,0.558125805784385,0.4444252264450095,Energy,1 -solar/10T/medium,sundial_base_128m,18.074612226277374,18.58121198296837,2.1586906070283125,0.9408304151961179,3.513908973666567,1.4718986742424243,6.205610561348746,4.251424728991139,0.9434079950279075,0.47902200021907093,0.37262989784620915,Energy,1 -solar/10T/long,sundial_base_128m,18.170098337388485,18.24339834245742,2.1740717904121047,0.9494114377480228,3.4878601122249973,1.4685918333079886,6.2731816205260404,4.262639832004163,0.9213013122809925,0.4698907889100312,0.3648130201629169,Energy,1 -solar/D/short,sundial_base_128m,122735.82871046229,125002.05936739659,276.988899026764,1.0812809630437707,0.9917641447118309,0.47106722718027677,9.725230133388655,350.3367361703056,0.5061172575408385,0.40015461546268866,0.3237486423313621,Energy,1 -solar/H/short,sundial_base_128m,542.3035359841209,549.8866372134717,10.220167384988475,0.7869252373242515,3.7249074247143055,1.3971023706120342,13.77821861863638,23.28741153464938,0.8583817689046194,0.3767187841110908,0.3286122560749378,Energy,1 -solar/H/medium,sundial_base_128m,500.6115875912409,502.3263077858881,10.058445483576643,0.766729353250346,3.6976240779496226,1.3902708475897203,12.006373891289906,22.374351109948215,0.8059763489713974,0.3623286828450824,0.30919993527455886,Energy,1 -solar/H/long,sundial_base_128m,427.0271289537713,427.0316707218167,9.861896162814274,0.7493449101098908,4.299106481595773,1.3845172953025917,11.844666051847392,20.66463474039092,0.7164805466759955,0.3419299127598036,0.29303772239399983,Energy,1 -solar/W/short,sundial_base_128m,1235502.9489051094,1256159.5328467153,902.0643818430657,0.9808747227904373,0.19737641828773667,0.17860336025265883,6.82843081120576,1111.5318029211353,0.22689143064276918,0.1841338930567514,0.14782531849850927,Energy,1 -temperature_rain/D/short,sundial_base_128m,165.39650524032874,170.26629099021108,6.104377582519322,1.429880420844037,51.386792891286774,1.4162385989179627,27.010122751001916,12.860657263154506,1.5140393681354736,0.7186466281452656,0.6199938727217345,Nature,1 -us_births/D/short,sundial_base_128m,157040.92,157050.24,264.0079166666667,0.3886940244462994,0.02511056900024414,0.02492879867553711,6.858113080172684,396.2838881408125,0.03714839885492529,0.024748599886822296,0.02157825531986882,Healthcare,1 -us_births/M/short,sundial_base_128m,150176117.33333334,150676010.66666666,10277.453125,1.158132553474985,0.031759614745775856,0.032484558721383415,22.12468864982641,12254.636564718407,0.0380628176189588,0.03192169933545982,0.02829970829691161,Healthcare,1 -us_births/W/short,sundial_base_128m,3689235.4285714286,3637910.285714286,1498.2752511160713,1.362387484772567,0.020157682044165477,0.020313912204333713,19.911947005998787,1920.7382509263016,0.026073833419438506,0.02033893962138144,0.017341925093980613,Healthcare,1 diff --git a/results/sundial_base_128m/config.json b/results/sundial_base_128m/config.json deleted file mode 100644 index a570ae731c3817115d6321382235d03ae9e407d4..0000000000000000000000000000000000000000 --- a/results/sundial_base_128m/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "model": "sundial_base_128m", - "model_type": "zero-shot", - "model_dtype": "float32", - "model_link": "https://huggingface.co/thuml/sundial-base-128m", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/sundial.ipynb", - "org": "Tsinghua University", - "testdata_leakage": "No", - "replication_code_available": "Yes" -} \ No newline at end of file diff --git a/results/tabpfn_ts/all_results.csv b/results/tabpfn_ts/all_results.csv index 1ccae012ee7cec23c3a77da80b0b5c0c164ace66..064c978c187a8586ab3d53f89eedc4440852690f 100644 --- a/results/tabpfn_ts/all_results.csv +++ b/results/tabpfn_ts/all_results.csv @@ -1,98 +1,98 @@ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/5T/long,TabPFN-TS,5094671.185438427,5094671.185438427,483.1624726930903,1.1526641321691184,6.84183501453906,0.8053138505812252,70.21928659273716,2257.1378304034574,5.964941308843822,1.2768541439646968,0.8850161893008147,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,TabPFN-TS,4622063.369807966,4622063.369807966,441.63762099470586,1.307633919865287,6.807939859955235,0.805070973047274,98.99555171639452,2149.8984556969117,6.531985259967827,1.3418170625410066,0.9486319340358308,Web/CloudOps,2 -bitbrains_fast_storage/5T/short,TabPFN-TS,2246919.058831472,2246919.058831472,263.97642698274865,0.9980513501588254,4.24383417854469,0.7465228041656109,83.52859027541167,1498.9726678066788,4.706403428343518,0.8288206900872653,0.6618188415356218,Web/CloudOps,2 -bitbrains_fast_storage/H/short,TabPFN-TS,3140360.326601572,3140360.326601572,318.8959748516379,1.1840948063141727,4.487001069476268,0.5588612078779172,23.623995660482628,1772.106183782894,5.05107832465668,0.9089571274757822,0.6698861614157302,Web/CloudOps,2 -bitbrains_rnd/5T/long,TabPFN-TS,2963661.558287043,2963661.558287043,265.86029517843235,3.874637617567236,4.921695993468071,0.7496938695174797,253.21231299800388,1721.5288432922182,6.594789314139544,1.0184509196742515,0.8191243201309302,Web/CloudOps,2 -bitbrains_rnd/5T/medium,TabPFN-TS,2541585.941135939,2541585.941135939,232.50274768348632,4.831154957418062,4.022562604208175,0.7654880519867127,388.8164612185016,1594.2352213948666,6.707760153017819,0.9782575654136576,0.8190933026377596,Web/CloudOps,2 -bitbrains_rnd/5T/short,TabPFN-TS,1957817.1753560656,1957817.1753560656,171.3280490517959,2.0308800366527606,2.408422210268895,0.6986985656768337,300.9469489327098,1399.22020259717,5.725205164718139,0.7010249205033943,0.607620615332135,Web/CloudOps,2 -bitbrains_rnd/H/short,TabPFN-TS,2116447.4668147573,2116447.4668147573,202.59939406173635,6.680151206331231,3.3270626716840086,0.6266154306757415,212.599432283774,1454.8015214505233,6.366923246390955,0.8866740739109548,0.7424817493089628,Web/CloudOps,2 -bizitobs_application/10S/long,TabPFN-TS,5377315.84,5377315.84,1395.8348611111112,3.094252917003883,0.044311006334092885,0.04513220893012153,52.85873738124467,2318.904016987335,0.08943085901094665,0.05383177128166312,0.04856447852707176,Web/CloudOps,2 -bizitobs_application/10S/medium,TabPFN-TS,4487442.773333333,4487442.773333333,1172.7091666666668,2.490061805199081,0.03584209442138672,0.03663747787475586,36.39132929151046,2118.3585091606506,0.08318170395311869,0.0460488374856937,0.04078035676061784,Web/CloudOps,2 -bizitobs_application/10S/short,TabPFN-TS,930296.4622222222,930296.4622222222,488.9087152777778,1.263384822147888,0.018171200222439236,0.018381890190972226,8.707376745126052,964.5187723534584,0.03719763375795841,0.018855254924278652,0.014849729690675231,Web/CloudOps,2 -bizitobs_l2c/5T/long,TabPFN-TS,99.68172619047618,99.68172619047618,6.976498635912699,0.6649990805871352,0.6381057311584813,0.8647107514880953,6.210544167714041,9.984073627056052,0.5605908079423119,0.3917199685223014,0.3059204801306481,Web/CloudOps,7 -bizitobs_l2c/5T/medium,TabPFN-TS,75.91723533163265,75.91723533163265,6.2581818664965985,0.6375318293573421,0.4760612674560484,0.7679077978847789,5.821976683337321,8.713049714745845,0.4584345459406488,0.32927239672763153,0.2612696761653268,Web/CloudOps,7 -bizitobs_l2c/5T/short,TabPFN-TS,23.84475795200893,23.84475795200893,2.981525384812128,0.30632387126659066,0.1701873925698132,0.21864854494730632,3.0561285239501843,4.883109455255834,0.16770573733284233,0.1023976459299554,0.08410266668617247,Web/CloudOps,7 -bizitobs_l2c/H/long,TabPFN-TS,98.97005208333331,98.97005208333331,6.150545247395834,0.6640742576152979,0.8472777875958788,0.846485343812004,4.987145840575011,9.94836931779944,0.607675110693205,0.3756930552776827,0.29232377484739197,Web/CloudOps,7 -bizitobs_l2c/H/medium,TabPFN-TS,78.76822916666667,78.76822916666667,4.93990246000744,0.4894740456225888,0.4643175915493395,0.7718949817475819,3.567479099723273,8.875146712402374,0.5374036140096928,0.2991185972344085,0.2373706201778537,Web/CloudOps,7 -bizitobs_l2c/H/short,TabPFN-TS,59.09327721974206,59.09327721974206,4.940431625124008,0.4850944799297807,0.5703324791338804,0.6686241513206845,3.649819219429149,7.687215179747609,0.41436058548672183,0.26630191725824864,0.2095964366848913,Web/CloudOps,7 -bizitobs_service/10S/long,TabPFN-TS,130439.41409575974,130439.41409575974,75.71636338076421,1.3653844836902598,0.0781809611458441,0.07604072143228141,22.041272650206317,361.1639712038837,0.26756181598978024,0.05609310258376492,0.05160131865096251,Web/CloudOps,2 -bizitobs_service/10S/medium,TabPFN-TS,96964.0729103637,96964.0729103637,60.81208243819929,1.2255220534460762,0.0723566747968883,0.07036805625353214,18.230412141337187,311.39054723989886,0.23435986380516352,0.04576860628638456,0.04089325236933028,Web/CloudOps,2 -bizitobs_service/10S/short,TabPFN-TS,24200.27255323497,24200.27255323497,33.68946281559726,0.8831190081752103,0.059359659958974006,0.05050596377240031,10.002682447876992,155.56436787784975,0.11524706193264458,0.024958232148864053,0.01928787319334337,Web/CloudOps,2 -car_parts/M/short,TabPFN-TS,1.4236728433498649,1.4236728433498649,0.46870565286262467,0.8476360552543757,0.8072569896537893,1.8997551115703168,20.796016600345137,1.1931776243920538,2.8611175466068777,1.123908075532646,0.9696277332778144,Sales,1 -covid_deaths/D/short,TabPFN-TS,582148.6825669118,582148.6825669118,131.52239363470866,39.24224311247435,0.10336535201007536,0.1573858522759746,693.2680572368323,762.9866857074976,0.28707744008204383,0.04948593833338505,0.0410740544263699,Healthcare,1 -electricity/15T/long,TabPFN-TS,424692.1608811323,424692.1608811323,63.71647982085639,0.9447757940923152,0.1490332890817207,0.13456292517572793,8.252942553633748,651.684095924653,1.028111539984608,0.10052055681683048,0.0806289122434531,Energy,1 -electricity/15T/medium,TabPFN-TS,368464.5532976957,368464.5532976957,58.99590192588996,0.8891286550107368,0.13686671560267977,0.13146080236911156,8.060963797324014,607.0128114773985,1.045944184165752,0.10165587833115326,0.08271194794165343,Energy,1 -electricity/15T/short,TabPFN-TS,180195.4665631592,180195.4665631592,59.99907240548513,1.1509364225225842,0.1950904271022857,0.174021023029497,12.587366719604278,424.49436576138345,0.8367620891213996,0.11827000125504854,0.09677125876773451,Energy,1 -electricity/D/short,TabPFN-TS,1609175176.344216,1609175176.344216,4549.606968468468,1.4917032121138645,0.23848364907625683,0.0954992348610818,15.13704799318191,40114.52575245302,0.6592149100615675,0.07476515532159875,0.06272141383204029,Energy,1 -electricity/H/long,TabPFN-TS,18510127.710963145,18510127.710963145,340.8466862517592,1.3126972825030705,0.3020939283528691,0.14370380876468228,11.204222553381076,4302.339794921264,1.6968416119329297,0.13442983773716952,0.1082691929278382,Energy,1 -electricity/H/medium,TabPFN-TS,10527303.475523552,10527303.475523552,277.4008767648814,1.1674595528269889,0.2434614956055417,0.13190289188954438,10.06489280908614,3244.580631687792,1.2664667356915,0.10827870309132294,0.08766377053562428,Energy,1 -electricity/H/short,TabPFN-TS,1960999.6209848511,1960999.6209848511,185.3523408889556,1.0364160710610155,0.19704922343224884,0.12920845526522548,11.457480338062892,1400.3569619867826,0.6610684965538965,0.0874995423669438,0.07155646763061667,Energy,1 -electricity/W/short,TabPFN-TS,55052578878.05669,55052578878.05669,30331.729800421108,1.547049206297067,0.141604463996508,0.09870668188753938,12.15276333611517,234632.85975765777,0.5353547750270788,0.06920700024821061,0.054579121515172774,Energy,1 -ett1/15T/long,TabPFN-TS,10.810811011904764,10.810811011904764,1.7222476438492065,1.1184637386647829,0.8913237899858845,0.418278304811508,9.161866535766084,3.2879797766873144,0.6266921964023711,0.32826210377732207,0.2594954561050949,Energy,7 -ett1/15T/medium,TabPFN-TS,10.776283191499257,10.776283191499257,1.7095601302858383,1.091773313351892,0.8240003332318827,0.4108899537343827,8.169808213732072,3.2827249643397263,0.6256906424593214,0.32584386077452987,0.2525006817913073,Energy,7 -ett1/15T/short,TabPFN-TS,5.212318674723307,5.212318674723307,1.1454845610119049,0.741309439205773,0.5129581426423955,0.25125504902430945,5.135743695739446,2.283050300524127,0.43159155098213625,0.21654426895444345,0.16685967571622629,Energy,7 -ett1/D/short,TabPFN-TS,44357.80952380953,44357.80952380953,140.70858134920636,1.6395479314177053,1.6393998403397818,0.46450670030381946,14.904811845730327,210.61293769331817,0.5531818669540168,0.3695757562650177,0.29776790612808235,Energy,7 -ett1/H/long,TabPFN-TS,178.98387896825398,178.98387896825398,7.994620639054233,1.4725737522845348,4706846393031.971,0.5004761710999504,11.333256034107908,13.378485675451238,0.6406667032090834,0.3828450672581384,0.29497878110262604,Energy,7 -ett1/H/medium,TabPFN-TS,163.67566964285714,163.67566964285714,7.655423990885416,1.4045375913078184,3339463150815.331,0.47588424682617186,10.659145898074607,12.793579235024776,0.6127821182850777,0.36667666204484833,0.283164148519388,Energy,7 -ett1/H/short,TabPFN-TS,109.13167782738095,109.13167782738095,5.290147472563244,0.8871397861227843,0.49000859487624393,0.2719866979689825,6.356439133275721,10.446610829708408,0.4875860398201332,0.24691281203624557,0.19394053714174952,Energy,7 -ett1/W/short,TabPFN-TS,1451985.142857143,1451985.142857143,929.0714285714286,1.6585971126590675,0.6431011472429548,0.5415669168744769,11.965506490709258,1204.9834616529567,0.4794916796344471,0.3696996962721728,0.2840047608954231,Energy,7 -ett2/15T/long,TabPFN-TS,13.555186011904762,13.555186011904762,2.363949497767857,0.9772597904462405,0.1481694417712973,0.17701786101810515,7.784936272726835,3.681736820021872,0.1962064195659214,0.12597914779496214,0.1011020985320766,Energy,7 -ett2/15T/medium,TabPFN-TS,14.45674851190476,14.45674851190476,2.3854561554439484,0.9810684377348272,0.14774220428808144,0.1766355514526367,7.469608562271954,3.8022031129208185,0.20262627298353544,0.1271252681111848,0.10034116464990252,Energy,7 -ett2/15T/short,TabPFN-TS,9.42276349748884,9.42276349748884,1.9720010303315665,0.8412907225001276,0.1146630124965249,0.1384262584504627,5.529071809687602,3.0696520156996363,0.14572731340126654,0.09361791196688794,0.07284079213293028,Energy,7 -ett2/D/short,TabPFN-TS,124689.80317460318,124689.80317460318,218.28549107142857,1.4298486930788783,0.4788421630859375,0.14597940596323164,27.91695960351564,353.1144335404646,0.20209670692527887,0.12493054580859496,0.1258210568162837,Energy,7 -ett2/H/long,TabPFN-TS,349.1736111111111,349.1736111111111,12.713215112433865,1.4446015639472831,0.24456672487448208,0.2568363734654018,11.838650179260124,18.6861877094048,0.2558929587280855,0.1740976961517922,0.13917833955228112,Energy,7 -ett2/H/medium,TabPFN-TS,260.9357886904762,260.9357886904762,10.916395786830355,1.2487580741246136,0.21442902452607412,0.2176322028750465,10.592664800499469,16.153507009020554,0.21767889487818903,0.1471054532989758,0.12063006509927268,Energy,7 -ett2/H/short,TabPFN-TS,126.3329613095238,126.3329613095238,7.186092558361235,0.8255326354728952,0.1333656464555541,0.12338819957914808,6.245625044327679,11.239793650664758,0.14255185712990415,0.09113964824800784,0.07305442987134397,Energy,7 -ett2/W/short,TabPFN-TS,3352975.4285714286,3352975.4285714286,1215.9320591517858,0.7651817774023886,0.12514420918055943,0.12832092387335642,13.173712071216404,1831.1131665114065,0.15360232199450594,0.1019980583882235,0.09869280067398715,Energy,7 -hierarchical_sales/D/short,TabPFN-TS,31.00775791907813,31.00775791907813,2.370928715378255,0.7595246942482423,0.6425987223410554,1.0572101569362986,8.035389477271963,5.56846100094794,1.7083421681052473,0.7273746734264227,0.5917485761396661,Sales,1 -hierarchical_sales/W/short,TabPFN-TS,462.451374635858,462.451374635858,8.994814727265956,0.7311473691538796,0.5269374437620046,0.46027796147233346,6.669578861706579,21.50468262113761,0.9872904978590784,0.41295634381991136,0.34547572723646164,Sales,1 -hospital/M/short,TabPFN-TS,3655.2802314211217,3655.2802314211217,18.772015097036615,0.7644714645492778,0.19299344092231893,0.17395839997863105,5.254385536046241,60.45891358121746,0.2194798957162256,0.06814677393013854,0.05375192910322669,Healthcare,1 -jena_weather/10T/long,TabPFN-TS,1693.3583560681218,1693.3583560681218,10.627256944444444,0.6669669872039639,0.773788063451945,0.6517540018394511,6.516622821458388,41.15043567288349,0.25180656563872267,0.06503000586951763,0.05302131807666021,Nature,21 -jena_weather/10T/medium,TabPFN-TS,1750.3887817911254,1750.3887817911254,10.549468315972222,0.6257176279534998,0.7843234355751396,0.6484787581802962,6.560058680442422,41.83764789984166,0.25675694414401257,0.06474191029183934,0.053889677251242135,Nature,21 -jena_weather/10T/short,TabPFN-TS,1029.9804992191375,1029.9804992191375,6.5642832619803295,0.3103315819396469,0.3655124845059705,0.5538089704418939,2.7476479117092367,32.09330925939451,0.1993396907652391,0.040772429697493937,0.0339592102375841,Nature,21 -jena_weather/D/short,TabPFN-TS,397.21959325396824,397.21959325396824,10.065989273313493,1.228557952722909,0.6865152271871034,0.48159678625682045,7.6517322873356175,19.93036861811563,0.1200133937581282,0.0606137075219578,0.047143761671973934,Nature,21 -jena_weather/H/long,TabPFN-TS,3960.9719576719576,3960.9719576719576,18.149107142857144,1.406140600615272,4.157182337832393,0.79033203125,11.165177368080114,62.93625312704878,0.3789459725153458,0.1092777328617197,0.10253437348096008,Nature,21 -jena_weather/H/medium,TabPFN-TS,1468.8757936507936,1468.8757936507936,11.595155164930556,1.08814100404732,2.9812427832367323,0.7352610754588293,7.7572016693125905,38.32591543134741,0.2345230737441197,0.07095281089086086,0.05799743420199071,Nature,21 -jena_weather/H/short,TabPFN-TS,1157.0227521929824,1157.0227521929824,8.288363680604506,0.5490473254936928,1.0643030052146252,0.6152072928802311,4.901563186782991,34.01503714819348,0.2085099962811183,0.050807137816434395,0.04210943877353269,Nature,21 -kdd_cup_2018/D/short,TabPFN-TS,2731.1231257064965,2731.1231257064965,20.537550805405942,1.1739298514294911,0.5386272222170274,0.45167684057258417,9.678648517860324,52.26014854271366,1.170682813452448,0.4600629433466736,0.3619470425522957,Nature,1 -kdd_cup_2018/H/long,TabPFN-TS,4164.16667203731,4164.16667203731,25.603662863763393,1.0932322336358322,1.2056718454704942,0.6444264480870014,8.694193144080682,64.5303546560633,1.514142939122559,0.6007654157096626,0.477949904103166,Nature,1 -kdd_cup_2018/H/medium,TabPFN-TS,5238.046655667353,5238.046655667353,27.09359559639831,1.1284777650578837,1.044518417287213,0.6036829312783302,9.260084449300775,72.37435081344324,1.514926251625949,0.5671180295035835,0.4495885779293134,Nature,1 -kdd_cup_2018/H/short,TabPFN-TS,4678.4982992292435,4678.4982992292435,24.941407209880296,1.0505311589202366,1.048782773133547,0.5517051294271736,8.468461581709045,68.39954896948696,1.4317264336898177,0.5220688225844491,0.41801589098488257,Nature,1 -loop_seattle/5T/long,TabPFN-TS,116.89548084365325,116.89548084365325,6.416364829237616,1.0065567656849457,0.22339298842738164,0.14145765512993735,8.568140909437178,10.811821347194618,0.1911552234581402,0.11344264886878824,0.09023827778424431,Transport,1 -loop_seattle/5T/medium,TabPFN-TS,108.92337177099458,108.92337177099458,6.126644162393576,0.9637143945419536,0.21405275193284293,0.13691525135493007,8.13248716007984,10.4366360371048,0.1856891996347391,0.1090055882869805,0.08690720550103873,Transport,1 -loop_seattle/5T/short,TabPFN-TS,48.56773726991825,48.56773726991825,3.801889258543158,0.5976997857380476,0.11060900648804026,0.08033697568226156,5.304121093302596,6.969055694275821,0.1195039192617756,0.06519400718354651,0.05256348721117568,Transport,1 -loop_seattle/D/short,TabPFN-TS,19.406321453334623,19.406321453334623,3.0272181523719683,0.907428724621304,0.05706404576729695,0.05580497363772555,7.002619763110791,4.405260656684757,0.07872789401866613,0.054100433196767675,0.043121840345089806,Transport,1 -loop_seattle/H/long,TabPFN-TS,70.10133339245786,70.10133339245786,4.439102173901359,0.9260622979769342,0.14156941117640004,0.10133589307262104,6.977403521697585,8.37265390377853,0.14963051021595694,0.07933268600555231,0.06256499189469966,Transport,1 -loop_seattle/H/medium,TabPFN-TS,77.16381357230392,77.16381357230392,4.755659628160474,0.9784901403759688,0.1191087661758911,0.10848097274670045,7.622978827805903,8.784293572752674,0.15526140687770348,0.08405575227927105,0.06681332001279122,Transport,1 -loop_seattle/H/short,TabPFN-TS,75.32349253407864,75.32349253407864,4.495341405754054,0.91095777978495,0.1251505313086151,0.10250271914816066,7.682045156903769,8.678910791918456,0.15362554967485056,0.07957211578648182,0.06341097341290575,Transport,1 -m4_daily/D/short,TabPFN-TS,702442.9924972118,702442.9924972118,187.73180634589713,4.194442221931702,0.03708484540331653,0.03092866981898531,30.152829872241675,838.1187221970475,0.12946617938967445,0.028999375713517687,0.02259971410479323,Econ/Fin,1 -m4_hourly/H/short,TabPFN-TS,2216507.236413823,2216507.236413823,285.81631179170716,0.7386401357780685,0.09308361316072768,0.08753064174007102,5.496123608735736,1488.7938864778507,0.20325325838709515,0.039020241283554385,0.02960508205927616,Econ/Fin,1 -m4_monthly/M/short,TabPFN-TS,1909204.3714108795,1909204.3714108795,555.5572942889178,0.9576589316497056,0.15374823170679586,0.1294481470540718,8.502672632199085,1381.739617804628,0.28717292636452374,0.11546387749783972,0.09366665886576336,Econ/Fin,1 -m4_quarterly/Q/short,TabPFN-TS,1901198.214072917,1901198.214072917,580.5208575846355,1.2241822728860832,0.12060847956935564,0.10457869998613992,9.947300963299902,1378.83944463194,0.2307934429356135,0.09716896912073542,0.07826275149269904,Econ/Fin,1 -m4_weekly/W/short,TabPFN-TS,217483.00664238268,217483.00664238268,248.66107108420823,2.04974390202117,0.061852850136471835,0.06312812623718143,18.73303138685925,466.3507335068561,0.0849618698162227,0.04530219002970306,0.0370084480828777,Econ/Fin,1 -m4_yearly/A/short,TabPFN-TS,3555054.903659209,3555054.903659209,893.0699246634783,3.3084560346745997,0.17128991236630944,0.1483512422599227,33.0173202698287,1885.485323108936,0.3023463106417499,0.1432079017310429,0.11767364143536849,Econ/Fin,1 -m_dense/D/short,TabPFN-TS,7652.655555555555,7652.655555555555,42.433567708333335,0.6783292595875375,0.12104778134266124,0.09753263120298032,6.526921713728361,87.47945790615964,0.15143988154244065,0.07345878245000709,0.061106410525425904,Transport,1 -m_dense/H/long,TabPFN-TS,75694.39604938272,75694.39604938272,120.74972222222222,1.02275823938943,0.32039782336527445,0.2554198615933642,8.829765092160066,275.1261457029897,0.4762840292517426,0.20903561922142588,0.16524406701635594,Transport,1 -m_dense/H/medium,TabPFN-TS,71212.77555555555,71212.77555555555,116.21049045138888,0.993734534883084,0.3077546622676556,0.24954473707411023,8.673153554524918,266.85721941809174,0.4642978479599947,0.20219157174991295,0.160040826351846,Transport,1 -m_dense/H/short,TabPFN-TS,68285.90555555555,68285.90555555555,106.8298410373264,0.906291178560046,0.30428158067341887,0.23916954464382595,10.038621879613546,261.3157200697186,0.46350902522521975,0.18948953959205128,0.15486162614965193,Transport,1 -restaurant/D/short,TabPFN-TS,147.50485234907654,147.50485234907654,7.279157133165657,0.6978371816511807,0.6452601482590519,0.3987383605096257,4.8369283043616305,12.145157567898266,0.5589482913220262,0.3350036769059037,0.26285464069074244,Sales,1 -saugeen/D/short,TabPFN-TS,996.2383333333332,996.2383333333332,14.171375325520833,3.1471346641095788,0.48842748006184894,0.41242085774739584,27.22096017470464,31.56324339058541,1.0223409738504536,0.4590142233422909,0.3731311321710199,Nature,1 -saugeen/M/short,TabPFN-TS,376.1626441592262,376.1626441592262,11.992789132254464,0.7027332947267324,0.3269654228573754,0.3455390476045154,4.861786421786346,19.39491284226939,0.5822934677185015,0.360059507780706,0.2762319185074306,Nature,1 -saugeen/W/short,TabPFN-TS,967.20927734375,967.20927734375,16.254945373535158,1.3185971950086537,0.4728340625762939,0.43860502243041993,10.0293112825053,31.099988381730142,0.9387790097305582,0.4906690424989388,0.3952868544502755,Nature,1 -solar/10T/long,TabPFN-TS,24.99329126115166,24.99329126115166,1.9885748127661191,0.8706305500987933,3.3978985345314263,1.4695288541349858,5.422036201988414,4.9993290811019495,1.0805249842097455,0.4297986256368609,0.3311104604624895,Energy,1 -solar/10T/medium,TabPFN-TS,23.20675610173358,23.20675610173358,1.9164985034033883,0.8397775518236055,3.5864761297978434,1.4588056166492134,5.404413331269677,4.817339110103583,1.0689867060696263,0.42527863941397,0.3259753271245175,Energy,1 -solar/10T/short,TabPFN-TS,25.936860660450883,25.936860660450883,2.1414197193155036,0.9439361088822628,4.492088894725105,1.5343012306231942,4.045348817118251,5.092824428590768,1.4815020459581518,0.6229387523376522,0.4579774575911444,Energy,1 -solar/D/short,TabPFN-TS,113549.29051094892,113549.29051094892,253.51575425790753,0.987481598204353,0.9896838705615116,0.4369196314011177,5.250447637876118,336.97075616579684,0.4868079796401177,0.3662439243733971,0.26917233844623834,Energy,1 -solar/H/long,TabPFN-TS,985.2254866180048,985.2254866180048,14.191483868106245,1.0832552757402476,3.761211700292542,1.4626715914563058,4.509945094067163,31.388301747912468,1.088289768604162,0.4920446738090314,0.3505921321640143,Energy,1 -solar/H/medium,TabPFN-TS,806.1380322384429,806.1380322384429,11.32804896593674,0.8642389418245519,4.488747114193476,1.4055882066415757,4.4199894526363,28.39257001820094,1.0227666406383358,0.40806276354872306,0.31347488384144956,Energy,1 -solar/H/short,TabPFN-TS,849.0887552623575,849.0887552623575,11.64371880399294,0.8972739471619915,4.454845891374051,1.4148532292467024,5.817322540440987,29.13912756522332,1.0740779514829115,0.42919135491076416,0.33566385955464595,Energy,1 -solar/W/short,TabPFN-TS,975858.0729927009,975858.0729927009,733.5384352189781,0.7927547220600458,0.1474037866522796,0.14471499241181535,6.555031527149288,987.8552894997832,0.20164595749357925,0.149733530508271,0.12002529687191076,Energy,1 -sz_taxi/15T/long,TabPFN-TS,17.683922053062677,17.683922053062677,2.953506095864494,0.5596229152773028,5905908731598.259,0.42311587428774927,7.194725257547046,4.205225565063387,0.3892281573916414,0.27337124198259016,0.2424540578815739,Transport,1 -sz_taxi/15T/medium,TabPFN-TS,17.31587122729701,17.31587122729701,2.9009017553084937,0.5657806694261462,9231205100342.7,0.4234086648011819,6.352440033271851,4.1612343393874145,0.3872723050167596,0.2699773235003315,0.2304960350862857,Transport,1 -sz_taxi/15T/short,TabPFN-TS,17.14560256976639,17.14560256976639,2.7944552709185886,0.5582249003801146,1157938620500.9626,0.4043481684720851,4.390036050528658,4.14072488457835,0.3871754966467506,0.26129352650416643,0.20867625780444496,Transport,1 -sz_taxi/H/short,TabPFN-TS,7.340465578258547,7.340465578258547,1.8834059462588055,0.5726814850300137,1.4804094673221946,0.3035417825747759,4.395105016649191,2.709329359501821,0.2523907457387066,0.17545088404917308,0.1403426654750027,Transport,1 -temperature_rain/D/short,TabPFN-TS,192.348170645723,192.348170645723,5.973571384244577,1.3788383285690875,32.83221152661815,1.5353158278965455,21.838367970591225,13.868964296072113,1.6327437788569656,0.7032472870339427,0.5690453839767815,Nature,1 -us_births/D/short,TabPFN-TS,112759.78666666668,112759.78666666668,215.285703125,0.31694359235351605,0.020599613189697264,0.02048578898111979,2.2508974742308343,335.79724040954636,0.03147826645095531,0.020181287725250033,0.015855619215830008,Healthcare,1 -us_births/M/short,TabPFN-TS,59338602.666666664,59338602.666666664,6332.360677083333,0.7153333155562595,0.019422224412361785,0.019690368324518204,4.7156280861706295,7703.155370798817,0.0239259480622513,0.01966826908953116,0.015286944232309424,Healthcare,1 -us_births/W/short,TabPFN-TS,1737954.0,1737954.0,983.3291015625,0.8942316441355983,0.013318025640078954,0.013351580926350184,6.532085345899004,1318.3148334142345,0.0178959945969944,0.01334859613394065,0.010610700543554966,Healthcare,1 +bitbrains_fast_storage/5T/long,TabPFN-TS,3940977.473961663,3940977.473961663,396.2051671955408,1.0388353010633071,5.466466474860468,0.7927045072161448,67.23911919744833,1985.189530992359,5.246263156727107,1.0470519508149014,0.7602884074722032,Web/CloudOps,2 +bitbrains_fast_storage/5T/medium,TabPFN-TS,3247133.159062092,3247133.159062092,324.4935908220456,1.1867037716263282,5.815028533985838,0.7800321416939235,95.43680935654676,1801.9803436947063,5.474913948877669,0.9859011464411452,0.7353447476277656,Web/CloudOps,2 +bitbrains_fast_storage/5T/short,TabPFN-TS,2045266.0192445407,2045266.0192445407,190.9517174659572,0.8882362319520551,2.484373134741108,0.6999759874291243,76.38956860526716,1430.127973030575,4.490248114456697,0.5995411637790902,0.4558547292997618,Web/CloudOps,2 +bitbrains_fast_storage/H/short,TabPFN-TS,2405371.011957238,2405371.011957238,272.2178745663353,1.1461108043110697,3.706066647986907,0.5534585983280202,21.973896483964754,1550.925856370071,4.420642537084117,0.775909377434113,0.591312581984499,Web/CloudOps,2 +bitbrains_rnd/5T/long,TabPFN-TS,3143232.871647869,3143232.871647869,241.6739175312724,3.5093995299063607,4.050204866894866,0.7111483674880203,242.29249979536945,1772.916487499586,6.791643806713404,0.9257983536270856,0.7300597264192352,Web/CloudOps,2 +bitbrains_rnd/5T/medium,TabPFN-TS,2090599.1775186085,2090599.1775186085,199.0785499293046,4.589946127888775,2.832675134166589,0.746569956612344,371.1720930175657,1445.890444507677,6.083598065793586,0.8376249292547827,0.7095510579591657,Web/CloudOps,2 +bitbrains_rnd/5T/short,TabPFN-TS,1682744.407120047,1682744.407120047,135.30342227375482,1.8673759793901137,1.2023257691773783,0.6515096053863655,284.946804834687,1297.2063857073963,5.307794073707693,0.5536225467355949,0.4553597018381872,Web/CloudOps,2 +bitbrains_rnd/H/short,TabPFN-TS,1768178.1854349263,1768178.1854349263,167.11547426876976,5.957801473087373,1.9668859182296852,0.552835291775346,203.4573914538545,1329.7286134527324,5.819543006761521,0.7313790797335689,0.6370421458232746,Web/CloudOps,2 +bizitobs_application/10S/long,TabPFN-TS,21300886.186666667,21300886.186666667,2472.8352777777777,7.728085487915151,0.1032213168674045,0.1071828460693359,143.1277644315143,4615.288310243106,0.1779932222914751,0.0953673724588003,0.0879505832371775,Web/CloudOps,2 +bizitobs_application/10S/medium,TabPFN-TS,21666054.826666668,21666054.826666668,2030.06125,6.925782643420624,0.0924721717834472,0.0951119867960612,148.49438643019815,4654.680958633649,0.1827756217009251,0.0797145304602413,0.0700137998283303,Web/CloudOps,2 +bizitobs_application/10S/short,TabPFN-TS,4943498.24,4943498.24,967.4614583333332,3.1888377415438267,0.0462684885660807,0.0465743086073133,49.96181592052012,2223.3979041098337,0.0857475699861046,0.0373111213939501,0.0313406007204244,Web/CloudOps,2 +bizitobs_l2c/5T/long,TabPFN-TS,339.5494444444444,339.5494444444444,13.08852554563492,1.2160180268733982,0.9811188743835858,1.2126699683779762,7.2539512545927,18.42686746152054,1.0346410597480369,0.7349011420065599,0.511339388914521,Web/CloudOps,7 +bizitobs_l2c/5T/medium,TabPFN-TS,144.35393282312924,144.35393282312924,8.755759061437075,0.8700326557683994,0.6392903928649001,0.9488015518707484,6.532302877185926,12.014738150418813,0.632151911088195,0.4606816856447503,0.3448825766018137,Web/CloudOps,7 +bizitobs_l2c/5T/short,TabPFN-TS,29.177198428199404,29.177198428199404,3.351733107793899,0.3376520522696331,0.1869919781934289,0.226708325885591,4.020672322763404,5.401592212320309,0.1855125331592748,0.1151120771172689,0.0989730973449374,Web/CloudOps,7 +bizitobs_l2c/H/long,TabPFN-TS,144.74971478174604,144.74971478174604,8.725088355654762,0.9194449990311916,1.3923079579433175,0.8217161148313492,7.88847966834854,12.031197562244005,0.7349002712763818,0.5329535789191325,0.4400927339795309,Web/CloudOps,7 +bizitobs_l2c/H/medium,TabPFN-TS,114.09707961309525,114.09707961309525,7.254261997767857,0.7482961914782938,1.0562951228521056,0.8080075218563988,7.184395907292426,10.68162345400245,0.6467885245941795,0.4392565825560715,0.3800913288674146,Web/CloudOps,7 +bizitobs_l2c/H/short,TabPFN-TS,85.44098772321429,85.44098772321429,6.418287004743304,0.6344522765963027,0.7020613489677009,0.7020347231910342,6.581317614660395,9.243429435183367,0.4982445193882983,0.3459621070727685,0.2898378847142646,Web/CloudOps,7 +bizitobs_service/10S/long,TabPFN-TS,208415.56110967736,208415.56110967736,131.40309476984555,3.902102389054756,0.2514900850213503,0.1719557895671443,67.5938471682915,456.5255317172057,0.3382087086505102,0.0973476134568513,0.0905342005234128,Web/CloudOps,2 +bizitobs_service/10S/medium,TabPFN-TS,153786.42608814148,153786.42608814148,107.66579814488334,3.777267746897072,0.3485729430163631,0.1520390156208532,70.09273852151425,392.1561246342348,0.2951459406018347,0.081031816840187,0.0670881764641895,Web/CloudOps,2 +bizitobs_service/10S/short,TabPFN-TS,23223.54763853943,23223.54763853943,49.30270014662591,2.003461979848056,0.2334731027970346,0.0912925157320221,33.28855091714436,152.39274142340057,0.112897419560105,0.0365250180022352,0.0308080142329048,Web/CloudOps,2 +car_parts/M/short,TabPFN-TS,1.4217560750180898,1.4217560750180898,0.4650315940521182,0.8429977450172786,0.8077849812660705,1.899363879513,20.097226810718563,1.1923741338263296,2.859190858652686,1.1150980593916195,0.9545496300490686,Sales,1 +covid_deaths/D/short,TabPFN-TS,608643.349030727,608643.349030727,132.21949222148453,37.798524709251794,0.0992231984946835,0.1511093960201699,677.8435674633554,780.1559773729399,0.29353746932196,0.0497482250567645,0.0399575272446472,Healthcare,1 +electricity/15T/long,TabPFN-TS,479385.9336663729,479385.9336663729,69.25283177134297,1.0202781447783311,0.1595782741210479,0.1449488764255062,9.295101062040777,692.3770169975119,1.0923096108172683,0.1092548306241962,0.0888787687381395,Energy,1 +electricity/15T/medium,TabPFN-TS,412523.4689212973,412523.4689212973,65.27677076596724,0.976918207169049,0.1516940762467859,0.1445092451330036,8.89593020035391,642.2798992038419,1.106712926772942,0.1124784476584751,0.0923857961554726,Energy,1 +electricity/15T/short,TabPFN-TS,194473.3980942518,194473.3980942518,65.37218991744707,1.2266456914333352,0.2023727743818738,0.1846499673239442,11.928914973863955,440.9913809750161,0.8692809586938318,0.1288614752463225,0.1036693276639552,Energy,1 +electricity/D/short,TabPFN-TS,1471778340.7486846,1471778340.7486846,4609.131754504505,1.4894429943811094,0.2354444740752776,0.0952267406566723,13.423388704270328,38363.76338093911,0.6304440686362233,0.0757433452848885,0.0600270541999052,Energy,1 +electricity/H/long,TabPFN-TS,19914932.00037977,19914932.00037977,352.74562123382617,1.3377679338170498,0.3050388547866177,0.1455674919289879,11.358791576901645,4462.614928534589,1.7600540798077915,0.1391227743664636,0.1124445719696832,Energy,1 +electricity/H/medium,TabPFN-TS,11796934.140468594,11796934.140468594,286.1099377427443,1.180699742250593,0.2148767347608029,0.1329174415781638,10.214039705176262,3434.666525365832,1.3406635236881663,0.111678136571216,0.0908690358272918,Energy,1 +electricity/H/short,TabPFN-TS,2003726.408484706,2003726.408484706,186.82288506429205,1.044375818547214,0.1971453508145225,0.1307061697499337,11.382491212876442,1415.5304336130346,0.6682314588183454,0.0881937442408192,0.0720731827284665,Energy,1 +electricity/W/short,TabPFN-TS,43809522548.24645,43809522548.24645,28410.723933908583,1.4485545546987582,0.1054109967005592,0.0939584561545392,11.252347284933158,209307.2443759328,0.4775700762465698,0.0648238986461813,0.0514000138896149,Energy,1 +ett1/15T/long,TabPFN-TS,10.510729166666666,10.510729166666666,1.6884466455853175,1.1075089883945437,0.9056581486460964,0.4095517113095238,9.189091726171444,3.2420254728590066,0.6179332606557163,0.3218196000877369,0.2595610162075561,Energy,7 +ett1/15T/medium,TabPFN-TS,9.810387369791666,9.810387369791666,1.6322027781653026,1.0538546298293985,0.7946897001730433,0.3998653339204334,8.562064287161736,3.132153790890809,0.5969916270759061,0.3110994725382204,0.2479122114007532,Energy,7 +ett1/15T/short,TabPFN-TS,6.399552481515067,6.399552481515067,1.2574796222505116,0.786973072399437,0.5675219319419111,0.2630254836309524,5.647841580246456,2.5297336779817488,0.4782249788369692,0.2377159979221376,0.1834630230394784,Energy,7 +ett1/D/short,TabPFN-TS,43568.688888888886,43568.688888888886,141.97459077380952,1.772226457572935,1.6984561011904762,0.474870130750868,13.765077920690596,208.73114019927377,0.5482392634160486,0.3729009720126996,0.2918356857005129,Energy,7 +ett1/H/long,TabPFN-TS,172.94669312169313,172.94669312169313,7.836870659722222,1.457595714011079,2736247275849.4355,0.4894441473420965,11.165649758363148,13.150919858386072,0.6297690691031738,0.3752907624106631,0.2897646542062353,Energy,7 +ett1/H/medium,TabPFN-TS,154.01130952380953,154.01130952380953,7.3551025390625,1.3585784429613703,2668500988477.909,0.4610775538853237,10.45425370159929,12.410129311325065,0.5944157759046996,0.3522919764120212,0.2764819505676313,Energy,7 +ett1/H/short,TabPFN-TS,112.4321765718006,112.4321765718006,5.309052894228981,0.8908363269123161,0.5164939834958031,0.2685810225350516,6.503018582568272,10.603404008704024,0.4949042185542229,0.2477952053627885,0.1937159661760259,Energy,7 +ett1/W/short,TabPFN-TS,1174346.5714285714,1174346.5714285714,849.2448381696429,1.577096937888621,0.6470521518162319,0.5367565836225238,10.057570618317031,1083.6727233941858,0.431219241467038,0.3379347906702841,0.256419494681339,Energy,7 +ett2/15T/long,TabPFN-TS,12.83592013888889,12.83592013888889,2.301945064484127,0.9578073986429992,0.1473062529443106,0.178276619078621,8.571694903641015,3.582725239100661,0.1909299131946711,0.1226748193091084,0.1009458616883307,Energy,7 +ett2/15T/medium,TabPFN-TS,12.95630867125496,12.95630867125496,2.277454136439732,0.9392566587102824,0.141396593654979,0.1718650009518578,8.060023977046217,3.599487278940566,0.1918231799623912,0.1213696454009845,0.0983757090868891,Energy,7 +ett2/15T/short,TabPFN-TS,8.865403820219495,8.865403820219495,1.944286382765997,0.8450960376199323,0.1162656731139621,0.1411288760957263,6.116299433959818,2.9774827993154718,0.1413517124493446,0.0923021989443899,0.0728750638145369,Energy,7 +ett2/D/short,TabPFN-TS,162956.95238095237,162956.95238095237,241.52847222222223,1.535311945472873,0.4684700133308532,0.1532284085712735,27.805417218699205,403.67926919889305,0.2310362964807856,0.1382331171665665,0.1285422202240154,Energy,7 +ett2/H/long,TabPFN-TS,363.742328042328,363.742328042328,12.264203042328042,1.3653142407810173,0.2206563951497026,0.2301058976107804,12.436676967686727,19.072029992696844,0.2611767718316169,0.1679488214369047,0.1356672864670785,Energy,7 +ett2/H/medium,TabPFN-TS,329.2125744047619,329.2125744047619,11.717957124255951,1.277450790102998,0.2048852038421092,0.2153702872140067,10.86438040815514,18.144216004136467,0.244504978764426,0.1579070077855928,0.1280160716008789,Energy,7 +ett2/H/short,TabPFN-TS,117.8634998139881,117.8634998139881,6.839659045991444,0.7870254439803768,0.1262451481190145,0.1175918352036249,6.192682885553366,10.856495742825496,0.1376905731691205,0.0867459073933945,0.0699977966289662,Energy,7 +ett2/W/short,TabPFN-TS,14502475.42857143,14502475.42857143,1635.7318638392858,0.9589009824999204,0.1474132367542811,0.145101819719587,13.189927237151512,3808.2115787560215,0.319450568015805,0.1372128260782456,0.1202574726141917,Energy,7 +hierarchical_sales/D/short,TabPFN-TS,30.57163511846321,30.57163511846321,2.3798296243605312,0.7660793450404748,0.6310163139953155,1.0490738299820432,7.771583528614682,5.529162243818064,1.6962857446252833,0.7301053737305642,0.5930043850043732,Sales,1 +hierarchical_sales/W/short,TabPFN-TS,468.4471890724312,468.4471890724312,8.973932945122153,0.723416703409247,0.5185541231281445,0.4571566258446645,6.623818653369418,21.64364084604139,0.9936701379339342,0.41199765098765,0.3417650702357695,Sales,1 +hospital/M/short,TabPFN-TS,2679.1871536831814,2679.1871536831814,17.352619912198502,0.7531851000735269,0.1923880595738138,0.1719025956294372,5.198411914036882,51.760865078582114,0.18790395984155,0.0629940398055021,0.0504805609674915,Healthcare,1 +jena_weather/10T/long,TabPFN-TS,1580.603511491402,1580.603511491402,10.20437644675926,0.656773253670893,0.7720577018595126,0.6560148370329034,6.982261500751263,39.756804593571175,0.2432786983121498,0.0624423276576957,0.0548977041176505,Nature,21 +jena_weather/10T/medium,TabPFN-TS,1886.342711715368,1886.342711715368,10.978890326253609,0.6245848971006992,0.7680008366190657,0.6628134899497204,6.55998411025622,43.4320470587718,0.2665417450671599,0.0673772659736879,0.0567937425736504,Nature,21 +jena_weather/10T/short,TabPFN-TS,1014.2037315005348,1014.2037315005348,6.599587485903785,0.3250009779636542,0.3718861119674825,0.5649007031841884,3.7936287448317167,31.84656545846875,0.197807102381248,0.0409917131943352,0.0354390154086276,Nature,21 +jena_weather/D/short,TabPFN-TS,392.590625,392.590625,9.983251178075395,1.2287532245574706,0.6514123184023372,0.4747197711278522,7.597867713507752,19.81389979282221,0.1193120610703935,0.0601154889594781,0.0465159837615308,Nature,21 +jena_weather/H/long,TabPFN-TS,1634.8796296296296,1634.8796296296296,12.753626198743389,1.0999403769809355,2.446339739877115,0.6785660032242063,9.942908231388692,40.43364477300593,0.243455339006535,0.0767909597863064,0.0659166328078663,Nature,21 +jena_weather/H/medium,TabPFN-TS,1497.2289682539683,1497.2289682539683,11.262710813492063,0.954497320291504,2.871433126547123,0.6940052819630457,7.893160249520058,38.69404305902872,0.2367757119864755,0.0689185249443742,0.0595068926034612,Nature,21 +jena_weather/H/short,TabPFN-TS,1140.7981199744152,1140.7981199744152,8.439706545824196,0.5951480288792269,1.1638988677264472,0.6275021163443276,5.439047960494513,33.77570310111123,0.2070428939213693,0.0517348598743774,0.0433749607608675,Nature,21 +kdd_cup_2018/D/short,TabPFN-TS,2724.3329307342574,2724.3329307342574,20.274988155794933,1.1739335474857984,0.5149150721308192,0.450249306230883,9.779238874612522,52.195142788714136,1.169226615544389,0.4541812612251059,0.3591637715617976,Nature,1 +kdd_cup_2018/H/long,TabPFN-TS,4021.7201519208343,4021.7201519208343,24.687106447847157,1.0575396852617072,1.2271145815783495,0.6184559653223703,8.973319969408267,63.41703361022837,1.4880199275644768,0.5792592976530704,0.4624436577729628,Nature,1 +kdd_cup_2018/H/medium,TabPFN-TS,5288.079190672816,5288.079190672816,27.137685229471725,1.136105934900563,1.229284216293004,0.5858576113913657,10.24346547878567,72.71918034929172,1.5221441583887936,0.5680409053818023,0.4624805251220042,Nature,1 +kdd_cup_2018/H/short,TabPFN-TS,4759.803006921011,4759.803006921011,26.16721637525615,1.0963647304693052,1.20523590775101,0.5491535691464683,8.96822139561829,68.99132559185257,1.4441133901222132,0.547727228395148,0.4366137476521748,Nature,1 +loop_seattle/5T/long,TabPFN-TS,122.0583144851508,122.0583144851508,6.643171560565016,1.046792008334055,0.2157579654133274,0.1466260668449808,9.325475135533132,11.04800047452709,0.1953309189687861,0.1174526384918547,0.0935730075998391,Transport,1 +loop_seattle/5T/medium,TabPFN-TS,108.86982964073788,108.86982964073788,6.15152889476103,0.9716574976595126,0.2035810928817135,0.1374303472890942,8.337243273977046,10.43407061700935,0.1856435555399793,0.1094483388073597,0.0873598980952376,Transport,1 +loop_seattle/5T/short,TabPFN-TS,46.96549013686992,46.96549013686992,3.739100572962018,0.587682274689241,0.1026436058237333,0.0796069208682506,5.314526420206936,6.853137247777102,0.1175161738226465,0.0641173198472113,0.051947251305268,Transport,1 +loop_seattle/D/short,TabPFN-TS,18.49710145365712,18.49710145365712,2.988963810750452,0.8988332716750893,0.0563158649409149,0.0551457534018438,7.200364515864249,4.3008256711539845,0.0768615012865513,0.0534167769984987,0.0438373093456756,Transport,1 +loop_seattle/H/long,TabPFN-TS,67.59270430211558,67.59270430211558,4.412962901374914,0.9223339565898164,0.1415486417441668,0.1011307854483453,7.48961052275084,8.221478230957956,0.1469287989883885,0.0788655422862782,0.0633112868629268,Transport,1 +loop_seattle/H/medium,TabPFN-TS,71.01074400154799,71.01074400154799,4.58358863539248,0.9478641312684954,0.1195942428951046,0.1041414400618388,7.735268938960752,8.426787288258081,0.148942523265887,0.0810144167183933,0.0651342563580121,Transport,1 +loop_seattle/H/short,TabPFN-TS,74.5299885470982,74.5299885470982,4.490799522058028,0.9121818732427218,0.1258212686213086,0.102138436771049,8.036389745904458,8.63307526592339,0.1528142142383595,0.0794917198248121,0.0637984701807385,Transport,1 +m4_daily/D/short,TabPFN-TS,698541.3674169455,698541.3674169455,197.73342738221973,4.310262159241137,0.0385161610501193,0.0324561628228029,29.27039887526933,835.7878722600284,0.1291061274923901,0.030544349747604,0.0235298029984748,Econ/Fin,1 +m4_hourly/H/short,TabPFN-TS,1974169.442342405,1974169.442342405,276.0602304946973,0.7804866407185013,0.0963661524792608,0.0932995079028241,5.657858857138344,1405.051402028554,0.1918205590830847,0.0376883206391206,0.0281976469731856,Econ/Fin,1 +m4_monthly/M/short,TabPFN-TS,1756663.0326296296,1756663.0326296296,526.6686806369357,0.895008061940836,0.14536284466143,0.1226754361655977,7.925692832251994,1325.3916525426096,0.275461884811926,0.1094598318627909,0.0882623211716558,Econ/Fin,1 +m4_quarterly/Q/short,TabPFN-TS,1773423.9763385416,1773423.9763385416,555.0584905192057,1.1712220135297755,0.1143357895165681,0.1000043484320243,9.74443607939694,1331.6996569566809,0.222903072567125,0.0929070172428722,0.0750751456929036,Econ/Fin,1 +m4_weekly/W/short,TabPFN-TS,200816.425112492,200816.425112492,244.96552255196056,2.070145368591285,0.0603222059238025,0.0608874742545942,16.100915789618952,448.1254568895768,0.0816415071190786,0.0446289184108609,0.0358837513094015,Econ/Fin,1 +m4_yearly/A/short,TabPFN-TS,3351283.2554191696,3351283.2554191696,858.1312017570586,3.160159979266896,0.1653807364774557,0.1417295112996192,32.49565409116194,1830.651046873535,0.2935533802946987,0.1376053155746721,0.113378683259915,Econ/Fin,1 +m_dense/D/short,TabPFN-TS,6584.881481481481,6584.881481481481,39.23614293981481,0.6343391992101958,0.1154036016443025,0.0916553243001302,6.472099200744528,81.14728265001534,0.1404779495226384,0.067923567214623,0.0571463211874774,Transport,1 +m_dense/H/long,TabPFN-TS,72915.27901234568,72915.27901234568,120.79466049382717,1.0390647350813125,0.3313089644319483,0.2607480951003086,8.591854018250109,270.02829298491235,0.4674588925970861,0.2091134140126585,0.164429129974029,Transport,1 +m_dense/H/medium,TabPFN-TS,69178.59333333334,69178.59333333334,116.22125,1.016694269579333,0.3125688394564185,0.255682127210829,7.960831761937791,263.01823764395755,0.4576185046768153,0.2022102920051717,0.1585943408314112,Transport,1 +m_dense/H/short,TabPFN-TS,71334.99722222223,71334.99722222223,109.0473996310764,0.9158806250212368,0.3099409964832582,0.2445430331759982,10.001841796408856,267.08612323035845,0.4737442837218426,0.1934229364114031,0.1543850478362672,Transport,1 +restaurant/D/short,TabPFN-TS,178.78818526285374,178.78818526285374,8.29543477096978,0.7821100931962905,0.7700873702984403,0.4375370417899797,5.118488585776721,13.371169928725523,0.6153722208093952,0.3817751284892775,0.2965685918933398,Sales,1 +saugeen/D/short,TabPFN-TS,947.38125,947.38125,14.878590494791666,3.303358954426323,0.503057352701823,0.4162318166097005,27.681241412470868,30.779558963701867,0.996957248538798,0.4819210911798914,0.3842106395309255,Nature,1 +saugeen/M/short,TabPFN-TS,376.4469401041667,376.4469401041667,12.06310017903646,0.706898496538842,0.3438984553019206,0.3432963916233608,4.669924664301697,19.4022405949459,0.5825134688369025,0.3621704563362662,0.2778272956964877,Nature,1 +saugeen/W/short,TabPFN-TS,1006.3185546875,1006.3185546875,15.425779724121094,1.2511660036320285,0.4077761650085449,0.4090907096862793,9.909876875391326,31.72252440597217,0.9575707772767752,0.4656399878991368,0.3804498298468377,Nature,1 +solar/10T/long,TabPFN-TS,25.8354749594485,25.8354749594485,2.0935281349794708,0.9152877937539702,3.283013031614603,1.4939689741420823,5.0969315666695385,5.082860903019922,1.098579050884435,0.4524826068246617,0.3523350277353385,Energy,1 +solar/10T/medium,TabPFN-TS,26.23753896745742,26.23753896745742,2.134519475520488,0.9354003397464944,3.687648892317398,1.4898048797452168,5.059326482355148,5.122259166369603,1.1366496791575578,0.4736583601499977,0.3587467328055458,Energy,1 +solar/10T/short,TabPFN-TS,34.44245373802463,34.44245373802463,2.4745698469398665,1.0809372089579272,4.177828026682609,1.5824794908509636,4.971023404696623,5.86876935464537,1.7072243365297972,0.7198520865016745,0.5451162209570433,Energy,1 +solar/D/short,TabPFN-TS,115559.18248175182,115559.18248175182,252.5994449513382,0.9850887951305594,1.015654671105155,0.4335812756614963,5.094867900990621,339.93996893826977,0.4910974808637134,0.3649201694952824,0.2688506624469601,Energy,1 +solar/H/long,TabPFN-TS,877.5276763990267,877.5276763990267,12.818477861415248,0.9770759245501968,4.404037612326265,1.4354754742624696,4.430542735515225,29.62309363316105,1.02708677819807,0.444440047049852,0.3243015754362526,Energy,1 +solar/H/medium,TabPFN-TS,867.4373935523114,867.4373935523114,12.00386110477494,0.9209503012117918,4.588050238841548,1.415317507555885,4.401836804057126,29.452290124068643,1.0609402322505224,0.4324070941429265,0.3243953703339113,Energy,1 +solar/H/short,TabPFN-TS,919.4725602069728,919.4725602069728,12.126564822134066,0.936711092050702,4.2082555663250165,1.4261984386094817,8.184709098187527,30.32280594217779,1.1177087308701164,0.4469892200282378,0.3582317194650038,Energy,1 +solar/W/short,TabPFN-TS,1083859.6350364964,1083859.6350364964,812.4620067290145,0.8776728357098831,0.1699104378693295,0.1593432600480796,6.179114064222528,1041.0857961938086,0.2125116344852166,0.165843804265081,0.1238329806565449,Energy,1 +sz_taxi/15T/long,TabPFN-TS,17.098993945868944,17.098993945868944,2.960354205617877,0.559861515226893,6013879437233.814,0.4328146040609419,7.534451973562091,4.135092979108081,0.3827368106654918,0.2740050907737393,0.2478785901755847,Transport,1 +sz_taxi/15T/medium,TabPFN-TS,17.602143012152776,17.602143012152776,2.996658194778312,0.5854737021918714,10236354323510.264,0.4397119766626602,7.082834116822467,4.195490795145757,0.3904604399549113,0.2788890583388779,0.2448335839801526,Transport,1 +sz_taxi/15T/short,TabPFN-TS,17.541289314391122,17.541289314391122,2.853190160059667,0.5712382957982439,992721185973.3234,0.4160638187394474,4.784492880911788,4.188232242174629,0.3916176378863725,0.2667854899906525,0.214915409859324,Transport,1 +sz_taxi/H/short,TabPFN-TS,7.419267312074319,7.419267312074319,1.9177468210204036,0.5822117538174562,1.5181874943594649,0.310376113296574,4.562929717078705,2.723833201955347,0.2537418681484218,0.1786499484080345,0.1440468609014281,Transport,1 +temperature_rain/D/short,TabPFN-TS,190.3859944198081,190.3859944198081,5.962294301692679,1.3824531132761009,36.76048413464811,1.533545632917945,21.54886608943876,13.798043137336832,1.6243944833910349,0.7019196762630732,0.5654525014897981,Nature,1 +us_births/D/short,TabPFN-TS,111350.18666666666,111350.18666666666,217.17235677083332,0.3196481603311801,0.020750192006429,0.0206375964482625,2.813680058790136,333.6917539686389,0.0312808941821572,0.0203581461952356,0.0176613012395234,Healthcare,1 +us_births/M/short,TabPFN-TS,41925221.333333336,41925221.333333336,5200.2734375,0.5880410553209511,0.0159080078204472,0.0160710861285527,3.327505102123412,6474.968828753798,0.0201112090363295,0.0161520138418586,0.0125006102341928,Healthcare,1 +us_births/W/short,TabPFN-TS,1749773.857142857,1749773.857142857,1021.7772739955356,0.9289999495036037,0.0138602065188544,0.0138870137078421,5.523556437789926,1322.7901788049596,0.017956746971846,0.0138705263047056,0.0107906323842147,Healthcare,1 \ No newline at end of file diff --git a/results/tabpfn_ts/config.json b/results/tabpfn_ts/config.json index 6273ea6da7c31504ff3cda59098cd93fb138536e..5be65acc213a78dd7f8478933c42710e84717d9d 100644 --- a/results/tabpfn_ts/config.json +++ b/results/tabpfn_ts/config.json @@ -1,10 +1,6 @@ { "model": "TabPFN-TS", - "model_type": "zero-shot", + "model_type": "pretrained", "model_dtype": "float32", - "model_link": "https://github.com/liam-sbhoo/tabpfn-time-series/tree/main", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/tabpfn_ts.ipynb", - "org": "PriorLabs", - "testdata_leakage": "No", - "replication_code_available": "Yes" + "model_link": "https://github.com/liam-sbhoo/tabpfn-time-series/tree/main" } \ No newline at end of file diff --git a/results/tempo_ensemble/all_results.csv b/results/tempo_ensemble/all_results.csv deleted file mode 100644 index d73f358b240139ea0696a116873dac4a118d2537..0000000000000000000000000000000000000000 --- a/results/tempo_ensemble/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bizitobs_application/10S/long,TEMPO_ENSEMBLE,487627.2711111111,487627.2711111111,396.0836111111111,1.2993675903516406,0.0178912925720214,0.0178272904290093,9.800100249012653,698.3031369764217,0.0269307607957717,0.0152753616891155,0.0120698276506844,Web/CloudOps,2.0 -bizitobs_application/10S/medium,TEMPO_ENSEMBLE,307498.0266666667,307498.0266666667,311.76875,0.9177696033266344,0.0131387344996134,0.013129768371582,5.872428510396111,554.5250460228705,0.0217745664925907,0.0122422412222421,0.0094315406776331,Web/CloudOps,2.0 -bizitobs_application/10S/short,TEMPO_ENSEMBLE,525641.8133333334,525641.8133333334,390.0379861111112,0.9921897290135696,0.0150116687350802,0.0149332237243652,6.206112108526064,725.0115953095739,0.0279607992769224,0.0150422060979192,0.0116365011231077,Web/CloudOps,2.0 -bizitobs_service/10S/long,TEMPO_ENSEMBLE,128864.03386243386,128864.03386243386,71.79506613756614,1.3563611507860072,0.1136272192253637,0.0937695441674933,21.32547687545452,358.976369504225,0.2659411707148045,0.0531880802252819,0.0497378287376309,Web/CloudOps,2.0 -bizitobs_service/10S/medium,TEMPO_ENSEMBLE,98009.9961904762,98009.9961904762,66.84960813492063,1.429575874324731,0.1006226942274305,0.089753195141989,16.277174249724983,313.0654822724412,0.2356204593806827,0.0503125904006926,0.0436408573760635,Web/CloudOps,2.0 -bizitobs_service/10S/short,TEMPO_ENSEMBLE,8438.823280423281,8438.823280423281,29.30463913690476,1.2884984418523877,0.0968797762431795,0.0843838533270295,10.922623337489345,91.86306809824762,0.0680550969687549,0.0217098133056419,0.0179825354410236,Web/CloudOps,2.0 -car_parts/M/short,TEMPO_ENSEMBLE,1.448840955093402,1.448840955093402,0.4170860887539882,17.86660919142645,0.9999799200634808,1.9999561636701624,713.7590747571373,1.2036780944643806,2.886296596697481,1.0001296559577155,1.0000739200613302,Sales,1.0 -covid_deaths/D/short,TEMPO_ENSEMBLE,3359486.528320802,3359486.528320802,376.5356868734336,115.6153349693266,0.1948706839966014,0.4917750161531234,1608.2690771273951,1832.8902117477744,0.6896338295236072,0.1416733779396501,0.1157821881701,Healthcare,1.0 -hospital/M/short,TEMPO_ENSEMBLE,6031.719578444155,6031.719578444155,23.99147407037701,4.696532893791447,0.2181512732342086,0.1840471568598741,37.147902714383534,77.6641460291952,0.2819388848011488,0.0870946220356949,0.0701390741514011,Healthcare,1.0 -m4_daily/D/short,TEMPO_ENSEMBLE,1121130.894859576,1121130.894859576,516.3883590607996,7.987928627932832,0.1088423646593421,0.0810604800101247,64.81370332574242,1058.8346872196698,0.163560696084881,0.0797677300125995,0.0606080461596886,Econ/Fin,1.0 -m4_hourly/H/short,TEMPO_ENSEMBLE,1724030.044535024,1724030.044535024,289.1650402419233,2.2278853386116246,0.1401686307504749,0.1173738365971907,16.376050421624175,1313.023245999485,0.1792566653066635,0.0394774129056468,0.031167746098992,Econ/Fin,1.0 -m4_monthly/M/short,TEMPO_ENSEMBLE,2815265.2656666664,2815265.2656666664,867.5761620008681,4.678383723019503,0.235788005510966,0.1978487661213037,57.8058061415511,1677.8752235093843,0.3476662674377067,0.1797672209080605,0.1494297959225055,Econ/Fin,1.0 -m4_quarterly/Q/short,TEMPO_ENSEMBLE,2514810.428,2514810.428,810.4905891927083,2.182249153802416,0.185947568098704,0.1463373041947682,23.608043900790165,1585.8153826974942,0.265437569067886,0.1356618520005551,0.1109355578587279,Econ/Fin,1.0 -m4_weekly/W/short,TEMPO_ENSEMBLE,541471.1017784444,541471.1017784444,439.3405694236126,4.832172082392305,0.1009065322761543,0.1079463725067849,45.31173448032055,735.8472000207953,0.1340599430870445,0.0800410353278435,0.0637560156785005,Econ/Fin,1.0 -m4_yearly/A/short,TEMPO_ENSEMBLE,3878844.870897536,3878844.870897536,945.4389228040392,2.3981068438654534,0.2092134684314751,0.158700357189562,30.97324149680404,1969.478324556413,0.3158149780005754,0.1516055134414823,0.1267824542222521,Econ/Fin,1.0 -restaurant/D/short,TEMPO_ENSEMBLE,154.6462244326142,154.6462244326142,7.553120731472402,1.9378580999513804,0.6267567772908494,0.4301861248252114,17.613543161794635,12.43568351288397,0.5723189684521193,0.3476121164672595,0.2785855176058847,Sales,1.0 -temperature_rain/D/short,TEMPO_ENSEMBLE,196.24055072576525,196.24055072576525,5.971279438835857,1.1188085728690171,25.89721481389996,1.5564323267332705,23.68132606745661,14.008588463002448,1.6491812374036674,0.7029774655619903,0.6018449486123362,Nature,1.0 -loop_seattle/5T/long,TEMPO_ENSEMBLE,45.255009312113,45.255009312113,3.82509889249011,0.609772921780666,0.115840638970689,0.0808924666786259,5.58609892597957,6.727184352469688,0.1138146269817346,0.064715366906391,0.0521126066018827,Transport,1.0 -loop_seattle/5T/medium,TEMPO_ENSEMBLE,33.792408340186405,33.792408340186405,3.272990517316096,0.5214548025956122,0.0851551320653704,0.0641574094711319,4.822810274952749,5.813123802241477,0.0960772879128691,0.0540948486503963,0.0436714854292893,Transport,1.0 -loop_seattle/5T/short,TEMPO_ENSEMBLE,25.731941133386947,25.731941133386947,2.6942812042708737,0.4298290643570778,0.07453944464094,0.0555199753512292,3.882450724435325,5.072666077457391,0.0858552006954052,0.0456008832417526,0.0365482163898581,Transport,1.0 -loop_seattle/D/short,TEMPO_ENSEMBLE,201.8897832817337,201.8897832817337,11.740500435371516,2.774823575326353,0.2117016866111165,0.1850822204652831,53.69285838634296,14.208792463884246,0.2539301027317229,0.2098184268123754,0.1896011897345642,Transport,1.0 -loop_seattle/H/long,TEMPO_ENSEMBLE,107.18005890952873,107.18005890952873,5.884305998989508,1.2059726001740343,0.2284862642973856,0.1319734503159131,10.89359255795443,10.35278025022886,0.1850180328102056,0.105160420106692,0.0858770469329422,Transport,1.0 -loop_seattle/H/medium,TEMPO_ENSEMBLE,102.0016705366357,102.0016705366357,5.591995009352425,1.1218170402210388,0.2061506399306227,0.1263956800210832,10.534682855933928,10.099587641910718,0.1785090812377754,0.0988378859413388,0.0806802675836623,Transport,1.0 -loop_seattle/H/short,TEMPO_ENSEMBLE,63.93484438226468,63.93484438226468,4.569113424888654,0.9321286612874126,0.1389670075459092,0.1052935476325374,7.722207374375397,7.995926736924537,0.1415360365351048,0.0808779552283411,0.0647137448640539,Transport,1.0 -m_dense/D/short,TEMPO_ENSEMBLE,5768.324444444444,5768.324444444444,37.38796875,0.4039148995409595,0.1074763044239946,0.0843358188205295,5.089627880380552,75.94948613680309,0.1314797949034482,0.0647241043112796,0.0539366130007633,Transport,1.0 -m_dense/H/long,TEMPO_ENSEMBLE,120260.29827160494,120260.29827160494,162.72006172839505,1.3319808133687625,0.4192954648363726,0.3339448905285493,10.119297399210735,346.7856661853326,0.6003372524100504,0.2816924812509744,0.221528231064794,Transport,1.0 -m_dense/H/medium,TEMPO_ENSEMBLE,95303.87555555555,95303.87555555555,142.89605034722223,1.1974521911956586,0.3782484602009441,0.3119469536675347,9.506814665422915,308.7132578227821,0.5371220668355627,0.2486210746063868,0.1993907938331504,Transport,1.0 -m_dense/H/short,TEMPO_ENSEMBLE,39899.67444444444,39899.67444444444,87.75685763888889,0.7629589384918614,0.2737046016924269,0.2153883404201931,6.4551554438991126,199.749028644558,0.3543050243689524,0.1556588074308831,0.1239959977566907,Transport,1.0 -sz_taxi/15T/long,TEMPO_ENSEMBLE,17.11225683315527,17.11225683315527,2.807824567919783,0.5592388252412251,891763845532.28,0.4074579570368145,4.479720970305226,4.136696367048864,0.3854328199908255,0.2616164313807031,0.2072789439258874,Transport,1.0 -sz_taxi/15T/medium,TEMPO_ENSEMBLE,17.181647858796296,17.181647858796296,2.802406906606125,0.5587890281052588,980879675330.1469,0.4046340985855146,4.435455309207469,4.145075133070123,0.3862134936765363,0.2611116390794976,0.2067451524244509,Transport,1.0 -sz_taxi/15T/short,TEMPO_ENSEMBLE,16.188086902585805,16.188086902585805,2.710469068217481,0.5421277246410363,1559028962787.2595,0.3815988051585662,3.9974291375358737,4.023442170901156,0.3754715629735235,0.252943627423149,0.1990819241581065,Transport,1.0 -sz_taxi/H/short,TEMPO_ENSEMBLE,7.189825987204527,7.189825987204527,1.849522386860644,0.5637851957501169,1.014901242704473,0.2964422356369149,4.431838456166187,2.681385087450985,0.2497875734919494,0.1722944277176315,0.136247323200995,Transport,1.0 -bitbrains_fast_storage/5T/long,TEMPO_ENSEMBLE,2440224.267545323,2440224.267545323,229.6813110807334,1.117978597062503,2.251204239869468,0.8221634438380008,27.51323770956846,1562.121719823818,4.745113207690169,0.6976817548453914,0.5854367008276827,Web/CloudOps,2.0 -bitbrains_fast_storage/5T/medium,TEMPO_ENSEMBLE,2631657.734381467,2631657.734381467,232.32689319711096,1.4213753528808146,2.3469343026571554,0.8229977424824282,36.54996504533897,1622.238494914193,4.928808505230026,0.7058732552417682,0.5948226525152782,Web/CloudOps,2.0 -bitbrains_fast_storage/5T/short,TEMPO_ENSEMBLE,2230525.403204316,2230525.403204316,205.1374160586084,1.064509249551392,1.617535049043656,0.7656597272656195,19.9251692395756,1493.4943599506212,3.9417512832669366,0.5414152839665344,0.4378417919631492,Web/CloudOps,2.0 -bitbrains_fast_storage/H/short,TEMPO_ENSEMBLE,2005001.4583279784,2005001.4583279784,235.38284965169785,1.468075876452505,2.1154490817398,0.6492987395299152,30.887620124678627,1415.9807408040472,4.036005106798449,0.6709175879804512,0.5386730388224907,Web/CloudOps,2.0 -bitbrains_rnd/5T/long,TEMPO_ENSEMBLE,2049491.817123375,2049491.817123375,154.23860818175223,5.354332726402957,1.044259314751908,0.7570510199845623,190.4855143937792,1431.604630169718,5.963076180299257,0.6424515198879186,0.5788157952641853,Web/CloudOps,2.0 -bitbrains_rnd/5T/medium,TEMPO_ENSEMBLE,2012628.349056859,2012628.349056859,154.0856175924186,2.0206660007438004,1.2854108619095217,0.7562830102005815,50.702933072419086,1418.6713322883702,5.969073466936588,0.6483167388205798,0.5803418284708388,Web/CloudOps,2.0 -bitbrains_rnd/5T/short,TEMPO_ENSEMBLE,1092676.6090575084,1092676.6090575084,102.72774254536428,1.4738776261427329,0.6909973121152531,0.724109150334899,27.830770596585733,1045.3117281737102,4.614003515916433,0.4534400146017512,0.3903323828435197,Web/CloudOps,2.0 -bitbrains_rnd/H/short,TEMPO_ENSEMBLE,1373237.396530412,1373237.396530412,140.5587084683091,6.4469252489463615,1.230340532002289,0.6496405541543678,212.9778414003044,1171.8521222963298,5.128598282316404,0.6151536845814527,0.5300669828610838,Web/CloudOps,2.0 -bizitobs_l2c/5T/long,TEMPO_ENSEMBLE,28.02539958112875,28.02539958112875,3.134216545414462,0.2992086737079788,0.2278988936816867,0.5175256799768518,1.9222874573646005,5.293902112915269,0.2724096828546203,0.1612781870392175,0.1255549472781602,Web/CloudOps,7.0 -bizitobs_l2c/5T/medium,TEMPO_ENSEMBLE,23.050455729166668,23.050455729166668,2.8030650111607143,0.28310808843281,0.217644268352139,0.6169422549293154,2.255901530024711,4.801089014918039,0.2471325504016436,0.1442857241341077,0.1131950526630068,Web/CloudOps,7.0 -bizitobs_l2c/5T/short,TEMPO_ENSEMBLE,3.1966607956659225,3.1966607956659225,1.1986464727492558,0.1151517649409104,0.0398835409255254,0.038441853296189,0.667002474954807,1.7879208024031497,0.0520254942308455,0.034878600365867,0.0269425800159113,Web/CloudOps,7.0 -bizitobs_l2c/H/long,TEMPO_ENSEMBLE,16.538295200892858,16.538295200892858,2.070875573536706,0.2180866660308821,0.2278338753179033,0.6110211084759425,1.3445094546037508,4.066730283765184,0.2484076230387728,0.1264950569465923,0.097937862969621,Web/CloudOps,7.0 -bizitobs_l2c/H/medium,TEMPO_ENSEMBLE,12.969489397321428,12.969489397321428,1.750689697265625,0.1794982441293087,0.1523565293116185,0.6096963065011161,1.113684244320353,3.6013177306815667,0.2180652586802799,0.1060069203143813,0.0810585643608789,Web/CloudOps,7.0 -bizitobs_l2c/H/short,TEMPO_ENSEMBLE,4.876285129123264,4.876285129123264,1.302290780203683,0.13201672288296,0.1161148839717288,0.436991464524042,0.9056123725875996,2.2082312218432345,0.1190293182352498,0.0701968082773084,0.0558677005802573,Web/CloudOps,7.0 -electricity/15T/long,TEMPO_ENSEMBLE,133835.22996771772,133835.22996771772,53.456839245495495,0.9939385780309388,0.1802039587686282,0.1575861328858155,9.52136853864833,365.8349764138439,0.6912502396245063,0.1010074359216439,0.0823081285199345,Energy,1.0 -electricity/15T/medium,TEMPO_ENSEMBLE,147086.25630236487,147086.25630236487,58.9102391258446,1.1008310035581168,0.2031415438085511,0.1748311696095509,10.063220095437282,383.5182607156599,0.731616319583604,0.1123797658411746,0.0906301323705994,Energy,1.0 -electricity/15T/short,TEMPO_ENSEMBLE,53514.92639850788,53514.92639850788,42.72801652238176,0.8701977534359403,0.1565123488839787,0.1385838226799492,6.664328952354419,231.33293409825563,0.424854101914554,0.078472065194558,0.0622887006592235,Energy,1.0 -electricity/D/short,TEMPO_ENSEMBLE,1773811077.968721,1773811077.968721,4735.994360360361,1.115364338640572,0.5424773373398062,0.1024850777290962,10.020402740809686,42116.6365937348,0.6921162106217391,0.0778281158070003,0.0615318072875596,Energy,1.0 -electricity/H/long,TEMPO_ENSEMBLE,21656666.85463063,21656666.85463063,496.1748430930931,2.0531234768133286,0.3964748661654531,0.2308328186585022,13.780891404513913,4653.672405168914,1.835406963645144,0.1956912053343266,0.1479181783703783,Energy,1.0 -electricity/H/medium,TEMPO_ENSEMBLE,11332664.060585586,11332664.060585586,370.4004170185811,1.6193840624647802,0.2917618788278519,0.192386435431403,11.603075966381072,3366.402242838129,1.3140177023969903,0.144579485702608,0.1126590158033672,Energy,1.0 -electricity/H/short,TEMPO_ENSEMBLE,1832069.8023648649,1832069.8023648649,198.91421179969032,1.0666581008254952,0.2378714794262591,0.1420216092977438,10.306280643530442,1353.5397306192622,0.6389674211684946,0.0939017142032927,0.0757270380140367,Energy,1.0 -electricity/W/short,TEMPO_ENSEMBLE,46064089239.29369,46064089239.29369,29281.876126126128,6.7051638386670405,0.2451873894841502,0.1105028925715266,78.41860400355218,214625.4627002437,0.4897044792472421,0.0668115782690427,0.0571458143436201,Energy,1.0 -ett1/15T/long,TEMPO_ENSEMBLE,2.426123046875,2.426123046875,0.8594006541418651,0.6251000870679456,0.2925888486169558,0.2088383556547619,4.284138746297137,1.5576016971212503,0.2899107367971693,0.1599571169617456,0.1245837835659594,Energy,7.0 -ett1/15T/medium,TEMPO_ENSEMBLE,3.665840308779762,3.665840308779762,0.9739117140997025,0.6656957195358719,0.3419423155295361,0.2164743913922991,4.975116861543028,1.9146384276880484,0.3483354661955551,0.1771864525741701,0.140029793907285,Energy,7.0 -ett1/15T/short,TEMPO_ENSEMBLE,0.6365459623790922,0.6365459623790922,0.4495159694126674,0.3504219026354622,0.148824625845035,0.1175648008074079,2.5677515620110003,0.7978383059110989,0.1424446592250454,0.0802557969513414,0.0629222896778817,Energy,7.0 -ett1/D/short,TEMPO_ENSEMBLE,8428.484126984127,8428.484126984127,50.722237723214285,0.416001643493369,0.6177615211123512,0.1943805270724826,2.4777156566549183,91.806776040683,0.241133542532727,0.1332236398538365,0.1027991523160863,Energy,7.0 -ett1/H/long,TEMPO_ENSEMBLE,186.56152447089949,186.56152447089949,6.966465153769842,1.1478718553875968,706542928866.8086,0.434202228525959,7.452181852979635,13.658752668926232,0.6540880825104363,0.3336089278980483,0.2606986690022358,Energy,7.0 -ett1/H/medium,TEMPO_ENSEMBLE,210.31737351190475,210.31737351190475,7.783360072544643,1.2921248816752966,2394988688740.979,0.4857444036574591,7.670278067717789,14.502323038461968,0.6946268959068199,0.3728044970849133,0.2862649623279022,Energy,7.0 -ett1/H/short,TEMPO_ENSEMBLE,36.55992838541667,36.55992838541667,3.31173822312128,0.6173025444422964,0.308757091703869,0.1946493421282087,4.307629355652223,6.0464806611298005,0.2822139896318907,0.1545723717553161,0.1206314820077215,Energy,7.0 -ett1/W/short,TEMPO_ENSEMBLE,680088.7142857143,680088.7142857143,625.6919642857143,6.086432786812403,0.5392628056662423,0.3573795161021351,37.66618612480504,824.6749143060642,0.328157831536192,0.248977766448001,0.1820576488837628,Energy,7.0 -ett2/15T/long,TEMPO_ENSEMBLE,5.956928323412699,5.956928323412699,1.6044109623015872,0.7047809146350211,0.1055386204469079,0.1197530691964285,4.620370019558849,2.440681938191189,0.123840215227059,0.0814078212220199,0.0636566074806678,Energy,7.0 -ett2/15T/medium,TEMPO_ENSEMBLE,5.548760230654762,5.548760230654762,1.5561342075892857,0.671412971844876,0.0967213048749063,0.1097720918201264,4.294814353598369,2.3555806567924527,0.1146702656271786,0.0757530091025556,0.0591136068282064,Energy,7.0 -ett2/15T/short,TEMPO_ENSEMBLE,2.1440130324590774,2.1440130324590774,1.039876229422433,0.4684627317471617,0.0646613530082772,0.1166317985171363,2.836496571230992,1.4642448676567308,0.0650103948386862,0.0461690293415903,0.0356368146844352,Energy,7.0 -ett2/D/short,TEMPO_ENSEMBLE,50878.18095238095,50878.18095238095,143.57472718253968,0.5795335298098135,0.115604255312965,0.0856832716200086,3.9209683971288687,225.561922656243,0.1290950395867559,0.0821716044579683,0.0652697198570775,Energy,7.0 -ett2/H/long,TEMPO_ENSEMBLE,146.3332175925926,146.3332175925926,7.887528935185185,0.938929495976326,0.1597060509673862,0.160266290896784,6.476700944863757,12.09682675715382,0.1656567320337467,0.1080136380767531,0.0849789777142148,Energy,7.0 -ett2/H/medium,TEMPO_ENSEMBLE,116.4034412202381,116.4034412202381,7.034888857886905,0.8417988347064325,0.1428101438678359,0.1461324146815708,5.497659780958897,10.789042646140484,0.1453892878304396,0.0947996513277619,0.0746216000768869,Energy,7.0 -ett2/H/short,TEMPO_ENSEMBLE,83.8348400297619,83.8348400297619,5.616767810639881,0.657868124921435,0.1062725054061009,0.1005470820835658,4.51715751563887,9.15613674153908,0.1161252899482322,0.0712362439524702,0.0559794964806631,Energy,7.0 -ett2/W/short,TEMPO_ENSEMBLE,50153408.0,50153408.0,5750.481584821428,18.879777231189173,0.5354306357247489,0.7778287484025764,145.92744525176246,7081.907087783629,0.5940634324121872,0.4823772447106784,0.3722205310770798,Energy,7.0 -hierarchical_sales/D/short,TEMPO_ENSEMBLE,26.144174374741628,26.144174374741628,2.2159887987998657,0.6776781916278962,0.6214346639809414,1.0697553971327254,7.844630481376959,5.11313742967482,1.5686539388430278,0.6798408228762413,0.5596516023662144,Sales,1.0 -hierarchical_sales/W/short,TEMPO_ENSEMBLE,208.41687632415253,208.41687632415253,6.893525980286679,1.5405539950087042,0.501563509229671,0.3862917948577363,12.07970736684027,14.436650453763592,0.6627936838232559,0.3164851496293178,0.2529514701591871,Sales,1.0 -jena_weather/10T/long,TEMPO_ENSEMBLE,1494.6670006613756,1494.6670006613756,9.313304398148148,0.6309415613166219,0.5533342606620286,0.6102226200810185,7.968598337655945,38.66092343260021,0.2365608812169572,0.0569868306252071,0.0484899654114696,Nature,21.0 -jena_weather/10T/medium,TEMPO_ENSEMBLE,1428.2385615079363,1428.2385615079363,9.73109343998016,0.770657587644626,0.45034885466974,0.5904903351314484,7.660509410812896,37.79204362703791,0.2300681949417704,0.0592403820402081,0.0492121746241922,Nature,21.0 -jena_weather/10T/short,TEMPO_ENSEMBLE,356.6833209325397,356.6833209325397,6.77793705047123,0.6259220489380137,0.1543256824463067,0.4635223751976376,4.5303511138071695,18.886061551645422,0.1160439813650122,0.0416465232111659,0.0315293674377684,Nature,21.0 -jena_weather/D/short,TEMPO_ENSEMBLE,409.1419642857143,409.1419642857143,9.749877542162698,1.3035383788786572,0.5480121025175815,0.3650206308516245,13.435620298887152,20.22725795271604,0.1218011527955401,0.0587101982397621,0.049186717650696,Nature,21.0 -jena_weather/H/long,TEMPO_ENSEMBLE,3616.809523809524,3616.809523809524,16.37919560185185,1.02478500142865,2.041743679492345,0.7062990864748677,8.303475084254293,60.13991622715752,0.3621089262445626,0.0986209044544351,0.0830668877325188,Nature,21.0 -jena_weather/H/medium,TEMPO_ENSEMBLE,2571.667857142857,2571.667857142857,15.079837859623016,0.948315476755886,1.4699010265727828,0.7289495558965774,10.947285563727604,50.7116146177861,0.3103133637080175,0.0922762023189436,0.0783835088305258,Nature,21.0 -jena_weather/H/short,TEMPO_ENSEMBLE,1018.5410792606516,1018.5410792606516,8.795254330487678,0.6307897479895602,1.1392179782126224,0.6025232810623564,6.694776691348025,31.91459038215361,0.1956343923102162,0.0539143449925314,0.0444314398134425,Nature,21.0 -kdd_cup_2018/D/short,TEMPO_ENSEMBLE,1516.345084031518,1516.345084031518,11.394115079439292,0.8988963505503572,0.3130111050055663,0.2621102319434963,8.453911474684531,38.94027585972547,0.8723035410275897,0.2552402804354663,0.2098857562368817,Nature,1.0 -kdd_cup_2018/H/long,TEMPO_ENSEMBLE,3226.9120112959263,3226.9120112959263,19.277435912975083,0.8769099918367069,1.0711117767331302,0.4946325637606668,8.774512919523671,56.80591528437797,1.3328962906449948,0.4523265348849569,0.3713646975262221,Nature,1.0 -kdd_cup_2018/H/medium,TEMPO_ENSEMBLE,4411.474204999715,4411.474204999715,21.78959218251548,0.9389850331762508,0.906548250640067,0.4969745987220428,8.641128078775742,66.41892956830692,1.3902684103310523,0.4560956022965295,0.3721063901737007,Nature,1.0 -kdd_cup_2018/H/short,TEMPO_ENSEMBLE,4213.191972082293,4213.191972082293,18.50047424942846,0.7755335648478038,0.7064144683127912,0.4231625609033334,7.290873307077078,64.9091054635811,1.358665135900395,0.3872484327245064,0.317560729859845,Nature,1.0 -saugeen/D/short,TEMPO_ENSEMBLE,100.51135416666666,100.51135416666666,2.6642049153645835,0.3402823763010718,0.0503207906087239,0.0514753182729085,2.1419360602772457,10.025535106250771,0.3247294708297791,0.0862942319965577,0.0661465559255415,Nature,1.0 -saugeen/M/short,TEMPO_ENSEMBLE,4.164726983933222,4.164726983933222,1.347757066999163,0.0789051595695627,0.0377655313128516,0.0387396386691502,0.8427224228584025,2.04076627371515,0.0612699257784091,0.040463710384647,0.0340753637601601,Nature,1.0 -saugeen/W/short,TEMPO_ENSEMBLE,17.543472290039062,17.543472290039062,2.67932243347168,0.1533740309245224,0.0759897470474243,0.0801339026334025,1.5005436040659956,4.188492842304862,0.1264331392828559,0.0808775755788074,0.0707038225619495,Nature,1.0 -solar/10T/long,TEMPO_ENSEMBLE,15.809275712185723,15.809275712185723,1.8056089219637064,0.7935415450083824,2.3277865434987204,1.4880007555885038,5.265772326000681,3.976087991001422,0.9494284174682992,0.4311515301538582,0.3259064764784643,Energy,1.0 -solar/10T/medium,TEMPO_ENSEMBLE,13.317824712971412,13.317824712971412,1.4643280073848084,0.6403629744856407,1.6871777505154564,1.4886846107055962,5.860875318184641,3.64935949352368,0.9297285097230794,0.3730592994380783,0.2855840260224301,Energy,1.0 -solar/10T/short,TEMPO_ENSEMBLE,4.579851478149712,4.579851478149712,0.9389028396049556,0.4148577858316679,1.290939476594771,1.4926070684354091,3.4334659028362795,2.140058755770437,1.3168532900106638,0.5777398821412396,0.461325856836472,Energy,1.0 -solar/D/short,TEMPO_ENSEMBLE,227997.78199513385,227997.78199513385,364.8942746350365,1.1822038893743294,1.1243099463247035,0.5479662967134277,26.830195352743505,477.4911328968673,0.6898120741982009,0.5271479596321818,0.4800547635005184,Energy,1.0 -solar/H/long,TEMPO_ENSEMBLE,2874.481508515815,2874.481508515815,33.32977699842863,2.5475516817625423,6.921687556510294,1.6789198232683242,20.71382607291551,53.61419129778808,1.858901945988148,1.1556042499569716,0.9908022974231144,Energy,1.0 -solar/H/medium,TEMPO_ENSEMBLE,3328.179333941606,3328.179333941606,27.74560725079836,2.1192776144205463,0.982378255220553,1.972083945169936,13.399058473722926,57.69037470793205,2.0781418580377284,0.9994614889653576,0.7343401482401601,Energy,1.0 -solar/H/short,TEMPO_ENSEMBLE,406.0480185122935,406.0480185122935,9.035267002742229,0.6971424677787719,2.1446248619803527,1.4069967350253358,7.611412271966008,20.150633203755497,0.7427590622937399,0.3330429559543392,0.2607230327213033,Energy,1.0 -solar/W/short,TEMPO_ENSEMBLE,25620636.96350365,25620636.96350365,4588.206660583942,59.20451480380479,0.9345781089615648,1.7547384193792417,517.900837376737,5061.683214455805,1.0332160923962808,0.9365676902529754,0.6280375794904386,Energy,1.0 -us_births/D/short,TEMPO_ENSEMBLE,186916.96,186916.96,239.07765625,0.2262047828348501,0.0234775368372599,0.0226943778991699,2.5604752821394974,432.3389411098658,0.0405282675008943,0.0224115902701547,0.0185739392260598,Healthcare,1.0 -us_births/M/short,TEMPO_ENSEMBLE,145867776.0,145867776.0,9889.141927083334,1.2073705555270309,0.0316123714049657,0.0308461654110728,8.33813811617069,12077.573266182242,0.0375128602209077,0.0307156074022029,0.0255509986968776,Healthcare,1.0 -us_births/W/short,TEMPO_ENSEMBLE,21440150.85714286,21440150.85714286,4361.426897321428,3.115257411522497,0.0587949837957109,0.0607828675922443,41.82710839191516,4630.351051177746,0.0628565614933249,0.0592059424739289,0.0511111792570683,Healthcare,1.0 \ No newline at end of file diff --git a/results/tempo_ensemble/config.json b/results/tempo_ensemble/config.json deleted file mode 100644 index f1a575b7b2f64b7e04fc4884a8a8180a4612d0f3..0000000000000000000000000000000000000000 --- a/results/tempo_ensemble/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "model": "TEMPO_ENSEMBLE", - "model_type": "fine-tuned", - "model_dtype": "float32", - "model_link": "https://huggingface.co/Melady/TEMPO", - "org": "Melady Lab @ USC", - "testdata_leakage": "Yes", - "replication_code_available": "No" -} \ No newline at end of file diff --git a/results/tft/config.json b/results/tft/config.json index 505b01fb92d6242128af25ddd4aafdb6561c4635..d5357d9b333d4c161a005369bd344ad93e2e574f 100644 --- a/results/tft/config.json +++ b/results/tft/config.json @@ -1,8 +1,5 @@ { "model": "TFT", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "Google Research", - "testdata_leakage": "No", - "replication_code_available": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/tide/config.json b/results/tide/config.json index 65a557d5d614ca4afb6ff184402571e915f2a852..678beca2330d8f0601496cebca4e9c3421e5acd4 100644 --- a/results/tide/config.json +++ b/results/tide/config.json @@ -1,8 +1,5 @@ { "model": "TIDE", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "Google Research", - "testdata_leakage": "No", - "replication_code_available": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/timer_small/config.json b/results/timer_small/config.json new file mode 100644 index 0000000000000000000000000000000000000000..05b27811e7cca7b1474dd39296a6a109d9747557 --- /dev/null +++ b/results/timer_small/config.json @@ -0,0 +1,5 @@ +{ + "model": "timer_small", + "model_type": "pretrained", + "model_dtype": "float32" +} \ No newline at end of file diff --git a/results/timesfm/config.json b/results/timesfm/config.json index bb746e4410fa5efedcd21ceae02f1017c433bbc6..8e553d7324a1dcd00a4cfec4514f31da50434568 100644 --- a/results/timesfm/config.json +++ b/results/timesfm/config.json @@ -2,9 +2,5 @@ "model": "TimesFM", "model_type": "pretrained", "model_dtype": "float32", - "model_link": "https://huggingface.co/google/timesfm-1.0-200m", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/timesfm.ipynb", - "org": "Google Research", - "testdata_leakage": "Yes", - "replication_code_available": "Yes" + "model_link": "https://huggingface.co/google/timesfm-1.0-200m" } \ No newline at end of file diff --git a/results/timesfm_2_0_500m/config.json b/results/timesfm_2_0_500m/config.json index 6359c579128139a6176c0a033ec08e044b51984b..8ef72136c33397d0cd202892c4270a690552c3c5 100644 --- a/results/timesfm_2_0_500m/config.json +++ b/results/timesfm_2_0_500m/config.json @@ -3,8 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/google/timesfm-2.0-500m-jax", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/timesfm.ipynb", - "org": "Google Research", - "testdata_leakage": "Yes", - "replication_code_available": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/timesfm.ipynb" } \ No newline at end of file diff --git a/results/visionts/config.json b/results/visionts/config.json index 6ff95013ba7ddd7d13dc23b9248bcd76a26571e0..ca0ac5b4842d868567d6c623a0d63a27831b1bee 100644 --- a/results/visionts/config.json +++ b/results/visionts/config.json @@ -1,9 +1,6 @@ { "model": "VisionTS", - "model_type": "zero-shot", + "model_type": "pretrained", "model_dtype": "float32", - "model_link": "https://github.com/Keytoyze/VisionTS", - "org": "Zhejiang University", - "testdata_leakage": "No", - "replication_code_available": "No" + "model_link": "https://github.com/Keytoyze/VisionTS" } \ No newline at end of file diff --git a/results/xLSTM-Mixer/all_results.csv b/results/xLSTM-Mixer/all_results.csv deleted file mode 100644 index 293be0aca83d40165038e125d5e0a7a984e60b63..0000000000000000000000000000000000000000 --- a/results/xLSTM-Mixer/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -car_parts/M/short,xLSTM-Mixer,1.3090908848243168,1.3090908848243168,0.5030945503839984,0.9024433846189117,0.7795546352239273,1.8606200863739497,13.504348002767681,1.1441550964901204,2.7435665534505054,1.2063691241606742,1.0582473276320692,Sales,1 -hierarchical_sales/W/short,xLSTM-Mixer,420.0871540651483,420.0871540651483,9.041981066687633,0.7478319384022623,0.5435999916207379,0.476607718710172,8.96592367888471,20.496027763084932,0.9409826599560921,0.41512177353353313,0.3601043843307868,Sales,1 -hierarchical_sales/D/short,xLSTM-Mixer,28.747154040926002,28.747154040926002,2.3314967243372777,0.7647522867228026,0.6245550960250786,1.0358640660526044,7.701123344877241,5.361637253761765,1.6448909329033607,0.7152773752580089,0.5883420864482761,Sales,1 -us_births/W/short,xLSTM-Mixer,2238128.0,2238128.0,1194.7433035714287,1.0860558207591045,0.016112481909138814,0.01617054428373064,47.66901438906672,1496.0374326867627,0.020308561455631573,0.0162185231961137,0.013132416030881824,Healthcare,1 -us_births/D/short,xLSTM-Mixer,105532.66666666667,105532.66666666667,212.5298828125,0.31346398350396215,0.020483808517456056,0.020321019490559897,2.964296475487072,324.85791766042377,0.030452793710709143,0.019922951933145163,0.01626929874381338,Healthcare,1 -us_births/M/short,xLSTM-Mixer,14875248.0,14875248.0,3149.4518229166665,0.3561632365498489,0.009756327917178472,0.009754804273446402,2.6502980639038314,3856.84430590606,0.011979332118511946,0.009782175889287908,0.0077304704818990135,Healthcare,1 -solar/W/short,xLSTM-Mixer,1038108.0656934306,1038108.0656934306,794.3018718635949,0.8661709014343238,0.1718543741824853,0.1592843445548176,10.512846205039212,1018.8758833603977,0.2079780409481077,0.16213687057422635,0.13635783881656743,Energy,1 -solar/D/short,xLSTM-Mixer,104918.42335766423,104918.42335766423,237.68281820255476,0.9275265555315503,0.9610215681312728,0.4184002247459987,5.108782060118507,323.91113497017085,0.4679412799021066,0.34337072780991507,0.267636083269292,Energy,1 -solar/10T/short,xLSTM-Mixer,21.49350600787903,21.49350600787903,2.012616534824789,0.8722258158647247,4.9797367400971035,1.5173966660696805,7.919026116430207,4.6361089296822,1.3486435622806805,0.585469921910822,0.4925914504709459,Energy,1 -solar/10T/medium,xLSTM-Mixer,21.166719631303916,21.166719631303916,1.9675955458727328,0.856219133882744,4.343284939753427,1.4492305979215052,5.136863483174339,4.600730336729584,1.0209203670260816,0.4366172802640708,0.33960620669945707,Energy,1 -solar/10T/long,xLSTM-Mixer,21.288989000405515,21.288989000405515,1.9168527125152068,0.8338748282584149,4.39608522299586,1.443000160622162,4.8452574803847845,4.61399924148298,0.9972421137598337,0.4142970448734928,0.32548706098839186,Energy,1 -solar/H/short,xLSTM-Mixer,695.2913265142784,695.2913265142784,11.550356434682739,0.8879679036607947,5.056878523214533,1.4058384080171809,4.973563273725097,26.368377396310876,0.9719471740768357,0.425749985580862,0.3318150323843736,Energy,1 -solar/H/medium,xLSTM-Mixer,820.4154577250608,820.4154577250608,12.244652310485098,0.9284325898365615,6.236494850712302,1.4073526943977153,5.43794401901163,28.642895414483863,1.0317839721683277,0.4410809667078507,0.34356307317551305,Energy,1 -solar/H/long,xLSTM-Mixer,818.4104141321978,818.4104141321978,13.421718338148825,1.0216348018249524,9.413687275501765,1.4159999132736465,5.231314824112916,28.607873289222283,0.9918872231522707,0.46535549139802074,0.3609977557032887,Energy,1 -electricity/D/short,xLSTM-Mixer,2513564012.820757,2513564012.820757,5465.714315315316,1.7652480380227566,0.5833048407595778,0.12579410862278295,21.063645960503248,50135.45664318574,0.8238920549233522,0.08981983810216625,0.07405474399175199,Energy,1 -electricity/W/short,xLSTM-Mixer,86244020675.11351,86244020675.11351,35075.457713963966,1.7442723163225893,0.25734118182232324,0.11996274295153919,18.42253105066406,293673.3230566125,0.6700656148331801,0.08003062005782438,0.06442243013558806,Energy,1 -electricity/15T/short,xLSTM-Mixer,184008.42838154561,184008.42838154561,67.09345579954955,1.250187094226151,0.2224599573150583,0.20399247813869167,16.99052591573704,428.96203606093815,0.8455687500068096,0.13225442996079706,0.11377769290988418,Energy,1 -electricity/15T/long,xLSTM-Mixer,362264.96041385137,362264.96041385137,65.66408072916667,1.0163204415191907,0.19181273710588514,0.1612049482649153,9.699410311785527,601.8845075376599,0.949546617217392,0.10359313946150812,0.08562628560270961,Energy,1 -electricity/H/long,xLSTM-Mixer,6933240.951687688,6933240.951687688,295.15791854354353,1.4006503061168776,0.4040925314806805,0.1771835761542793,14.007471593488733,2633.104812134847,1.0384957757074669,0.11641019763492079,0.09542343210415181,Energy,1 -electricity/15T/medium,xLSTM-Mixer,264585.66808502254,264585.66808502254,60.034273595861485,0.9849085368863463,0.18141690625490062,0.16187459859762107,9.65320766448979,514.3789148915637,0.8863266456116942,0.103445096207281,0.085440459423351,Energy,1 -bitbrains_fast_storage/5T/short,xLSTM-Mixer,1636964.9417956087,1636964.9417956087,166.33659605757177,1.1520584286888411,2.381287625480112,0.7303779723535063,19.008111422691908,1279.439307585791,4.017122975639783,0.5222557707511892,0.40656882761444024,Web/CloudOps,2 -electricity/H/short,xLSTM-Mixer,1492514.3456024774,1492514.3456024774,181.83079484269425,1.037050593551154,0.31550757726688566,0.14598514496743142,13.26915314160423,1221.6850435371948,0.5767225851467638,0.08583712030831356,0.07303304395002409,Energy,1 -electricity/H/medium,xLSTM-Mixer,6383668.832601352,6383668.832601352,268.97976932010135,1.2442632846161077,0.33887354630830324,0.16034637605821764,12.469510972038972,2526.59233605292,0.9862122469837552,0.10499166759476514,0.08709879257768377,Energy,1 -hospital/M/short,xLSTM-Mixer,3073.9198989569754,3073.9198989569754,19.392369104125923,0.8142524412031868,0.22299956612253335,0.18626777320672822,16.87909253701953,55.44294273356146,0.20127075675972972,0.07039880303086328,0.06286870305570816,Healthcare,1 -bitbrains_fast_storage/5T/medium,xLSTM-Mixer,2542755.716979079,2542755.716979079,270.8545370802788,1.604290613469785,3.408032405522072,0.896536373201042,43.35690387220296,1594.6020559936196,4.844841302223767,0.8229308642923787,0.6367968069166179,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,xLSTM-Mixer,3228637.9913132,3228637.9913132,323.20629088756283,1.6800679267488197,2.9826060807407258,0.9345967249405688,23.866486025418816,1796.841114654604,4.748514510359813,0.8541377140149549,0.7180349660655163,Web/CloudOps,2 -bitbrains_fast_storage/H/short,xLSTM-Mixer,2346232.6759375883,2346232.6759375883,278.7959850996924,1.5445710156771428,4.185333759281022,0.6561348067261575,25.902509480791444,1531.7417131937057,4.365961477554519,0.794659125985551,0.5798266081420145,Web/CloudOps,2 -sz_taxi/15T/short,xLSTM-Mixer,17.555792243780527,17.555792243780527,2.842215513571715,0.5695924314291132,884907313506.7827,0.4126540243407309,4.408793544302494,4.189963274753196,0.3917794995703419,0.2657593154307013,0.2104269288947753,Transport,1 -sz_taxi/15T/medium,xLSTM-Mixer,17.462999131944443,17.462999131944443,2.88266846746461,0.5645964671300953,11198342566833.963,0.4146930009890825,4.330537482677258,4.178875342953465,0.38891408726887183,0.26828040654842783,0.21586544081459094,Transport,1 -sz_taxi/15T/long,xLSTM-Mixer,18.289436431623933,18.289436431623933,3.0750399945468305,0.5836751384479797,8897144058312.23,0.4375622262981882,8.583760334489929,4.276615067038876,0.3958358522101578,0.28462020961464723,0.23002680866332056,Transport,1 -bizitobs_l2c/5T/short,xLSTM-Mixer,15.560067894345238,15.560067894345238,2.3272341773623513,0.24059934666267835,0.12059001633346875,0.19167442321777345,2.46497079321447,3.9446251906036953,0.13547439568313632,0.079926641583672,0.06637708011275892,Web/CloudOps,7 -bizitobs_l2c/5T/medium,xLSTM-Mixer,48.84644982993197,48.84644982993197,4.446133609693877,0.45769994987371426,0.3661583337858384,0.594612962372449,3.8808087054099767,6.989023524780266,0.3677254154453787,0.23393201109617745,0.19054911945904462,Web/CloudOps,7 -bizitobs_l2c/5T/long,xLSTM-Mixer,54.59717261904762,54.59717261904762,4.818187934027778,0.4898392579666989,0.38523270226878176,0.6863716827876984,3.7435611398191537,7.388989959327839,0.4148807396564253,0.2705340492917587,0.21779808389890243,Web/CloudOps,7 -bizitobs_l2c/H/short,xLSTM-Mixer,51.18578714037698,51.18578714037698,4.709080771794395,0.48737246832138004,0.5542949795437088,0.6256941538008433,3.7892921976000515,7.154424305307659,0.38564183313094685,0.25383154027490834,0.1957944989397356,Web/CloudOps,7 -bizitobs_l2c/H/medium,xLSTM-Mixer,67.6580822172619,67.6580822172619,5.17023693266369,0.520790295098905,0.6284847024607668,0.7588110060918899,4.526117542149557,8.22545331378532,0.49806369189582655,0.3130656994117177,0.2521358462238879,Web/CloudOps,7 -sz_taxi/H/short,xLSTM-Mixer,7.6962582840878735,7.6962582840878735,1.9220321524856436,0.5868201434965089,0.9948896994957557,0.30888866359352046,4.710455145351472,2.774213092768447,0.25843507522859455,0.1790491599993556,0.14393056592502423,Transport,1 -bizitobs_l2c/H/long,xLSTM-Mixer,97.78001612103175,97.78001612103175,7.0328272259424605,0.7465758454451291,1.1097633651439807,0.7991431826636904,9.316347817493059,9.888377830616696,0.6040106575099884,0.42958538494991233,0.34273832732556736,Web/CloudOps,7 -jena_weather/D/short,xLSTM-Mixer,410.2938988095238,410.2938988095238,10.982314143105159,1.7805581297777973,1.6074673157254653,0.5282246907552083,22.0360552501372,20.255712745038714,0.12197249715252424,0.06613148090165911,0.05856120990215999,Nature,21 -jena_weather/10T/short,xLSTM-Mixer,841.9650793650794,841.9650793650794,6.424024987599206,0.32878954248288295,0.4688900231414494,0.5628389873201884,4.042545629251149,29.016634528578248,0.18022968147332957,0.03990124961429993,0.03454537611253848,Nature,21 -jena_weather/10T/medium,xLSTM-Mixer,1276.2103896103897,1276.2103896103897,9.98548318001443,0.6989418553410285,1.107985103057808,0.6657796576253607,6.7406879620158895,35.724086966784604,0.2192380867314258,0.06128073278711535,0.050746873778934415,Nature,21 -jena_weather/10T/long,xLSTM-Mixer,1339.110582010582,1339.110582010582,10.029196635251322,0.7034930774507729,1.0886829494098325,0.653090251942791,8.303305290825541,36.59385989494115,0.22392410350000225,0.06137037393216765,0.05238509358696487,Nature,21 -jena_weather/H/short,xLSTM-Mixer,1055.654344193818,1055.654344193818,8.934890122441521,0.5765735293927284,1.7051230786654374,0.6299955516231986,6.091493980595372,32.490834772191036,0.19916674143404967,0.05477030563342313,0.04549523479105985,Nature,21 -jena_weather/H/medium,xLSTM-Mixer,1184.2864087301587,1184.2864087301587,11.053494698660714,0.8473068876486645,3.0987606977902553,0.6784102182539683,8.908747612176624,34.41346260884189,0.21058206505806656,0.06763828930002277,0.05488695889948214,Nature,21 -jena_weather/H/long,xLSTM-Mixer,1267.2126984126985,1267.2126984126985,14.213711144179895,1.223430584015947,7.9024375503626105,0.7142744760664682,22.104181647151897,35.59793109736433,0.214339006406447,0.085582297343604,0.07002697297126292,Nature,21 -m4_yearly/A/short,xLSTM-Mixer,3227819.7871216736,3227819.7871216736,814.8882964506072,2.983798263240972,0.16014203230747512,0.1330535787163579,38.38787934144634,1796.6134217247943,0.28809528908893434,0.13067111516719393,0.11214969271919809,Econ/Fin,1 -m4_hourly/H/short,xLSTM-Mixer,1077932.1222826086,1077932.1222826086,263.8721336082176,3.344093665626611,0.4963328657902765,0.25398920584416046,25.488007568623914,1038.2350997161523,0.14174202655669532,0.03602437538443198,0.030174955223969353,Econ/Fin,1 -bizitobs_service/10S/short,xLSTM-Mixer,4420.350740740741,4420.350740740741,24.010808531746033,1.3294919144068744,0.2879023847630415,0.10227424581214864,14.817229875016531,66.48571832161205,0.049254746592970265,0.017787974915193846,0.015015403812166863,Web/CloudOps,2 -m4_monthly/M/short,xLSTM-Mixer,1917944.1969560185,1917944.1969560185,571.3057212999132,0.9784021264040583,0.16182981106970046,0.13333318744765388,10.404589434909356,1384.8986233497449,0.28782947602392994,0.11873694120186856,0.10055737351673073,Econ/Fin,1 -bizitobs_service/10S/medium,xLSTM-Mixer,34332.75174603175,34332.75174603175,42.534553571428575,1.2238973033227636,0.17751929873511904,0.10111889067150298,19.8006897925698,185.29099208011098,0.13945435038226991,0.03201250352493594,0.02885613529906213,Web/CloudOps,2 -bitbrains_rnd/5T/short,xLSTM-Mixer,1216118.1248422898,1216118.1248422898,113.44668822614177,2.014678469300612,1.1416828731117952,0.6830046939080144,60.24823612894399,1102.7774593463043,4.512247035387541,0.46419110064725355,0.3951803446757601,Web/CloudOps,2 -bizitobs_service/10S/long,xLSTM-Mixer,130594.02497354498,130594.02497354498,73.69123677248677,1.380347455122025,0.16322871455439814,0.11429633246527778,27.68302186744627,361.3779530817354,0.2677203406057215,0.05459282405068263,0.0518160513269081,Web/CloudOps,2 -bitbrains_rnd/5T/medium,xLSTM-Mixer,2117319.067788678,2117319.067788678,159.405586056821,5.16757957957734,2.570218004064394,0.8558369832038596,180.81512457179005,1455.101050713894,6.122352061547271,0.6707005798245557,0.6134957852154741,Web/CloudOps,2 -bitbrains_rnd/5T/long,xLSTM-Mixer,2230679.6638839343,2230679.6638839343,163.8518266310325,3.768747895664321,0.8302652052752273,0.7833970106671007,125.37750041397204,1493.546003270048,5.721438371831344,0.6276794461830465,0.6204646680442245,Web/CloudOps,2 -bitbrains_rnd/H/short,xLSTM-Mixer,1639580.206811304,1639580.206811304,144.6232997048293,6.181453521825273,1.1070171854423583,0.61844212744355,221.76809604340917,1280.460935292953,5.603923525678548,0.6329423172849313,0.5891727380150137,Web/CloudOps,2 -kdd_cup_2018/D/short,xLSTM-Mixer,2627.1230529789214,2627.1230529789214,19.88271243464908,1.1764799532187793,0.5731247194793537,0.44922817982814994,11.797385689287214,51.25546851779741,1.148176950940339,0.4453938828344158,0.3687901889245506,Nature,1 -kdd_cup_2018/H/short,xLSTM-Mixer,4731.34205080619,4731.34205080619,24.695972944779733,1.0212562281066062,1.1060074768574013,0.5258321428853638,8.256752499038733,68.78475158642495,1.4397894246594032,0.5169314398539744,0.4197649308293398,Nature,1 -kdd_cup_2018/H/medium,xLSTM-Mixer,4951.027078497365,4951.027078497365,24.71140263056566,1.034879154564373,1.052541747160774,0.548794621321053,8.803356099819824,70.36353514781193,1.4728361784404789,0.5172543951614987,0.418684208307097,Nature,1 -kdd_cup_2018/H/long,xLSTM-Mixer,3996.9082725962467,3996.9082725962467,24.002929395700164,1.0225517908996742,1.1429416240587185,0.6053678530553709,8.492814208069538,63.22110622724223,1.4834225900403444,0.5632057048280829,0.4545284330441431,Nature,1 -ett2/W/short,xLSTM-Mixer,9730923.42857143,9730923.42857143,2382.190848214286,1.8128512051280425,0.2981648785727365,0.4047889028276716,149.1972229121539,3119.4428073890745,0.2616733785705616,0.1998292535370582,0.18963874168927936,Energy,7 -ett2/D/short,xLSTM-Mixer,96646.5142857143,96646.5142857143,186.59202628968254,1.1890798049701246,0.35495469835069443,0.1239653814406622,12.325586535665433,310.8802249833757,0.17792493485146071,0.10679152758323289,0.08515793861692439,Energy,7 -ett2/15T/short,xLSTM-Mixer,7.285466657366071,7.285466657366071,1.7422462100074405,0.7833321186243325,0.11542925127165166,0.13654314676920573,5.686020647325815,2.6991603615506197,0.12813874450672397,0.08271062555308825,0.06552077625915878,Energy,7 -ett2/15T/long,xLSTM-Mixer,10.796207837301587,10.796207837301587,2.134755549355159,0.9062246192806765,0.13930569887857977,0.16731453062996032,7.276305694178737,3.2857583351947213,0.17510399776450805,0.11376497989463501,0.09161415351501066,Energy,7 -ett2/H/short,xLSTM-Mixer,105.99518229166667,105.99518229166667,6.506583658854167,0.7697677587924242,0.12794113216039088,0.11603220985049294,5.443533319605859,10.29539616972881,0.13057426937692432,0.08252158473442181,0.06526108440753636,Energy,7 -ett2/15T/medium,xLSTM-Mixer,10.155925719246031,10.155925719246031,2.0033590649801587,0.8535649263532894,0.1334195180557095,0.15942881750682045,6.435274776505964,3.1868363182388317,0.16983226476748037,0.106762623860164,0.0853872546601013,Energy,7 -loop_seattle/5T/short,xLSTM-Mixer,46.234683394456674,46.234683394456674,3.979713189564991,0.6280774623593094,0.11013029084240074,0.0837703872145268,6.432636704508227,6.799609061884122,0.11659828240214071,0.06824329430313517,0.05623182251112598,Transport,1 -covid_deaths/D/short,xLSTM-Mixer,319485.38394423557,319485.38394423557,99.48731080631266,37.475268047521475,0.10472297530901238,0.3556498993608288,745.9998892248128,565.2303812997277,0.212670671664027,0.03743258308687187,0.03145338102540153,Healthcare,1 -ett2/H/medium,xLSTM-Mixer,230.36264880952382,230.36264880952382,9.451364281063988,1.048217053594007,0.18224784984601516,0.17091166178385417,7.7805994377758605,15.177702356072338,0.2045292868462401,0.12736320365095655,0.10329934076658956,Energy,7 -ett2/H/long,xLSTM-Mixer,222.31939484126985,222.31939484126985,9.517354910714285,1.0728884025023557,0.18511503179196184,0.1813831470630787,8.413443111542362,14.910378762501972,0.20418616127743966,0.13033285040491155,0.1048352725044332,Energy,7 -loop_seattle/D/short,xLSTM-Mixer,18.28954035047407,18.28954035047407,2.9440480786334375,0.8901862864085156,0.05480469312707214,0.0542823314543725,9.350008711597297,4.276627216683034,0.0764290403699412,0.05261407133527105,0.045194089025068586,Transport,1 -loop_seattle/H/short,xLSTM-Mixer,70.69584849283376,70.69584849283376,4.757005374621065,0.9625821438653933,0.14021704831599127,0.10836229339362577,10.059589761219568,8.408082331473317,0.14883160755896108,0.08420383259346749,0.07045754646043072,Transport,1 -loop_seattle/5T/medium,xLSTM-Mixer,81.23696654895511,81.23696654895511,5.380417266310307,0.8536972808707037,0.17728535769278542,0.12452499275483091,7.930460991016367,9.013155193879395,0.16036255078607087,0.09572867864350727,0.07854159603447974,Transport,1 -loop_seattle/5T/long,xLSTM-Mixer,89.57888089811948,89.57888089811948,5.629618270553836,0.8894461519240908,0.17903673477968948,0.1281369680976146,8.302668996472567,9.464612031040653,0.16733628565877315,0.09953280789341198,0.08198939836411206,Transport,1 -loop_seattle/H/medium,xLSTM-Mixer,65.19003312905137,65.19003312905137,4.531136485002358,0.927992208800648,0.12588265394893838,0.10385742477834287,11.449786206100526,8.074034501353792,0.14270765617576162,0.08008733025338745,0.06840652225880374,Transport,1 -loop_seattle/H/long,xLSTM-Mixer,61.39205857661732,61.39205857661732,4.3090293272771,0.9015355642043574,0.12391147190326261,0.10045535107597191,12.631843944302252,7.835308454465422,0.14002743121923172,0.07700811158277665,0.0666412034534923,Transport,1 -m4_daily/D/short,xLSTM-Mixer,500588.84871489403,500588.84871489403,179.79147996167282,3.2631571597973696,0.043145434304625134,0.030544646799175578,31.230266645136627,707.5230375859813,0.10929275497826389,0.027772814626184677,0.023078183313523095,Econ/Fin,1 -m_dense/H/short,xLSTM-Mixer,40281.74,40281.74,89.062265625,0.8285298743793105,0.29825591136617696,0.2382666778564453,12.293964874872863,200.70311407648862,0.355997339767915,0.15797427848638257,0.13315976867435197,Transport,1 -m_dense/D/short,xLSTM-Mixer,7670.885925925926,7670.885925925926,42.489496527777774,0.6507734042769915,0.11686196408508617,0.09426957872178819,10.009986878795248,87.58359393131757,0.15162015640572493,0.07355560350942153,0.06223312592356331,Transport,1 -m_dense/H/long,xLSTM-Mixer,33357.5387654321,33357.5387654321,79.96048611111111,0.7030953943415768,0.26003922214116537,0.20783845124421296,8.910814262419013,182.64046311108638,0.3161776408152081,0.13842342176755965,0.11495745540617948,Transport,1 -m_dense/H/medium,xLSTM-Mixer,33243.05777777778,33243.05777777778,78.14989583333333,0.6937738736588148,0.27164471402798585,0.21129723442925347,8.801154425032411,182.32678842610534,0.3172255773191902,0.13597094555971553,0.11293215361169555,Transport,1 -m4_weekly/W/short,xLSTM-Mixer,258752.74051853438,258752.74051853438,261.81233929719303,2.004220154921183,0.06487076691700454,0.06406637175016237,17.780274944230708,508.67744251001966,0.09267313783436212,0.04769814617039488,0.03911820108437382,Econ/Fin,1 -restaurant/D/short,xLSTM-Mixer,140.96555699542355,140.96555699542355,7.174122293133289,0.69101470610006,0.649298804274146,0.3961392473281571,4.849169147123154,11.87289168633419,0.546417984618738,0.330169730204335,0.26131343247403316,Sales,1 -bizitobs_application/10S/long,xLSTM-Mixer,3151556.2666666666,3151556.2666666666,1169.9818055555556,3.9545968667499527,0.0494436772664388,0.05176664564344618,66.5900531472528,1775.2623092564847,0.06846477134177621,0.04512152168934862,0.03987077609994868,Web/CloudOps,2 -bizitobs_application/10S/short,xLSTM-Mixer,512826.24,512826.24,380.80857638888887,1.5025243369544496,0.02111683103773329,0.021420035892062716,27.850944290258987,716.1188728137249,0.02761784251548521,0.014686264655938119,0.011978679177306021,Web/CloudOps,2 -bizitobs_application/10S/medium,xLSTM-Mixer,2426919.466666667,2426919.466666667,880.2226041666667,3.2618337949051943,0.03981379191080729,0.04164868036905924,81.63875759993877,1557.8573319359725,0.06117247237703806,0.03456375101570787,0.028891268181794057,Web/CloudOps,2 -saugeen/W/short,xLSTM-Mixer,950.322265625,950.322265625,16.525418090820313,1.3409040637059948,0.47144784927368166,0.45189313888549804,21.50565783762255,30.827297410330996,0.9305476060094254,0.4988334863751061,0.40210013559063534,Nature,1 -saugeen/D/short,xLSTM-Mixer,1171.990625,1171.990625,13.817360026041667,3.0684622893783238,0.3590826924641927,0.3773080444335937,40.00531942125322,34.23434861363657,1.1088587084570842,0.44754758344255724,0.39030066663742413,Nature,1 -m4_quarterly/Q/short,xLSTM-Mixer,1812414.5575833332,1812414.5575833332,561.2061874186198,1.1735510071916742,0.11864607830842336,0.10101981536547343,13.57946077699553,1346.2594688927293,0.22534012779173776,0.09393603306979027,0.08079329328964445,Econ/Fin,1 -saugeen/M/short,xLSTM-Mixer,339.25083705357144,339.25083705357144,11.658219837007069,0.6831756575357362,0.35995242709205266,0.3404898416428339,4.701048536810744,18.41876317925749,0.5529865264133227,0.3500147338388941,0.2703253620017059,Nature,1 -ett1/W/short,xLSTM-Mixer,1248640.0,1248640.0,831.8988560267857,1.3857290212914262,0.66413300377982,0.4832890374319894,25.701124931168685,1117.4256127367048,0.4446503217419151,0.3310324103660944,0.24920141158936474,Energy,7 -ett1/D/short,xLSTM-Mixer,38641.67619047619,38641.67619047619,130.80813492063493,1.6459486486799015,2.1649615575396823,0.44283001612103173,16.183484975040148,196.57486154255886,0.5163103942963043,0.3435719053192117,0.2789415821497968,Energy,7 -ett1/15T/medium,xLSTM-Mixer,8.513025173611112,8.513025173611112,1.5807660590277777,1.0425206274245766,0.7960759501928468,0.3994483584449405,7.346183999211945,2.917708891169767,0.5561181337979784,0.30129553821432375,0.23562340926385236,Energy,7 -ett1/15T/short,xLSTM-Mixer,4.313387044270834,4.313387044270834,1.0920971098400298,0.7400023200491433,0.4677413749581224,0.24499003092447916,5.401746123521267,2.0768695299105415,0.39261474802921803,0.20645178978659745,0.16148149591007746,Energy,7 -ett1/15T/long,xLSTM-Mixer,8.14642361111111,8.14642361111111,1.6059824528769842,1.076763661003786,0.8246409906260562,0.413135501922123,7.569054712130197,2.854194038798188,0.5440122058582525,0.3061018434216659,0.2369236144216448,Energy,7 -ett1/H/long,xLSTM-Mixer,141.1986441798942,141.1986441798942,7.272064112103174,1.4142776163425466,3986046397069.0464,0.4677515423487103,11.529211523173219,11.882703571994641,0.5690369379136653,0.3482434001312402,0.2772902462400328,Energy,7 -ett1/H/short,xLSTM-Mixer,89.4819568452381,89.4819568452381,4.986718168712797,0.8630924177634672,0.4580759684244792,0.2685322897774833,5.820611403571445,9.459490305784879,0.44151311294873524,0.2327505384428556,0.1814527338247582,Energy,7 -ett1/H/medium,xLSTM-Mixer,143.36238839285716,143.36238839285716,6.97075427827381,1.3257292358753034,5364981424595.539,0.4497766403924851,9.479215361758744,11.973403375517638,0.5734976388347032,0.33388260581458157,0.2664280701496993,Energy,7 -temperature_rain/D/short,xLSTM-Mixer,186.71643253113578,186.71643253113578,6.1176918986436,1.4611967688290612,56.44753267605158,1.4652979163596815,21.726682011857744,13.66442214406214,1.6086637648399829,0.7202140842874653,0.5962630474528218,Nature,1 diff --git a/results/xLSTM-Mixer/config.json b/results/xLSTM-Mixer/config.json deleted file mode 100644 index 2614a0e75aeabdf3e5b0db9c791033c16b40f6b5..0000000000000000000000000000000000000000 --- a/results/xLSTM-Mixer/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "model": "xLSTM-Mixer", - "model_type": "deep-learning", - "model_dtype": "float32", - "model_link": "https://github.com/mauricekraus/xlstm-mixer", - "org": "AIML Lab @ TU Darmstadt", - "testdata_leakage": "No", - "replication_code_available": "No" -} \ No newline at end of file diff --git a/src/about.py b/src/about.py index d23041a4ff515b67c699d12ae49d9655f41e525d..d12d4e1419206f58a37f82ac704ddbd4812fbb03 100644 --- a/src/about.py +++ b/src/about.py @@ -42,24 +42,7 @@ points, spanning seven domains, 10 frequencies, multivariate inputs, and predict # Which evaluations are you running? how can people reproduce what you have? LLM_BENCHMARKS_TEXT = f""" -## Update Log - -### 2025-10-17 -- Added new column: Replication Code to indicate whether the model' evaluation replication code is made available. This column is a binary indicator specifying whether the evaluation code is made available to the public by the submission author. The preferable way to share the evaluation code is to share a notebook in the GIFT-Eval github repository (as many previous submissions have done), but a standalone repo for the evaluation code is also acceptable as long as it is accessible to the public and the link is provided in the config.json file through the `code_link` field. - -### 2025-08-25 -- Added new model type: Zero-shot to distinguish between foundation model submissions that don't use training data of GIFT-Eval. Now models tagged with zero-shot indicate that the model is not trained on the GIFT-Eval training data. Test data leakage is still separately tracked with the TestData Leakage column. For a model be tagged as `zero-shot`, it must both not have test data leakage and not use any training split from GIFT-Eval. - -### 2025-08-05 -- Added new columns to the leaderboard: Organization, TestData Leakage, and MASE_Rank. TestData Leakage is a binary indicator specifying whether any test data was present in the training set. MASE_Rank reflects the model's ranking based on the MASE metric, aligned with the ranking scheme used for CRPS_Rank. These additions were made in response to multiple requests from independent groups seeking fairer comparisons. With these updates, the leaderboard now supports sorting by models that do not leak test data, and viewers can choose to rank models based on either MASE_Rank or CRPS_Rank, depending on their use case. -- Added new model type: Agentic to indicate submissions that use agentic system to generate the forecasts. - - -### 2025‑07‑24 -- Corrected the Naive and Seasonal Naive scores to match the latest GIFT‑Eval notebooks. Most model rankings remain unchanged; only a few near the bottom shifted slightly (AutoETS and Timer each dropped two places now at 35th and 36th places respectively, while NBEATS moved up one now at 27th place). - - -## How It Works +How It Works To participate in the GIFT-Eval leaderboard, follow these steps to evaluate your Time Series Model: @@ -78,6 +61,7 @@ cd gift-eval Follow the instruction in the README.md file to install the required dependencies, set up your environment and obtain the evaluation results. + """ EVALUATION_QUEUE_TEXT = """ diff --git a/src/display/utils.py b/src/display/utils.py index 4e5e1794373f72b11f40d6c4a0c1c1a528ce90e3..85f2cdb969c269c3b61dbbc3044d6029bfc66c7b 100644 --- a/src/display/utils.py +++ b/src/display/utils.py @@ -34,9 +34,6 @@ model_info_dict.append(["license", ColumnContent, ColumnContent("Hub License", " model_info_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False, True)]) model_info_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False, True)]) model_info_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)]) -model_info_dict.append(["org", ColumnContent, ColumnContent("Organization", "str", True, hidden=False)]) -model_info_dict.append(["testdata_leakage", ColumnContent, ColumnContent("Test Leak.", "str", True, hidden=False)]) -model_info_dict.append(["replication_code_available", ColumnContent, ColumnContent("Replication Code", "str", True, hidden=False)]) # model_info_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)]) # We use make dataclass to dynamically fill the scores from Tasks @@ -62,12 +59,9 @@ class ModelDetails: class ModelType(Enum): PT = ModelDetails(name="🟢 pretrained", symbol="🟢") - ZT = ModelDetails(name="🔴 zero-shot", symbol="🔴") - FT = ModelDetails(name="🟣 fine-tuned", symbol="🟣") - AG = ModelDetails(name="🟡 agentic", symbol="🟡") + FT = ModelDetails(name="🔶 fine-tuned", symbol="🔶") DL = ModelDetails(name="🔷 deep-learning", symbol="🔷") - ST = ModelDetails(name="🔶 statistical", symbol="🔶") - + ST = ModelDetails(name="🟣 statistical", symbol="🟣") Unknown = ModelDetails(name="", symbol="?") @@ -80,10 +74,6 @@ class ModelType(Enum): return ModelType.FT if "pretrained" in type or "🟢" in type: return ModelType.PT - if "zero-shot" in type or "🔴" in type: - return ModelType.ZT - if "agentic" in type or "🟡" in type: - return ModelType.AG if "deep-learning" in type or "🟦" in type: return ModelType.DL if "statistical" in type or "🟣" in type: diff --git a/src/leaderboard/read_evals.py b/src/leaderboard/read_evals.py index 47a873674d550821507f62ec149909b4f2524750..49659668c33dbbef683cf21d872b9e5321d2c046 100644 --- a/src/leaderboard/read_evals.py +++ b/src/leaderboard/read_evals.py @@ -21,12 +21,9 @@ class ModelConfig: model_type: ModelType = ModelType.Unknown code_link: str = "" precision: Precision = Precision.Unknown - org: str = "" license: str = "?" likes: int = 0 num_params: int | str = 0 - testdata_leakage: str = "NA" - replication_code_available: str = "NA" @classmethod def init_from_json_file(cls, json_filepath): @@ -42,11 +39,7 @@ class ModelConfig: model = data.get("model", "") model_link = data.get("model_link", "") code_link = data.get("code_link", "") - org = data.get("org", "") - testdata_leakage = data.get("testdata_leakage", "N/A") - replication_code_available = data.get("replication_code_available", "NA") - return cls(model=model, model_link=model_link, model_type=model_type, code_link=code_link, org=org, - precision=precision, testdata_leakage=testdata_leakage, replication_code_available=replication_code_available) + return cls(model=model, model_link=model_link, model_type=model_type, code_link=code_link, precision=precision) def to_dict(self): """Converts the model info to a dict compatible with our dataframe display""" @@ -60,9 +53,6 @@ class ModelConfig: ModelInfoColumn.license.name: self.license, ModelInfoColumn.likes.name: self.likes, ModelInfoColumn.params.name: self.num_params, - ModelInfoColumn.org.name: self.org, - ModelInfoColumn.testdata_leakage.name: self.testdata_leakage, - ModelInfoColumn.replication_code_available.name: self.replication_code_available, } return data_dict @@ -73,14 +63,14 @@ class EvalResult: """ eval_name: str # org_model_precision (uid) full_model: str # org/model (path on hub) - org: str + org: str model: str revision: str # commit hash, "" if main results: dict precision: Precision = Precision.Unknown model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ... weight_type: WeightType = WeightType.Original # Original or Adapter - architecture: str = "Unknown" + architecture: str = "Unknown" license: str = "?" likes: int = 0 num_params: int = 0 @@ -140,7 +130,7 @@ class EvalResult: org=org, model=model, results=results, - precision=precision, + precision=precision, revision= config.get("model_sha", ""), still_on_hub=still_on_hub, architecture=architecture @@ -247,7 +237,6 @@ def get_model_info(results_path: str, requests_path: str) -> list[ModelConfig]: for v in model_infos.values(): try: v.to_dict() # we test if the dict version is complete - # ipdb.set_trace() results.append(v) except KeyError: # not all eval values present continue diff --git a/src/utils.py b/src/utils.py index 08458bca90bbee059252027012c505504cf16aec..b23d44871a222b1804598123234fdce9f6dc75d5 100644 --- a/src/utils.py +++ b/src/utils.py @@ -48,8 +48,7 @@ def rename_metrics(df): df = df.rename(columns={ 'eval_metrics/MASE[0.5]': 'MASE', 'eval_metrics/mean_weighted_sum_quantile_loss': 'CRPS', - 'rank': 'CRPS_Rank', - 'Rank_MASE': 'MASE_Rank', + 'rank': 'Rank' }) return df @@ -106,59 +105,6 @@ def pivot_existed_df(df, tab_name): return df_pivot -def get_all_res_dfs(root_dir='results', ds_properties='results/dataset_properties.csv'): - df_list = [] - - # Walk through all folders and subfolders in the root directory - for subdir, _, files in os.walk(root_dir): - for file in files: - if file == 'all_results.csv': - file_path = os.path.join(subdir, file) - df = pd.read_csv(file_path) - df_list.append(df) - # Concatenate all dataframes into one - all_results_df = pd.concat(df_list, ignore_index=True) - all_results_df = all_results_df.sort_values(by=['model', 'dataset']).reset_index(drop=True) - all_results_df[['dataset', 'frequency', 'term_length']] = all_results_df['dataset'].str.split('/', expand=True) - - dataset_properties = pd.read_csv(ds_properties) - # Reforemat the the first element of each row after the header following these rules: - # 1. make all characters lowercase - dataset_properties['dataset'] = dataset_properties['dataset'].apply(lambda x: x.lower()) - # 2. replace all spaces with underscores - dataset_properties['dataset'] = dataset_properties['dataset'].apply(lambda x: x.replace(' ', '_')) - # 3. Replace all dashes with underscores - dataset_properties['dataset'] = dataset_properties['dataset'].apply(lambda x: x.replace('-', '_')) - # 4. Replace consecutive underscores with a single underscore. There maybe more than 2 consecutive underscores - dataset_properties['dataset'] = dataset_properties['dataset'].apply(lambda x: re.sub('_+', '_', x)) - # 5. Remove all leading and trailing underscores - dataset_properties['dataset'] = dataset_properties['dataset'].apply(lambda x: x.strip('_')) - - df = all_results_df - - # convert it to a dictionary, with dataset as the key, and the value as another dictionary. The inner dictionary has the column names as the key, and the value as the value. - dataset_properties_dict = dataset_properties.set_index('dataset').T.to_dict('dict') - dataset_properties_dict.keys() - - # # match the dataset name in model_properties_dict with the dataset name in df and add a new column for each key value pair in the inner dictionary. - for dataset in dataset_properties_dict.keys(): - for key in dataset_properties_dict[dataset].keys(): - # set the row with the dataset name to the value of the key think step by step - # First, get the row with the dataset name - # Second, set the value of the key to the value of the key - if key == 'frequency': - # only set the frequency if the frequency column for all rows for the dataset is empty string - if all(df[df['dataset'] == dataset]['frequency'].isna()): - df.loc[df['dataset'] == dataset, key] = dataset_properties_dict[dataset][key] - else: - df.loc[df['dataset'] == dataset, key] = dataset_properties_dict[dataset][key] - - # unify the frequency - df = unify_freq(df) - # standardize by seasonal naive - df = standardize_df(df) - ipdb.set_trace() - return None def get_grouped_dfs(root_dir='results', ds_properties='results/dataset_properties.csv'): df_list = [] @@ -217,24 +163,6 @@ def get_grouped_dfs(root_dir='results', ds_properties='results/dataset_propertie RANKING_METRIC = "eval_metrics/mean_weighted_sum_quantile_loss" df['rank'] = df.groupby(['dataset', 'term_length', 'frequency'])[f'{RANKING_METRIC}'].rank(method='first', ascending=True) - MASE_RANKING_METRICS = "eval_metrics/MASE[0.5]" - df['Rank_MASE'] = df.groupby(['dataset', 'term_length', 'frequency'])[f'{MASE_RANKING_METRICS}'].rank(method='first', - ascending=True) - # Add a new column to identify top 5 ranks - df['is_top5_CRPS'] = df['rank'] <= 5 - - # Count top 3 performances for each model - top5_counts = df.groupby('model')['is_top5_CRPS'].sum().reset_index() - top5_counts = top5_counts.rename(columns={'is_top5_CRPS': 'count_top5_CRPS'}) - - # Add a new column to identify top 5 ranks - df['is_top5_MASE'] = df['Rank_MASE'] <= 5 - - # Count top 3 performances for each model - top5_counts_MASE = df.groupby('model')['is_top5_MASE'].sum().reset_index() - top5_counts_MASE = top5_counts_MASE.rename(columns={'is_top5_MASE': 'count_top5_MASE'}) - - # ipdb.set_trace() # create a new column called rank metric_columns.append('rank') # create a new column called univariate. Set it to true if column num_variates is 1, otherwise set it to false @@ -245,20 +173,7 @@ def get_grouped_dfs(root_dir='results', ds_properties='results/dataset_propertie # ipdb.set_trace() grouped_results_overall = df.groupby(['model'])[METRIC_CHOICES].agg(stats.gmean) grouped_results_overall_rank = df.groupby(['model'])[['rank']].mean() - grouped_results_overall_rank_mase = df.groupby(['model'])[['Rank_MASE']].mean() - grouped_results_overall = pd.concat([grouped_results_overall, grouped_results_overall_rank,grouped_results_overall_rank_mase], axis=1) - - # grouped_results_overall_avg_rank = pd.concat([grouped_results_overall_rank, grouped_results_overall_rank_mase], axis=1) - # grouped_results_overall_avg_rank['Avg_Rank'] = grouped_results_overall_avg_rank.mean(axis=1) - # grouped_results_overall_avg_rank = grouped_results_overall_avg_rank[['Avg_Rank']] - # grouped_results_overall = pd.concat([grouped_results_overall, grouped_results_overall_avg_rank], axis=1) - # switch the order to ['model', 'eval_metrics/MASE[0.5]', 'Rank_MASE', 'eval_metrics/mean_weighted_sum_quantile_loss', 'rank'] - # grouped_results_overall = grouped_results_overall[['eval_metrics/MASE[0.5]', 'Rank_MASE', 'eval_metrics/mean_weighted_sum_quantile_loss', 'rank', 'Avg_Rank']] - grouped_results_overall = grouped_results_overall[['eval_metrics/MASE[0.5]', 'Rank_MASE', 'eval_metrics/mean_weighted_sum_quantile_loss', 'rank']] - # ipdb.set_trace() - # Add top3 performance statistics to the overall results - # grouped_results_overall = pd.merge(grouped_results_overall, top5_counts, on='model') - # grouped_results_overall = pd.merge(grouped_results_overall, top5_counts_MASE, on='model') + grouped_results_overall = pd.concat([grouped_results_overall, grouped_results_overall_rank], axis=1) # grouped_results_overall = grouped_results_overall.rename(columns={'model':'Model'}) # grouped_results.to_csv(f'artefacts/grouped_results_by_model.csv')