Initial WaveLSFromer project

.gitattributes

@@ -0,0 +1 @@
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,25 @@
+**/__pycache__/
+*.pyc
+.venv/
+.env
+.vscode
+.codex
+.tmux.conf
+
+# Notebook/editor artifacts
+**/.ipynb_checkpoints/
+*~
+*.un~
+:w
+'
+
+checkpoints
+data
+results
+
+# Pytorch Lightning
+lightning_logs
+.lr_find_*
+.scale_batch_size_*
+
+bbtest_logs

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,26 @@
+![Python 3.10](https://img.shields.io/badge/python-3.10-green.svg?style=plastic)
+![PyTorch 1.13](https://img.shields.io/badge/pytorch-1.13-green.svg?style=plastic)
+![PyTorch Lightning 1.8](https://img.shields.io/badge/pytorch%20lightning-1.8-green.svg?style=plastic)
+
+# WaveLSFromer
+
+WaveLSFromer is a research codebase for long-sequence financial time-series
+forecasting. It extends the Informer/Stockformer style transformer stack with
+stock-specific training objectives, PyTorch Lightning experiment loops,
+config-driven model runs, and learnable wavelet front-end components for
+low/high frequency feature extraction.
+
+The repository includes:
+
+- transformer, Informer, DLinear, LSTM, and MLP model baselines;
+- learnable 1D wavelet filters with frequency-domain regularization;
+- PyTorch Lightning training, validation, prediction, and checkpoint workflows;
+- stock-return metrics and differentiable trading-oriented loss functions;
+- YAML experiment configs for financial and benchmark time-series datasets;
+- notebooks and scripts for data collection, preparation, and result analysis.
+
+Initially forked from the [Informer: Beyond Efficient Transformer for Long Sequence Time-Series Forecasting repo](https://github.com/zhouhaoyi/Informer2020).
+
+Thanks to [polygon.io](http://polygon.io/) for being our financial data provider.
+
+Contributors: [Zac Schulwolf](https://github.com/zacswolf) and [Shuozhe Li](https://github.com/ShuoZheLi) 

Stockformer.py

@@ -0,0 +1,146 @@
+import os
+import torch
+import torch.nn as nn
+
+from layers.encoder import Encoder, EncoderLayer, ConvLayer
+from layers.attn import FullAttention, AttentionLayer, ProbAttention
+from layers.embed import DataEmbedding
+from utils.masking import QuestionMask
+from .wavelet import WaveletFront
+
+
+class Stockformer(nn.Module):
+    def __init__(self, config):
+        super(Stockformer, self).__init__()
+        self.pred_len = config.pred_len
+        assert self.pred_len == 1, "Stockformer needs pred_len to be 1"
+        self.attn = config.attn
+        self.output_attention = config.output_attention
+        self.seq_len = config.seq_len
+        self.final_mode = config.final_mode
+
+        self.wave_model = WaveletFront(in_channels=config.enc_in,
+                                       d_model=config.d_model-config.emb_t2v_app_dim,
+                                       kernel_size=31,
+                                       n_fft=128)
+
+        # Embedding
+        self.enc_embedding = DataEmbedding(
+            config.d_model-config.emb_t2v_app_dim,
+            #  config.enc_in,
+            config.d_model,
+            config.t_embed,
+            config.freq,
+            config.dropout_emb,
+            emb_t2v_app_dim=config.emb_t2v_app_dim,
+            tok_emb=config.tok_emb,
+        )
+        # Attention
+        Attn = ProbAttention if config.attn == "prob" else FullAttention
+        # Encoder
+        self.encoder = Encoder(
+            [
+                EncoderLayer(
+                    AttentionLayer(
+                        Attn(
+                            True if config.final_mode == "mode3" else False,
+                            config.factor,
+                            attention_dropout=config.dropout,
+                            output_attention=config.output_attention,
+                        ),
+                        config.d_model,
+                        config.n_heads,
+                        mix=False,
+                    ),
+                    config.d_model,
+                    config.d_ff,
+                    dropout=config.dropout,
+                    activation=config.activation,
+                    ln_mode=config.ln_mode,
+                )
+                for l in range(config.e_layers)
+            ],
+            [ConvLayer(config.d_model) for l in range(config.e_layers - 1)]
+            if config.distil
+            else None,
+            #  norm_layer=torch.nn.LayerNorm(config.d_model),
+            norm_layer=torch.nn.RMSNorm(config.d_model),
+        )
+
+        if config.final_mode == "mode1":
+            self.final = nn.Linear(
+                config.d_model * config.seq_len, config.c_out, bias=True
+            )
+        elif config.final_mode == "mode2" or config.final_mode == "mode3":
+            self.final = nn.Linear(config.d_model, config.c_out, bias=True)
+        else:
+            raise Exception(f"Invalid final_mode: {config.final_mode}")
+        # nn.init.xavier_normal_(self.final.weight, gain=nn.init.calculate_gain("tanh"))
+
+        # self.final = nn.Sequential(*[
+        #     nn.Linear(config.d_model * config.seq_len, config.d_model * 4, bias=True),
+        #     nn.GELU(),
+        #     nn.Linear(config.d_model * 4, config.c_out, bias=True)
+        # ])
+
+        # Load pre-trained model
+        if config.load_model_path is not None:
+            path = os.path.join(config.checkpoints, config.load_model_path)
+            print(f"Loading Model from {path}")
+            self.load_state_dict(torch.load(path))
+
+    def forward(
+        self,
+        x_enc,
+        x_mark_enc,
+        x_dec,
+        x_mark_dec,
+        enc_self_mask=None,
+        dec_self_mask=None,
+        dec_enc_mask=None,
+        pre_train=False,
+    ):
+        # x_enc is (batch_size / num gpus, seq_len, enc_in)
+        # x_mark_enc is (batch_size / num gpus, seq_len, date-representation (7forhours)
+        assert len(x_enc.shape) == 3
+        assert x_enc.shape[1] == self.seq_len
+
+        x_enc, reg, _ = self.wave_model(x_enc.permute(0, 2, 1))
+
+        #  print(reg)
+        lambda_low=100; lambda_high=100; lambda_overlap=100; lambda_parse=1e-2; lambda_shape=10
+        loss_reg = (lambda_low*reg["L_low"] + lambda_high*reg["L_high"]
+                    + lambda_overlap*reg["L_overlap"]
+                    + lambda_parse*reg["L_parseval"]
+                    + lambda_shape*reg["L_shape"]
+                    )
+
+        if self.final_mode == "mode3":
+            # This gives the encoder a question input as the last token
+            # TODO: Maybe this should be initialized differently, like to the mean of x_enc, random, mean of dataset?
+            zeros = torch.zeros([x_enc.shape[0], 1, x_enc.shape[2]]).to(x_enc)
+            x_enc = torch.cat([x_enc, zeros], 1)
+            x_mark_enc = torch.cat([x_mark_enc, x_mark_dec], 1)
+            assert enc_self_mask is None
+            enc_self_mask = QuestionMask(
+                x_enc.shape[0], x_enc.shape[1], device=x_enc.device
+            )
+
+        # emb_out is (batch_size / num gpus, seq_len, d_model)
+        emb_out = self.enc_embedding(x_enc, x_mark_enc)
+
+        # enc_out is (batch_size / num gpus, seq_len, d_model) but seq_len will change if distil
+        enc_out, attns = self.encoder(emb_out, attn_mask=enc_self_mask)
+
+        if self.final_mode == "mode1":
+            out = self.final(enc_out.flatten(start_dim=1))
+        elif self.final_mode == "mode2" or self.final_mode == "mode3":
+            out = self.final(enc_out[:, -1, :])
+        else:
+            assert False, f"Forward missing valid final mode {self.final_mode}"
+
+        # The None below is just adding a dummy dimension
+        if self.output_attention:
+            return out[:, None, :], attns
+        else:
+            return out[:, None, :], loss_reg  # (batch_size, 1, c_out)

clean_ipynb.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# This script removes all output from a Jupyter notebook
+# Generated using copilot
+# Usage: source clean_ipynb.sh notebook.ipynb
+
+# Check if the file exists
+if [ ! -f $1 ]; then
+    echo "File $1 does not exist"
+    exit 1
+fi
+
+# Check if the file is a Jupyter notebook
+if [ "${1: -6}" != ".ipynb" ]; then
+    echo "File $1 is not a Jupyter notebook"
+    exit 1
+fi
+
+# Remove all output from the notebook
+python -m nbconvert --clear-output --inplace $1
+# jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace $1

configs/lstm/basic_PEMSBAY.yaml

@@ -0,0 +1,368 @@
+activation: gelu
+attn: full
+batch_size: 256
+c_out: 1
+cols:
+- 400001_logpctchange
+- 400017_logpctchange
+- 400030_logpctchange
+- 400040_logpctchange
+- 400045_logpctchange
+- 400052_logpctchange
+- 400057_logpctchange
+- 400059_logpctchange
+- 400065_logpctchange
+- 400069_logpctchange
+- 400073_logpctchange
+- 400084_logpctchange
+- 400085_logpctchange
+- 400088_logpctchange
+- 400096_logpctchange
+- 400097_logpctchange
+- 400100_logpctchange
+- 400104_logpctchange
+- 400109_logpctchange
+- 400122_logpctchange
+- 400147_logpctchange
+- 400148_logpctchange
+- 400149_logpctchange
+- 400158_logpctchange
+- 400160_logpctchange
+- 400168_logpctchange
+- 400172_logpctchange
+- 400174_logpctchange
+- 400178_logpctchange
+- 400185_logpctchange
+- 400201_logpctchange
+- 400206_logpctchange
+- 400209_logpctchange
+- 400213_logpctchange
+- 400221_logpctchange
+- 400222_logpctchange
+- 400227_logpctchange
+- 400236_logpctchange
+- 400238_logpctchange
+- 400240_logpctchange
+- 400246_logpctchange
+- 400253_logpctchange
+- 400257_logpctchange
+- 400258_logpctchange
+- 400268_logpctchange
+- 400274_logpctchange
+- 400278_logpctchange
+- 400280_logpctchange
+- 400292_logpctchange
+- 400296_logpctchange
+- 400298_logpctchange
+- 400330_logpctchange
+- 400336_logpctchange
+- 400343_logpctchange
+- 400353_logpctchange
+- 400372_logpctchange
+- 400394_logpctchange
+- 400400_logpctchange
+- 400414_logpctchange
+- 400418_logpctchange
+- 400429_logpctchange
+- 400435_logpctchange
+- 400436_logpctchange
+- 400440_logpctchange
+- 400449_logpctchange
+- 400457_logpctchange
+- 400461_logpctchange
+- 400464_logpctchange
+- 400479_logpctchange
+- 400485_logpctchange
+- 400499_logpctchange
+- 400507_logpctchange
+- 400508_logpctchange
+- 400514_logpctchange
+- 400519_logpctchange
+- 400528_logpctchange
+- 400545_logpctchange
+- 400560_logpctchange
+- 400563_logpctchange
+- 400567_logpctchange
+- 400581_logpctchange
+- 400582_logpctchange
+- 400586_logpctchange
+- 400637_logpctchange
+- 400643_logpctchange
+- 400648_logpctchange
+- 400649_logpctchange
+- 400654_logpctchange
+- 400664_logpctchange
+- 400665_logpctchange
+- 400668_logpctchange
+- 400673_logpctchange
+- 400677_logpctchange
+- 400687_logpctchange
+- 400688_logpctchange
+- 400690_logpctchange
+- 400700_logpctchange
+- 400709_logpctchange
+- 400713_logpctchange
+- 400714_logpctchange
+- 400715_logpctchange
+- 400717_logpctchange
+- 400723_logpctchange
+- 400743_logpctchange
+- 400750_logpctchange
+- 400760_logpctchange
+- 400772_logpctchange
+- 400790_logpctchange
+- 400792_logpctchange
+- 400794_logpctchange
+- 400799_logpctchange
+- 400804_logpctchange
+- 400822_logpctchange
+- 400823_logpctchange
+- 400828_logpctchange
+- 400832_logpctchange
+- 400837_logpctchange
+- 400842_logpctchange
+- 400863_logpctchange
+- 400869_logpctchange
+- 400873_logpctchange
+- 400895_logpctchange
+- 400904_logpctchange
+- 400907_logpctchange
+- 400911_logpctchange
+- 400916_logpctchange
+- 400922_logpctchange
+- 400934_logpctchange
+- 400951_logpctchange
+- 400952_logpctchange
+- 400953_logpctchange
+- 400964_logpctchange
+- 400965_logpctchange
+- 400970_logpctchange
+- 400971_logpctchange
+- 400973_logpctchange
+- 400995_logpctchange
+- 400996_logpctchange
+- 401014_logpctchange
+- 401129_logpctchange
+- 401154_logpctchange
+- 401163_logpctchange
+- 401167_logpctchange
+- 401210_logpctchange
+- 401224_logpctchange
+- 401327_logpctchange
+- 401351_logpctchange
+- 401388_logpctchange
+- 401391_logpctchange
+- 401400_logpctchange
+- 401403_logpctchange
+- 401440_logpctchange
+- 401457_logpctchange
+- 401464_logpctchange
+- 401489_logpctchange
+- 401495_logpctchange
+- 401507_logpctchange
+- 401534_logpctchange
+- 401541_logpctchange
+- 401555_logpctchange
+- 401560_logpctchange
+- 401567_logpctchange
+- 401597_logpctchange
+- 401606_logpctchange
+- 401611_logpctchange
+- 401655_logpctchange
+- 401808_logpctchange
+- 401809_logpctchange
+- 401810_logpctchange
+- 401811_logpctchange
+- 401816_logpctchange
+- 401817_logpctchange
+- 401845_logpctchange
+- 401846_logpctchange
+- 401890_logpctchange
+- 401891_logpctchange
+- 401906_logpctchange
+- 401908_logpctchange
+- 401926_logpctchange
+- 401936_logpctchange
+- 401937_logpctchange
+- 401942_logpctchange
+- 401943_logpctchange
+- 401948_logpctchange
+- 401957_logpctchange
+- 401958_logpctchange
+- 401994_logpctchange
+- 401996_logpctchange
+- 401997_logpctchange
+- 401998_logpctchange
+- 402056_logpctchange
+- 402057_logpctchange
+- 402058_logpctchange
+- 402059_logpctchange
+- 402060_logpctchange
+- 402061_logpctchange
+- 402067_logpctchange
+- 402117_logpctchange
+- 402118_logpctchange
+- 402119_logpctchange
+- 402120_logpctchange
+- 402121_logpctchange
+- 402281_logpctchange
+- 402282_logpctchange
+- 402283_logpctchange
+- 402284_logpctchange
+- 402285_logpctchange
+- 402286_logpctchange
+- 402287_logpctchange
+- 402288_logpctchange
+- 402289_logpctchange
+- 402359_logpctchange
+- 402360_logpctchange
+- 402361_logpctchange
+- 402362_logpctchange
+- 402363_logpctchange
+- 402364_logpctchange
+- 402365_logpctchange
+- 402366_logpctchange
+- 402367_logpctchange
+- 402368_logpctchange
+- 402369_logpctchange
+- 402370_logpctchange
+- 402371_logpctchange
+- 402372_logpctchange
+- 402373_logpctchange
+- 403225_logpctchange
+- 403265_logpctchange
+- 403329_logpctchange
+- 403401_logpctchange
+- 403402_logpctchange
+- 403404_logpctchange
+- 403406_logpctchange
+- 403409_logpctchange
+- 403412_logpctchange
+- 403414_logpctchange
+- 403419_logpctchange
+- 404370_logpctchange
+- 404434_logpctchange
+- 404435_logpctchange
+- 404444_logpctchange
+- 404451_logpctchange
+- 404452_logpctchange
+- 404453_logpctchange
+- 404461_logpctchange
+- 404462_logpctchange
+- 404521_logpctchange
+- 404522_logpctchange
+- 404553_logpctchange
+- 404554_logpctchange
+- 404585_logpctchange
+- 404586_logpctchange
+- 404640_logpctchange
+- 404753_logpctchange
+- 404759_logpctchange
+- 405613_logpctchange
+- 405619_logpctchange
+- 405701_logpctchange
+- 407150_logpctchange
+- 407151_logpctchange
+- 407152_logpctchange
+- 407153_logpctchange
+- 407155_logpctchange
+- 407157_logpctchange
+- 407161_logpctchange
+- 407165_logpctchange
+- 407172_logpctchange
+- 407173_logpctchange
+- 407174_logpctchange
+- 407176_logpctchange
+- 407177_logpctchange
+- 407179_logpctchange
+- 407180_logpctchange
+- 407181_logpctchange
+- 407184_logpctchange
+- 407185_logpctchange
+- 407186_logpctchange
+- 407187_logpctchange
+- 407190_logpctchange
+- 407191_logpctchange
+- 407194_logpctchange
+- 407200_logpctchange
+- 407202_logpctchange
+- 407204_logpctchange
+- 407206_logpctchange
+- 407207_logpctchange
+- 407321_logpctchange
+- 407323_logpctchange
+- 407325_logpctchange
+- 407328_logpctchange
+- 407331_logpctchange
+- 407332_logpctchange
+- 407335_logpctchange
+- 407336_logpctchange
+- 407337_logpctchange
+- 407339_logpctchange
+- 407341_logpctchange
+- 407342_logpctchange
+- 407344_logpctchange
+- 407348_logpctchange
+- 407352_logpctchange
+- 407359_logpctchange
+- 407360_logpctchange
+- 407361_logpctchange
+- 407364_logpctchange
+- 407367_logpctchange
+- 407370_logpctchange
+- 407372_logpctchange
+- 407373_logpctchange
+- 407374_logpctchange
+- 407710_logpctchange
+- 407711_logpctchange
+- 408907_logpctchange
+- 408911_logpctchange
+- 409524_logpctchange
+- 409525_logpctchange
+- 409526_logpctchange
+- 409528_logpctchange
+- 409529_logpctchange
+- 413026_logpctchange
+- 413845_logpctchange
+- 413877_logpctchange
+- 413878_logpctchange
+- 414284_logpctchange
+- 414694_logpctchange
+d_ff: 512
+d_model: 512
+data_path: PEMSBAY.csv
+# date_end: '2020-01-01'
+# date_start: '2012-01-01'
+# date_test: '2019-06-01'
+des: lstmPEMs
+distil: false
+dont_shuffle_train: false
+dropout: 0.5
+dropout_emb: 0.0
+e_layers: 1
+emb_t2v_app_dim: 16
+enc_in: 325
+features: MS
+final_mode: mode1
+freq: m
+inverse_output: false
+inverse_pred: true
+label_len: 0
+learning_rate: 1.0e-04
+loss: stock_tanhv1
+lradj: null
+max_epochs: 50
+model: lstm
+no_early_stop: false
+no_scale_mean: true
+optim: Adam
+patience: 1000
+pre_loss: null
+pred_len: 1
+pre_epochs: 0
+root_path: ./data/other/
+scale: true
+seed: 2
+seq_len: 16
+# t_embed: time2vec_add
+target: 400001_logpctchange

configs/mlp/basic.yaml

@@ -0,0 +1,58 @@
+activation: gelu
+attn: full
+batch_size: 256
+c_out: 1
+cols:
+- XOM_logpctchange
+- CVX_logpctchange
+- COP_logpctchange
+- BP_logpctchange
+- PBR_logpctchange
+- WTI_logpctchange
+- EOG_logpctchange
+- ENB_logpctchange
+- SLB_logpctchange
+d_ff: 4096
+d_model: 512
+data_path: full_1h.csv
+# date_end: '2020-01-01'
+# date_start: '2012-01-01'
+# date_test: '2019-06-01'
+des: full_1h
+distil: false
+dont_shuffle_train: false
+dropout: 0.5
+dropout_emb: 0.0
+e_layers: 4
+emb_t2v_app_dim: 16
+enc_in: 9
+factor: 5
+features: MS
+final_mode: mode1
+freq: m
+inverse_output: false
+inverse_pred: true
+label_len: 0
+learning_rate: 1.0e-05
+# ln_mode: post
+loss: stock_tanhv1
+lradj: null
+max_epochs: 50
+# mix: false
+model: mlp
+# n_heads: 16
+no_early_stop: false
+no_scale_mean: true
+optim: Adam
+# output_attention: false
+patience: 1000
+pre_loss: null #stock_tanhv4
+pred_len: 1
+pre_epochs: 0
+root_path: ./data/stock/
+scale: true
+seed: 2
+seq_len: 32
+# t_embed: time2vec_add
+target: WTI_logpctchange
+

configs/mlp/basic_PEMSBAY.yaml

@@ -0,0 +1,366 @@
+activation: gelu
+attn: full
+batch_size: 256
+c_out: 1
+cols:
+- 400001_logpctchange
+- 400017_logpctchange
+- 400030_logpctchange
+- 400040_logpctchange
+- 400045_logpctchange
+- 400052_logpctchange
+- 400057_logpctchange
+- 400059_logpctchange
+- 400065_logpctchange
+- 400069_logpctchange
+- 400073_logpctchange
+- 400084_logpctchange
+- 400085_logpctchange
+- 400088_logpctchange
+- 400096_logpctchange
+- 400097_logpctchange
+- 400100_logpctchange
+- 400104_logpctchange
+- 400109_logpctchange
+- 400122_logpctchange
+- 400147_logpctchange
+- 400148_logpctchange
+- 400149_logpctchange
+- 400158_logpctchange
+- 400160_logpctchange
+- 400168_logpctchange
+- 400172_logpctchange
+- 400174_logpctchange
+- 400178_logpctchange
+- 400185_logpctchange
+- 400201_logpctchange
+- 400206_logpctchange
+- 400209_logpctchange
+- 400213_logpctchange
+- 400221_logpctchange
+- 400222_logpctchange
+- 400227_logpctchange
+- 400236_logpctchange
+- 400238_logpctchange
+- 400240_logpctchange
+- 400246_logpctchange
+- 400253_logpctchange
+- 400257_logpctchange
+- 400258_logpctchange
+- 400268_logpctchange
+- 400274_logpctchange
+- 400278_logpctchange
+- 400280_logpctchange
+- 400292_logpctchange
+- 400296_logpctchange
+- 400298_logpctchange
+- 400330_logpctchange
+- 400336_logpctchange
+- 400343_logpctchange
+- 400353_logpctchange
+- 400372_logpctchange
+- 400394_logpctchange
+- 400400_logpctchange
+- 400414_logpctchange
+- 400418_logpctchange
+- 400429_logpctchange
+- 400435_logpctchange
+- 400436_logpctchange
+- 400440_logpctchange
+- 400449_logpctchange
+- 400457_logpctchange
+- 400461_logpctchange
+- 400464_logpctchange
+- 400479_logpctchange
+- 400485_logpctchange
+- 400499_logpctchange
+- 400507_logpctchange
+- 400508_logpctchange
+- 400514_logpctchange
+- 400519_logpctchange
+- 400528_logpctchange
+- 400545_logpctchange
+- 400560_logpctchange
+- 400563_logpctchange
+- 400567_logpctchange
+- 400581_logpctchange
+- 400582_logpctchange
+- 400586_logpctchange
+- 400637_logpctchange
+- 400643_logpctchange
+- 400648_logpctchange
+- 400649_logpctchange
+- 400654_logpctchange
+- 400664_logpctchange
+- 400665_logpctchange
+- 400668_logpctchange
+- 400673_logpctchange
+- 400677_logpctchange
+- 400687_logpctchange
+- 400688_logpctchange
+- 400690_logpctchange
+- 400700_logpctchange
+- 400709_logpctchange
+- 400713_logpctchange
+- 400714_logpctchange
+- 400715_logpctchange
+- 400717_logpctchange
+- 400723_logpctchange
+- 400743_logpctchange
+- 400750_logpctchange
+- 400760_logpctchange
+- 400772_logpctchange
+- 400790_logpctchange
+- 400792_logpctchange
+- 400794_logpctchange
+- 400799_logpctchange
+- 400804_logpctchange
+- 400822_logpctchange
+- 400823_logpctchange
+- 400828_logpctchange
+- 400832_logpctchange
+- 400837_logpctchange
+- 400842_logpctchange
+- 400863_logpctchange
+- 400869_logpctchange
+- 400873_logpctchange
+- 400895_logpctchange
+- 400904_logpctchange
+- 400907_logpctchange
+- 400911_logpctchange
+- 400916_logpctchange
+- 400922_logpctchange
+- 400934_logpctchange
+- 400951_logpctchange
+- 400952_logpctchange
+- 400953_logpctchange
+- 400964_logpctchange
+- 400965_logpctchange
+- 400970_logpctchange
+- 400971_logpctchange
+- 400973_logpctchange
+- 400995_logpctchange
+- 400996_logpctchange
+- 401014_logpctchange
+- 401129_logpctchange
+- 401154_logpctchange
+- 401163_logpctchange
+- 401167_logpctchange
+- 401210_logpctchange
+- 401224_logpctchange
+- 401327_logpctchange
+- 401351_logpctchange
+- 401388_logpctchange
+- 401391_logpctchange
+- 401400_logpctchange
+- 401403_logpctchange
+- 401440_logpctchange
+- 401457_logpctchange
+- 401464_logpctchange
+- 401489_logpctchange
+- 401495_logpctchange
+- 401507_logpctchange
+- 401534_logpctchange
+- 401541_logpctchange
+- 401555_logpctchange
+- 401560_logpctchange
+- 401567_logpctchange
+- 401597_logpctchange
+- 401606_logpctchange
+- 401611_logpctchange
+- 401655_logpctchange
+- 401808_logpctchange
+- 401809_logpctchange
+- 401810_logpctchange
+- 401811_logpctchange
+- 401816_logpctchange
+- 401817_logpctchange
+- 401845_logpctchange
+- 401846_logpctchange
+- 401890_logpctchange
+- 401891_logpctchange
+- 401906_logpctchange
+- 401908_logpctchange
+- 401926_logpctchange
+- 401936_logpctchange
+- 401937_logpctchange
+- 401942_logpctchange
+- 401943_logpctchange
+- 401948_logpctchange
+- 401957_logpctchange
+- 401958_logpctchange
+- 401994_logpctchange
+- 401996_logpctchange
+- 401997_logpctchange
+- 401998_logpctchange
+- 402056_logpctchange
+- 402057_logpctchange
+- 402058_logpctchange
+- 402059_logpctchange
+- 402060_logpctchange
+- 402061_logpctchange
+- 402067_logpctchange
+- 402117_logpctchange
+- 402118_logpctchange
+- 402119_logpctchange
+- 402120_logpctchange
+- 402121_logpctchange
+- 402281_logpctchange
+- 402282_logpctchange
+- 402283_logpctchange
+- 402284_logpctchange
+- 402285_logpctchange
+- 402286_logpctchange
+- 402287_logpctchange
+- 402288_logpctchange
+- 402289_logpctchange
+- 402359_logpctchange
+- 402360_logpctchange
+- 402361_logpctchange
+- 402362_logpctchange
+- 402363_logpctchange
+- 402364_logpctchange
+- 402365_logpctchange
+- 402366_logpctchange
+- 402367_logpctchange
+- 402368_logpctchange
+- 402369_logpctchange
+- 402370_logpctchange
+- 402371_logpctchange
+- 402372_logpctchange
+- 402373_logpctchange
+- 403225_logpctchange
+- 403265_logpctchange
+- 403329_logpctchange
+- 403401_logpctchange
+- 403402_logpctchange
+- 403404_logpctchange
+- 403406_logpctchange
+- 403409_logpctchange
+- 403412_logpctchange
+- 403414_logpctchange
+- 403419_logpctchange
+- 404370_logpctchange
+- 404434_logpctchange
+- 404435_logpctchange
+- 404444_logpctchange
+- 404451_logpctchange
+- 404452_logpctchange
+- 404453_logpctchange
+- 404461_logpctchange
+- 404462_logpctchange
+- 404521_logpctchange
+- 404522_logpctchange
+- 404553_logpctchange
+- 404554_logpctchange
+- 404585_logpctchange
+- 404586_logpctchange
+- 404640_logpctchange
+- 404753_logpctchange
+- 404759_logpctchange
+- 405613_logpctchange
+- 405619_logpctchange
+- 405701_logpctchange
+- 407150_logpctchange
+- 407151_logpctchange
+- 407152_logpctchange
+- 407153_logpctchange
+- 407155_logpctchange
+- 407157_logpctchange
+- 407161_logpctchange
+- 407165_logpctchange
+- 407172_logpctchange
+- 407173_logpctchange
+- 407174_logpctchange
+- 407176_logpctchange
+- 407177_logpctchange
+- 407179_logpctchange
+- 407180_logpctchange
+- 407181_logpctchange
+- 407184_logpctchange
+- 407185_logpctchange
+- 407186_logpctchange
+- 407187_logpctchange
+- 407190_logpctchange
+- 407191_logpctchange
+- 407194_logpctchange
+- 407200_logpctchange
+- 407202_logpctchange
+- 407204_logpctchange
+- 407206_logpctchange
+- 407207_logpctchange
+- 407321_logpctchange
+- 407323_logpctchange
+- 407325_logpctchange
+- 407328_logpctchange
+- 407331_logpctchange
+- 407332_logpctchange
+- 407335_logpctchange
+- 407336_logpctchange
+- 407337_logpctchange
+- 407339_logpctchange
+- 407341_logpctchange
+- 407342_logpctchange
+- 407344_logpctchange
+- 407348_logpctchange
+- 407352_logpctchange
+- 407359_logpctchange
+- 407360_logpctchange
+- 407361_logpctchange
+- 407364_logpctchange
+- 407367_logpctchange
+- 407370_logpctchange
+- 407372_logpctchange
+- 407373_logpctchange
+- 407374_logpctchange
+- 407710_logpctchange
+- 407711_logpctchange
+- 408907_logpctchange
+- 408911_logpctchange
+- 409524_logpctchange
+- 409525_logpctchange
+- 409526_logpctchange
+- 409528_logpctchange
+- 409529_logpctchange
+- 413026_logpctchange
+- 413845_logpctchange
+- 413877_logpctchange
+- 413878_logpctchange
+- 414284_logpctchange
+- 414694_logpctchange
+d_model: 256
+data_path: PEMSBAY.csv
+# date_end: '2020-01-01'
+# date_start: '2012-01-01'
+# date_test: '2019-06-01'
+des: mlpPEMs
+dont_shuffle_train: false
+dropout: 0.75
+dropout_emb: 0.0
+e_layers: 4
+emb_t2v_app_dim: 16
+enc_in: 325
+features: MS
+final_mode: mode1
+freq: m
+inverse_output: false
+inverse_pred: true
+label_len: 0
+learning_rate: 1.0e-05
+loss: stock_tanhv1
+lradj: null
+max_epochs: 100
+model: mlp
+no_early_stop: false
+no_scale_mean: true
+optim: Adam
+patience: 1000
+pre_loss: null
+pred_len: 1
+pre_epochs: 0
+root_path: ./data/other/
+scale: true
+seed: 2
+seq_len: 16
+t_embed: null
+target: 400001_logpctchange

configs/stockformer/OneCycleLRSchedule.yaml

@@ -0,0 +1,59 @@
+activation: gelu
+attn: full
+batch_size: 128
+c_out: 1
+cols:
+- XOM_logpctchange
+- CVX_logpctchange
+- COP_logpctchange
+- BP_logpctchange
+- PBR_logpctchange
+- WTI_logpctchange
+- EOG_logpctchange
+# - ENB_logpctchange
+# - SLB_logpctchange
+d_ff: 4096 #2048
+d_model: 512
+# data_path: full_1h.csv
+data_path: material_1h.csv
+# date_end: '2020-01-01'
+# date_start: '2012-01-01'
+# date_test: '2019-06-01'
+date_end: '2025-10-23'
+date_start: '2020-10-26'
+date_test: '2025-06-26'
+des: full_1h
+distil: false
+dont_shuffle_train: true
+dropout: 0.5
+dropout_emb: 0.0
+e_layers: 12
+emb_t2v_app_dim: 16
+t_embed: time2vec_app
+enc_in: 7
+factor: 5
+features: MS
+final_mode: mode1
+freq: h
+inverse_output: false
+inverse_pred: true
+label_len: 0
+learning_rate: 1.0e-05
+ln_mode: post
+loss: stock_tanhv1
+lradj: type3
+mix: false
+model: stockformer
+n_heads: 512
+no_early_stop: false
+no_scale_mean: true
+optim: AdamW
+output_attention: false
+patience: 100
+pred_len: 1
+root_path: ./data/stock/
+scale: true
+seed: null
+seq_len: 16
+target: WTI_logpctchange
+max_epochs: 20

configs/stockformer/basic.yaml

@@ -0,0 +1,62 @@
+activation: gelu
+attn: full
+batch_size: 256
+c_out: 1
+cols:
+- XOM_logpctchange
+- CVX_logpctchange
+- COP_logpctchange
+- BP_logpctchange
+- PBR_logpctchange
+- WTI_logpctchange
+- EOG_logpctchange
+- ENB_logpctchange
+- SLB_logpctchange
+d_ff: 4096 #2048
+d_model: 512
+data_path: full_1h.csv
+# date_end: '2020-01-01'
+# date_start: '2012-01-01'
+# date_test: '2019-06-01'
+# date_test: '2020-11-01'
+# date_val: '2019-11-01'
+date_end: '2022-11-01'
+date_test: '2022-09-01'
+date_val: '2022-08-01'
+des: full_1h
+distil: false
+dont_shuffle_train: false
+dropout: 0.2
+dropout_emb: 0.0
+e_layers: 4
+emb_t2v_app_dim: 16
+enc_in: 9
+factor: 5
+features: MS
+final_mode: mode3
+freq: h
+inverse_output: false
+inverse_pred: true
+label_len: 0
+learning_rate: 1.0e-05
+ln_mode: post
+loss: stock_tanhv1
+lradj: null
+max_epochs: 100
+mix: false
+model: stockformer
+n_heads: 8
+no_early_stop: false
+no_scale_mean: true
+optim: Adam #Ranger
+output_attention: false
+patience: 1000
+pred_len: 1
+pre_loss: stock_tanhv4
+pre_epochs: 60
+root_path: ./data/stock/
+scale: true
+seed: 1
+seq_len: 32
+t_embed: time2vec_app
+target: WTI_logpctchange

configs/stockformer/basic_PEMSBAY.yaml

@@ -0,0 +1,374 @@
+activation: gelu
+attn: full
+batch_size: 256
+c_out: 1
+cols:
+- 400001_logpctchange
+- 400017_logpctchange
+- 400030_logpctchange
+- 400040_logpctchange
+- 400045_logpctchange
+- 400052_logpctchange
+- 400057_logpctchange
+- 400059_logpctchange
+- 400065_logpctchange
+- 400069_logpctchange
+- 400073_logpctchange
+- 400084_logpctchange
+- 400085_logpctchange
+- 400088_logpctchange
+- 400096_logpctchange
+- 400097_logpctchange
+- 400100_logpctchange
+- 400104_logpctchange
+- 400109_logpctchange
+- 400122_logpctchange
+- 400147_logpctchange
+- 400148_logpctchange
+- 400149_logpctchange
+- 400158_logpctchange
+- 400160_logpctchange
+- 400168_logpctchange
+- 400172_logpctchange
+- 400174_logpctchange
+- 400178_logpctchange
+- 400185_logpctchange
+- 400201_logpctchange
+- 400206_logpctchange
+- 400209_logpctchange
+- 400213_logpctchange
+- 400221_logpctchange
+- 400222_logpctchange
+- 400227_logpctchange
+- 400236_logpctchange
+- 400238_logpctchange
+- 400240_logpctchange
+- 400246_logpctchange
+- 400253_logpctchange
+- 400257_logpctchange
+- 400258_logpctchange
+- 400268_logpctchange
+- 400274_logpctchange
+- 400278_logpctchange
+- 400280_logpctchange
+- 400292_logpctchange
+- 400296_logpctchange
+- 400298_logpctchange
+- 400330_logpctchange
+- 400336_logpctchange
+- 400343_logpctchange
+- 400353_logpctchange
+- 400372_logpctchange
+- 400394_logpctchange
+- 400400_logpctchange
+- 400414_logpctchange
+- 400418_logpctchange
+- 400429_logpctchange
+- 400435_logpctchange
+- 400436_logpctchange
+- 400440_logpctchange
+- 400449_logpctchange
+- 400457_logpctchange
+- 400461_logpctchange
+- 400464_logpctchange
+- 400479_logpctchange
+- 400485_logpctchange
+- 400499_logpctchange
+- 400507_logpctchange
+- 400508_logpctchange
+- 400514_logpctchange
+- 400519_logpctchange
+- 400528_logpctchange
+- 400545_logpctchange
+- 400560_logpctchange
+- 400563_logpctchange
+- 400567_logpctchange
+- 400581_logpctchange
+- 400582_logpctchange
+- 400586_logpctchange
+- 400637_logpctchange
+- 400643_logpctchange
+- 400648_logpctchange
+- 400649_logpctchange
+- 400654_logpctchange
+- 400664_logpctchange
+- 400665_logpctchange
+- 400668_logpctchange
+- 400673_logpctchange
+- 400677_logpctchange
+- 400687_logpctchange
+- 400688_logpctchange
+- 400690_logpctchange
+- 400700_logpctchange
+- 400709_logpctchange
+- 400713_logpctchange
+- 400714_logpctchange
+- 400715_logpctchange
+- 400717_logpctchange
+- 400723_logpctchange
+- 400743_logpctchange
+- 400750_logpctchange
+- 400760_logpctchange
+- 400772_logpctchange
+- 400790_logpctchange
+- 400792_logpctchange
+- 400794_logpctchange
+- 400799_logpctchange
+- 400804_logpctchange
+- 400822_logpctchange
+- 400823_logpctchange
+- 400828_logpctchange
+- 400832_logpctchange
+- 400837_logpctchange
+- 400842_logpctchange
+- 400863_logpctchange
+- 400869_logpctchange
+- 400873_logpctchange
+- 400895_logpctchange
+- 400904_logpctchange
+- 400907_logpctchange
+- 400911_logpctchange
+- 400916_logpctchange
+- 400922_logpctchange
+- 400934_logpctchange
+- 400951_logpctchange
+- 400952_logpctchange
+- 400953_logpctchange
+- 400964_logpctchange
+- 400965_logpctchange
+- 400970_logpctchange
+- 400971_logpctchange
+- 400973_logpctchange
+- 400995_logpctchange
+- 400996_logpctchange
+- 401014_logpctchange
+- 401129_logpctchange
+- 401154_logpctchange
+- 401163_logpctchange
+- 401167_logpctchange
+- 401210_logpctchange
+- 401224_logpctchange
+- 401327_logpctchange
+- 401351_logpctchange
+- 401388_logpctchange
+- 401391_logpctchange
+- 401400_logpctchange
+- 401403_logpctchange
+- 401440_logpctchange
+- 401457_logpctchange
+- 401464_logpctchange
+- 401489_logpctchange
+- 401495_logpctchange
+- 401507_logpctchange
+- 401534_logpctchange
+- 401541_logpctchange
+- 401555_logpctchange
+- 401560_logpctchange
+- 401567_logpctchange
+- 401597_logpctchange
+- 401606_logpctchange
+- 401611_logpctchange
+- 401655_logpctchange
+- 401808_logpctchange
+- 401809_logpctchange
+- 401810_logpctchange
+- 401811_logpctchange
+- 401816_logpctchange
+- 401817_logpctchange
+- 401845_logpctchange
+- 401846_logpctchange
+- 401890_logpctchange
+- 401891_logpctchange
+- 401906_logpctchange
+- 401908_logpctchange
+- 401926_logpctchange
+- 401936_logpctchange
+- 401937_logpctchange
+- 401942_logpctchange
+- 401943_logpctchange
+- 401948_logpctchange
+- 401957_logpctchange
+- 401958_logpctchange
+- 401994_logpctchange
+- 401996_logpctchange
+- 401997_logpctchange
+- 401998_logpctchange
+- 402056_logpctchange
+- 402057_logpctchange
+- 402058_logpctchange
+- 402059_logpctchange
+- 402060_logpctchange
+- 402061_logpctchange
+- 402067_logpctchange
+- 402117_logpctchange
+- 402118_logpctchange
+- 402119_logpctchange
+- 402120_logpctchange
+- 402121_logpctchange
+- 402281_logpctchange
+- 402282_logpctchange
+- 402283_logpctchange
+- 402284_logpctchange
+- 402285_logpctchange
+- 402286_logpctchange
+- 402287_logpctchange
+- 402288_logpctchange
+- 402289_logpctchange
+- 402359_logpctchange
+- 402360_logpctchange
+- 402361_logpctchange
+- 402362_logpctchange
+- 402363_logpctchange
+- 402364_logpctchange
+- 402365_logpctchange
+- 402366_logpctchange
+- 402367_logpctchange
+- 402368_logpctchange
+- 402369_logpctchange
+- 402370_logpctchange
+- 402371_logpctchange
+- 402372_logpctchange
+- 402373_logpctchange
+- 403225_logpctchange
+- 403265_logpctchange
+- 403329_logpctchange
+- 403401_logpctchange
+- 403402_logpctchange
+- 403404_logpctchange
+- 403406_logpctchange
+- 403409_logpctchange
+- 403412_logpctchange
+- 403414_logpctchange
+- 403419_logpctchange
+- 404370_logpctchange
+- 404434_logpctchange
+- 404435_logpctchange
+- 404444_logpctchange
+- 404451_logpctchange
+- 404452_logpctchange
+- 404453_logpctchange
+- 404461_logpctchange
+- 404462_logpctchange
+- 404521_logpctchange
+- 404522_logpctchange
+- 404553_logpctchange
+- 404554_logpctchange
+- 404585_logpctchange
+- 404586_logpctchange
+- 404640_logpctchange
+- 404753_logpctchange
+- 404759_logpctchange
+- 405613_logpctchange
+- 405619_logpctchange
+- 405701_logpctchange
+- 407150_logpctchange
+- 407151_logpctchange
+- 407152_logpctchange
+- 407153_logpctchange
+- 407155_logpctchange
+- 407157_logpctchange
+- 407161_logpctchange
+- 407165_logpctchange
+- 407172_logpctchange
+- 407173_logpctchange
+- 407174_logpctchange
+- 407176_logpctchange
+- 407177_logpctchange
+- 407179_logpctchange
+- 407180_logpctchange
+- 407181_logpctchange
+- 407184_logpctchange
+- 407185_logpctchange
+- 407186_logpctchange
+- 407187_logpctchange
+- 407190_logpctchange
+- 407191_logpctchange
+- 407194_logpctchange
+- 407200_logpctchange
+- 407202_logpctchange
+- 407204_logpctchange
+- 407206_logpctchange
+- 407207_logpctchange
+- 407321_logpctchange
+- 407323_logpctchange
+- 407325_logpctchange
+- 407328_logpctchange
+- 407331_logpctchange
+- 407332_logpctchange
+- 407335_logpctchange
+- 407336_logpctchange
+- 407337_logpctchange
+- 407339_logpctchange
+- 407341_logpctchange
+- 407342_logpctchange
+- 407344_logpctchange
+- 407348_logpctchange
+- 407352_logpctchange
+- 407359_logpctchange
+- 407360_logpctchange
+- 407361_logpctchange
+- 407364_logpctchange
+- 407367_logpctchange
+- 407370_logpctchange
+- 407372_logpctchange
+- 407373_logpctchange
+- 407374_logpctchange
+- 407710_logpctchange
+- 407711_logpctchange
+- 408907_logpctchange
+- 408911_logpctchange
+- 409524_logpctchange
+- 409525_logpctchange
+- 409526_logpctchange
+- 409528_logpctchange
+- 409529_logpctchange
+- 413026_logpctchange
+- 413845_logpctchange
+- 413877_logpctchange
+- 413878_logpctchange
+- 414284_logpctchange
+# - 414694_logpctchange
+d_ff: 4096 #2048
+d_model: 512
+data_path: PEMSBAY.csv
+# date_end: '2020-01-01'
+# date_start: '2012-01-01'
+# date_test: '2019-06-01'
+des: stockPEMS
+distil: false
+dont_shuffle_train: false
+dropout: 0.5
+dropout_emb: 0.0
+e_layers: 4
+emb_t2v_app_dim: 16
+enc_in: 325
+factor: 5
+features: MS
+final_mode: mode3
+freq: m
+inverse_output: false
+inverse_pred: true
+label_len: 0
+learning_rate: 1.0e-04
+ln_mode: post
+loss: stock_tanhv1
+lradj: null
+max_epochs: 30
+mix: false
+model: stockformer
+n_heads: 8
+no_early_stop: false
+no_scale_mean: true
+optim: Adam #Ranger
+output_attention: false
+patience: 1000
+pred_len: 1
+pre_loss: null
+pre_epochs: 0
+root_path: ./data/other/
+scale: true
+seed: 2
+seq_len: 16
+t_embed: null
+tok_emb: default
+target: 400001_logpctchange

configs/stockformer/basic_PEMSBAY_small.yaml

@@ -0,0 +1,373 @@
+activation: gelu
+attn: full
+batch_size: 256
+c_out: 1
+cols:
+- 400001_logpctchange
+- 400017_logpctchange
+- 400030_logpctchange
+- 400040_logpctchange
+- 400045_logpctchange
+- 400052_logpctchange
+- 400057_logpctchange
+- 400059_logpctchange
+- 400065_logpctchange
+- 400069_logpctchange
+# - 400073_logpctchange
+# - 400084_logpctchange
+# - 400085_logpctchange
+# - 400088_logpctchange
+# - 400096_logpctchange
+# - 400097_logpctchange
+# - 400100_logpctchange
+# - 400104_logpctchange
+# - 400109_logpctchange
+# - 400122_logpctchange
+# - 400147_logpctchange
+# - 400148_logpctchange
+# - 400149_logpctchange
+# - 400158_logpctchange
+# - 400160_logpctchange
+# - 400168_logpctchange
+# - 400172_logpctchange
+# - 400174_logpctchange
+# - 400178_logpctchange
+# - 400185_logpctchange
+# - 400201_logpctchange
+# - 400206_logpctchange
+# - 400209_logpctchange
+# - 400213_logpctchange
+# - 400221_logpctchange
+# - 400222_logpctchange
+# - 400227_logpctchange
+# - 400236_logpctchange
+# - 400238_logpctchange
+# - 400240_logpctchange
+# - 400246_logpctchange
+# - 400253_logpctchange
+# - 400257_logpctchange
+# - 400258_logpctchange
+# - 400268_logpctchange
+# - 400274_logpctchange
+# - 400278_logpctchange
+# - 400280_logpctchange
+# - 400292_logpctchange
+# - 400296_logpctchange
+# - 400298_logpctchange
+# - 400330_logpctchange
+# - 400336_logpctchange
+# - 400343_logpctchange
+# - 400353_logpctchange
+# - 400372_logpctchange
+# - 400394_logpctchange
+# - 400400_logpctchange
+# - 400414_logpctchange
+# - 400418_logpctchange
+# - 400429_logpctchange
+# - 400435_logpctchange
+# - 400436_logpctchange
+# - 400440_logpctchange
+# - 400449_logpctchange
+# - 400457_logpctchange
+# - 400461_logpctchange
+# - 400464_logpctchange
+# - 400479_logpctchange
+# - 400485_logpctchange
+# - 400499_logpctchange
+# - 400507_logpctchange
+# - 400508_logpctchange
+# - 400514_logpctchange
+# - 400519_logpctchange
+# - 400528_logpctchange
+# - 400545_logpctchange
+# - 400560_logpctchange
+# - 400563_logpctchange
+# - 400567_logpctchange
+# - 400581_logpctchange
+# - 400582_logpctchange
+# - 400586_logpctchange
+# - 400637_logpctchange
+# - 400643_logpctchange
+# - 400648_logpctchange
+# - 400649_logpctchange
+# - 400654_logpctchange
+# - 400664_logpctchange
+# - 400665_logpctchange
+# - 400668_logpctchange
+# - 400673_logpctchange
+# - 400677_logpctchange
+# - 400687_logpctchange
+# - 400688_logpctchange
+# - 400690_logpctchange
+# - 400700_logpctchange
+# - 400709_logpctchange
+# - 400713_logpctchange
+# - 400714_logpctchange
+# - 400715_logpctchange
+# - 400717_logpctchange
+# - 400723_logpctchange
+# - 400743_logpctchange
+# - 400750_logpctchange
+# - 400760_logpctchange
+# - 400772_logpctchange
+# - 400790_logpctchange
+# - 400792_logpctchange
+# - 400794_logpctchange
+# - 400799_logpctchange
+# - 400804_logpctchange
+# - 400822_logpctchange
+# - 400823_logpctchange
+# - 400828_logpctchange
+# - 400832_logpctchange
+# - 400837_logpctchange
+# - 400842_logpctchange
+# - 400863_logpctchange
+# - 400869_logpctchange
+# - 400873_logpctchange
+# - 400895_logpctchange
+# - 400904_logpctchange
+# - 400907_logpctchange
+# - 400911_logpctchange
+# - 400916_logpctchange
+# - 400922_logpctchange
+# - 400934_logpctchange
+# - 400951_logpctchange
+# - 400952_logpctchange
+# - 400953_logpctchange
+# - 400964_logpctchange
+# - 400965_logpctchange
+# - 400970_logpctchange
+# - 400971_logpctchange
+# - 400973_logpctchange
+# - 400995_logpctchange
+# - 400996_logpctchange
+# - 401014_logpctchange
+# - 401129_logpctchange
+# - 401154_logpctchange
+# - 401163_logpctchange
+# - 401167_logpctchange
+# - 401210_logpctchange
+# - 401224_logpctchange
+# - 401327_logpctchange
+# - 401351_logpctchange
+# - 401388_logpctchange
+# - 401391_logpctchange
+# - 401400_logpctchange
+# - 401403_logpctchange
+# - 401440_logpctchange
+# - 401457_logpctchange
+# - 401464_logpctchange
+# - 401489_logpctchange
+# - 401495_logpctchange
+# - 401507_logpctchange
+# - 401534_logpctchange
+# - 401541_logpctchange
+# - 401555_logpctchange
+# - 401560_logpctchange
+# - 401567_logpctchange
+# - 401597_logpctchange
+# - 401606_logpctchange
+# - 401611_logpctchange
+# - 401655_logpctchange
+# - 401808_logpctchange
+# - 401809_logpctchange
+# - 401810_logpctchange
+# - 401811_logpctchange
+# - 401816_logpctchange
+# - 401817_logpctchange
+# - 401845_logpctchange
+# - 401846_logpctchange
+# - 401890_logpctchange
+# - 401891_logpctchange
+# - 401906_logpctchange
+# - 401908_logpctchange
+# - 401926_logpctchange
+# - 401936_logpctchange
+# - 401937_logpctchange
+# - 401942_logpctchange
+# - 401943_logpctchange
+# - 401948_logpctchange
+# - 401957_logpctchange
+# - 401958_logpctchange
+# - 401994_logpctchange
+# - 401996_logpctchange
+# - 401997_logpctchange
+# - 401998_logpctchange
+# - 402056_logpctchange
+# - 402057_logpctchange
+# - 402058_logpctchange
+# - 402059_logpctchange
+# - 402060_logpctchange
+# - 402061_logpctchange
+# - 402067_logpctchange
+# - 402117_logpctchange
+# - 402118_logpctchange
+# - 402119_logpctchange
+# - 402120_logpctchange
+# - 402121_logpctchange
+# - 402281_logpctchange
+# - 402282_logpctchange
+# - 402283_logpctchange
+# - 402284_logpctchange
+# - 402285_logpctchange
+# - 402286_logpctchange
+# - 402287_logpctchange
+# - 402288_logpctchange
+# - 402289_logpctchange
+# - 402359_logpctchange
+# - 402360_logpctchange
+# - 402361_logpctchange
+# - 402362_logpctchange
+# - 402363_logpctchange
+# - 402364_logpctchange
+# - 402365_logpctchange
+# - 402366_logpctchange
+# - 402367_logpctchange
+# - 402368_logpctchange
+# - 402369_logpctchange
+# - 402370_logpctchange
+# - 402371_logpctchange
+# - 402372_logpctchange
+# - 402373_logpctchange
+# - 403225_logpctchange
+# - 403265_logpctchange
+# - 403329_logpctchange
+# - 403401_logpctchange
+# - 403402_logpctchange
+# - 403404_logpctchange
+# - 403406_logpctchange
+# - 403409_logpctchange
+# - 403412_logpctchange
+# - 403414_logpctchange
+# - 403419_logpctchange
+# - 404370_logpctchange
+# - 404434_logpctchange
+# - 404435_logpctchange
+# - 404444_logpctchange
+# - 404451_logpctchange
+# - 404452_logpctchange
+# - 404453_logpctchange
+# - 404461_logpctchange
+# - 404462_logpctchange
+# - 404521_logpctchange
+# - 404522_logpctchange
+# - 404553_logpctchange
+# - 404554_logpctchange
+# - 404585_logpctchange
+# - 404586_logpctchange
+# - 404640_logpctchange
+# - 404753_logpctchange
+# - 404759_logpctchange
+# - 405613_logpctchange
+# - 405619_logpctchange
+# - 405701_logpctchange
+# - 407150_logpctchange
+# - 407151_logpctchange
+# - 407152_logpctchange
+# - 407153_logpctchange
+# - 407155_logpctchange
+# - 407157_logpctchange
+# - 407161_logpctchange
+# - 407165_logpctchange
+# - 407172_logpctchange
+# - 407173_logpctchange
+# - 407174_logpctchange
+# - 407176_logpctchange
+# - 407177_logpctchange
+# - 407179_logpctchange
+# - 407180_logpctchange
+# - 407181_logpctchange
+# - 407184_logpctchange
+# - 407185_logpctchange
+# - 407186_logpctchange
+# - 407187_logpctchange
+# - 407190_logpctchange
+# - 407191_logpctchange
+# - 407194_logpctchange
+# - 407200_logpctchange
+# - 407202_logpctchange
+# - 407204_logpctchange
+# - 407206_logpctchange
+# - 407207_logpctchange
+# - 407321_logpctchange
+# - 407323_logpctchange
+# - 407325_logpctchange
+# - 407328_logpctchange
+# - 407331_logpctchange
+# - 407332_logpctchange
+# - 407335_logpctchange
+# - 407336_logpctchange
+# - 407337_logpctchange
+# - 407339_logpctchange
+# - 407341_logpctchange
+# - 407342_logpctchange
+# - 407344_logpctchange
+# - 407348_logpctchange
+# - 407352_logpctchange
+# - 407359_logpctchange
+# - 407360_logpctchange
+# - 407361_logpctchange
+# - 407364_logpctchange
+# - 407367_logpctchange
+# - 407370_logpctchange
+# - 407372_logpctchange
+# - 407373_logpctchange
+# - 407374_logpctchange
+# - 407710_logpctchange
+# - 407711_logpctchange
+# - 408907_logpctchange
+# - 408911_logpctchange
+# - 409524_logpctchange
+# - 409525_logpctchange
+# - 409526_logpctchange
+# - 409528_logpctchange
+# - 409529_logpctchange
+# - 413026_logpctchange
+# - 413845_logpctchange
+# - 413877_logpctchange
+# - 413878_logpctchange
+# - 414284_logpctchange
+# - 414694_logpctchange
+d_ff: 4096 #2048
+d_model: 512
+data_path: PEMSBAY.csv
+# date_end: '2020-01-01'
+# date_start: '2012-01-01'
+# date_test: '2019-06-01'
+des: full_1h
+distil: false
+dont_shuffle_train: false
+dropout: 0.5
+dropout_emb: 0.05
+e_layers: 3
+emb_t2v_app_dim: 16
+enc_in: 10
+factor: 5
+features: MS
+final_mode: mode3
+freq: m
+inverse_output: false
+inverse_pred: true
+label_len: 0
+learning_rate: 1.0e-04
+ln_mode: post
+loss: stock_tanhv1
+lradj: null
+max_epochs: 30
+mix: false
+model: stockformer
+n_heads: 8
+no_early_stop: false
+no_scale_mean: true
+optim: Adam #Ranger
+output_attention: false
+patience: 1000
+pred_len: 1
+pre_loss: null
+pre_epochs: 0
+root_path: ./data/other/
+scale: true
+seed: 2
+seq_len: 16
+t_embed: time2vec_add
+target: 400001_logpctchange

configs/stockformer/basic_WTH.yaml

@@ -0,0 +1,60 @@
+activation: gelu
+attn: full
+batch_size: 256
+c_out: 1
+cols:
+- Visibility_logpctchange
+- DryBulbFarenheit_logpctchange
+- DryBulbCelsius_logpctchange
+- WetBulbFarenheit_logpctchange
+- DewPointFarenheit_logpctchange
+- DewPointCelsius_logpctchange
+- RelativeHumidity_logpctchange
+- WindSpeed_logpctchange
+- WindDirection_logpctchange
+- StationPressure_logpctchange
+- Altimeter_logpctchange
+- WetBulbCelsius_logpctchange
+d_ff: 4096 #2048
+d_model: 512
+data_path: WTH.csv
+# date_end: '2020-01-01'
+# date_start: '2012-01-01'
+# date_test: '2019-06-01'
+des: full_1h
+distil: false
+dont_shuffle_train: false
+dropout: 0.5
+dropout_emb: 0.0
+e_layers: 4
+emb_t2v_app_dim: 16
+enc_in: 12
+factor: 5
+features: MS
+final_mode: mode3
+freq: h
+inverse_output: false
+inverse_pred: true
+label_len: 0
+learning_rate: 1.0e-05
+ln_mode: post
+loss: stock_tanhv1
+lradj: null
+max_epochs: 30
+mix: false
+model: stockformer
+n_heads: 32
+no_early_stop: false
+no_scale_mean: true
+optim: Adam #Ranger
+output_attention: false
+patience: 1000
+pred_len: 1
+pre_loss: stock_tanhv4
+pre_epochs: 15
+root_path: ./data/other/
+scale: true
+seed: 5
+seq_len: 16
+t_embed: null
+target: WetBulbFarenheit_logpctchange

configs/stockformer/basic_material.yaml

@@ -0,0 +1,48 @@
+activation: gelu
+attn: full
+batch_size: 256
+c_out: 1
+cols: null
+d_ff: 4096 #2048
+d_model: 512
+data_path: material_1h.csv
+# date_end: '2020-01-01'
+# date_start: '2012-01-01'
+# date_test: '2019-06-01'
+des: full_1h
+distil: false
+dont_shuffle_train: false
+dropout: 0.5
+dropout_emb: 0.0
+e_layers: 4
+emb_t2v_app_dim: 16
+enc_in: 9
+factor: 5
+features: MS
+final_mode: mode3
+freq: h
+inverse_output: false
+inverse_pred: true
+label_len: 0
+learning_rate: 1.0e-05
+ln_mode: post
+loss: stock_tanhv1
+lradj: null
+max_epochs: 50
+mix: false
+model: stockformer
+n_heads: 16
+no_early_stop: false
+no_scale_mean: true
+optim: Adam #Ranger
+output_attention: false
+patience: 1000
+pred_len: 1
+pre_loss: stock_tanhv4
+pre_epochs: 30
+root_path: ./data/stock/
+scale: true
+seed: 1
+seq_len: 32
+t_embed: time2vec_add
+target: FCX_logpctchange

configs/stockformer/general.yaml

@@ -0,0 +1,61 @@
+activation: gelu
+attn: full
+batch_size: 208
+c_out: 1
+cols:
+- XOM_logpctchange
+- CVX_logpctchange
+- COP_logpctchange
+- BP_logpctchange
+- PBR_logpctchange
+- EOG_logpctchange
+- WTI_logpctchange
+# - ENB_logpctchange
+# - SLB_logpctchange
+d_ff: 4096 #2048
+d_model: 512
+data_path: material_1h.csv
+# date_end: '2020-01-01'
+# date_start: '2012-01-01'
+# date_test: '2019-06-01'
+date_end: '2025-10-23'
+date_start: '2020-10-26'
+date_test: '2025-06-26'
+des: full_1h
+distil: false
+dont_shuffle_train: false 
+dropout: 0.5
+dropout_emb: 0.0
+t_embed: time2vec_app
+enc_in: 7
+factor: 5
+features: MS
+freq: h
+inverse_pred: true
+# inverse_output: false
+loss: stock_tanhv1 
+# inverse_output: true 
+# loss: stock_tanhv1+mae
+final_mode: mode3
+label_len: 0
+learning_rate: 1.0e-05
+seq_len: 64
+e_layers: 12
+emb_t2v_app_dim: 16
+ln_mode: post
+lradj: null
+mix: false
+model: stockformer
+n_heads: 512
+no_early_stop: true
+no_scale_mean: true
+# optim: Ranger
+optim: AdamW
+output_attention: false
+patience: 100
+pred_len: 1
+root_path: ./data/stock/
+scale: true
+seed: 4
+target: WTI_logpctchange
+max_epochs: 100 

configs/stockformer/general_PEMSBAY.yaml

@@ -0,0 +1,373 @@
+activation: gelu
+attn: full
+batch_size: 256
+c_out: 1
+cols:
+- 400001_logpctchange
+- 400017_logpctchange
+- 400030_logpctchange
+- 400040_logpctchange
+- 400045_logpctchange
+- 400052_logpctchange
+- 400057_logpctchange
+- 400059_logpctchange
+- 400065_logpctchange
+- 400069_logpctchange
+# - 400073_logpctchange
+# - 400084_logpctchange
+# - 400085_logpctchange
+# - 400088_logpctchange
+# - 400096_logpctchange
+# - 400097_logpctchange
+# - 400100_logpctchange
+# - 400104_logpctchange
+# - 400109_logpctchange
+# - 400122_logpctchange
+# - 400147_logpctchange
+# - 400148_logpctchange
+# - 400149_logpctchange
+# - 400158_logpctchange
+# - 400160_logpctchange
+# - 400168_logpctchange
+# - 400172_logpctchange
+# - 400174_logpctchange
+# - 400178_logpctchange
+# - 400185_logpctchange
+# - 400201_logpctchange
+# - 400206_logpctchange
+# - 400209_logpctchange
+# - 400213_logpctchange
+# - 400221_logpctchange
+# - 400222_logpctchange
+# - 400227_logpctchange
+# - 400236_logpctchange
+# - 400238_logpctchange
+# - 400240_logpctchange
+# - 400246_logpctchange
+# - 400253_logpctchange
+# - 400257_logpctchange
+# - 400258_logpctchange
+# - 400268_logpctchange
+# - 400274_logpctchange
+# - 400278_logpctchange
+# - 400280_logpctchange
+# - 400292_logpctchange
+# - 400296_logpctchange
+# - 400298_logpctchange
+# - 400330_logpctchange
+# - 400336_logpctchange
+# - 400343_logpctchange
+# - 400353_logpctchange
+# - 400372_logpctchange
+# - 400394_logpctchange
+# - 400400_logpctchange
+# - 400414_logpctchange
+# - 400418_logpctchange
+# - 400429_logpctchange
+# - 400435_logpctchange
+# - 400436_logpctchange
+# - 400440_logpctchange
+# - 400449_logpctchange
+# - 400457_logpctchange
+# - 400461_logpctchange
+# - 400464_logpctchange
+# - 400479_logpctchange
+# - 400485_logpctchange
+# - 400499_logpctchange
+# - 400507_logpctchange
+# - 400508_logpctchange
+# - 400514_logpctchange
+# - 400519_logpctchange
+# - 400528_logpctchange
+# - 400545_logpctchange
+# - 400560_logpctchange
+# - 400563_logpctchange
+# - 400567_logpctchange
+# - 400581_logpctchange
+# - 400582_logpctchange
+# - 400586_logpctchange
+# - 400637_logpctchange
+# - 400643_logpctchange
+# - 400648_logpctchange
+# - 400649_logpctchange
+# - 400654_logpctchange
+# - 400664_logpctchange
+# - 400665_logpctchange
+# - 400668_logpctchange
+# - 400673_logpctchange
+# - 400677_logpctchange
+# - 400687_logpctchange
+# - 400688_logpctchange
+# - 400690_logpctchange
+# - 400700_logpctchange
+# - 400709_logpctchange
+# - 400713_logpctchange
+# - 400714_logpctchange
+# - 400715_logpctchange
+# - 400717_logpctchange
+# - 400723_logpctchange
+# - 400743_logpctchange
+# - 400750_logpctchange
+# - 400760_logpctchange
+# - 400772_logpctchange
+# - 400790_logpctchange
+# - 400792_logpctchange
+# - 400794_logpctchange
+# - 400799_logpctchange
+# - 400804_logpctchange
+# - 400822_logpctchange
+# - 400823_logpctchange
+# - 400828_logpctchange
+# - 400832_logpctchange
+# - 400837_logpctchange
+# - 400842_logpctchange
+# - 400863_logpctchange
+# - 400869_logpctchange
+# - 400873_logpctchange
+# - 400895_logpctchange
+# - 400904_logpctchange
+# - 400907_logpctchange
+# - 400911_logpctchange
+# - 400916_logpctchange
+# - 400922_logpctchange
+# - 400934_logpctchange
+# - 400951_logpctchange
+# - 400952_logpctchange
+# - 400953_logpctchange
+# - 400964_logpctchange
+# - 400965_logpctchange
+# - 400970_logpctchange
+# - 400971_logpctchange
+# - 400973_logpctchange
+# - 400995_logpctchange
+# - 400996_logpctchange
+# - 401014_logpctchange
+# - 401129_logpctchange
+# - 401154_logpctchange
+# - 401163_logpctchange
+# - 401167_logpctchange
+# - 401210_logpctchange
+# - 401224_logpctchange
+# - 401327_logpctchange
+# - 401351_logpctchange
+# - 401388_logpctchange
+# - 401391_logpctchange
+# - 401400_logpctchange
+# - 401403_logpctchange
+# - 401440_logpctchange
+# - 401457_logpctchange
+# - 401464_logpctchange
+# - 401489_logpctchange
+# - 401495_logpctchange
+# - 401507_logpctchange
+# - 401534_logpctchange
+# - 401541_logpctchange
+# - 401555_logpctchange
+# - 401560_logpctchange
+# - 401567_logpctchange
+# - 401597_logpctchange
+# - 401606_logpctchange
+# - 401611_logpctchange
+# - 401655_logpctchange
+# - 401808_logpctchange
+# - 401809_logpctchange
+# - 401810_logpctchange
+# - 401811_logpctchange
+# - 401816_logpctchange
+# - 401817_logpctchange
+# - 401845_logpctchange
+# - 401846_logpctchange
+# - 401890_logpctchange
+# - 401891_logpctchange
+# - 401906_logpctchange
+# - 401908_logpctchange
+# - 401926_logpctchange
+# - 401936_logpctchange
+# - 401937_logpctchange
+# - 401942_logpctchange
+# - 401943_logpctchange
+# - 401948_logpctchange
+# - 401957_logpctchange
+# - 401958_logpctchange
+# - 401994_logpctchange
+# - 401996_logpctchange
+# - 401997_logpctchange
+# - 401998_logpctchange
+# - 402056_logpctchange
+# - 402057_logpctchange
+# - 402058_logpctchange
+# - 402059_logpctchange
+# - 402060_logpctchange
+# - 402061_logpctchange
+# - 402067_logpctchange
+# - 402117_logpctchange
+# - 402118_logpctchange
+# - 402119_logpctchange
+# - 402120_logpctchange
+# - 402121_logpctchange
+# - 402281_logpctchange
+# - 402282_logpctchange
+# - 402283_logpctchange
+# - 402284_logpctchange
+# - 402285_logpctchange
+# - 402286_logpctchange
+# - 402287_logpctchange
+# - 402288_logpctchange
+# - 402289_logpctchange
+# - 402359_logpctchange
+# - 402360_logpctchange
+# - 402361_logpctchange
+# - 402362_logpctchange
+# - 402363_logpctchange
+# - 402364_logpctchange
+# - 402365_logpctchange
+# - 402366_logpctchange
+# - 402367_logpctchange
+# - 402368_logpctchange
+# - 402369_logpctchange
+# - 402370_logpctchange
+# - 402371_logpctchange
+# - 402372_logpctchange
+# - 402373_logpctchange
+# - 403225_logpctchange
+# - 403265_logpctchange
+# - 403329_logpctchange
+# - 403401_logpctchange
+# - 403402_logpctchange
+# - 403404_logpctchange
+# - 403406_logpctchange
+# - 403409_logpctchange
+# - 403412_logpctchange
+# - 403414_logpctchange
+# - 403419_logpctchange
+# - 404370_logpctchange
+# - 404434_logpctchange
+# - 404435_logpctchange
+# - 404444_logpctchange
+# - 404451_logpctchange
+# - 404452_logpctchange
+# - 404453_logpctchange
+# - 404461_logpctchange
+# - 404462_logpctchange
+# - 404521_logpctchange
+# - 404522_logpctchange
+# - 404553_logpctchange
+# - 404554_logpctchange
+# - 404585_logpctchange
+# - 404586_logpctchange
+# - 404640_logpctchange
+# - 404753_logpctchange
+# - 404759_logpctchange
+# - 405613_logpctchange
+# - 405619_logpctchange
+# - 405701_logpctchange
+# - 407150_logpctchange
+# - 407151_logpctchange
+# - 407152_logpctchange
+# - 407153_logpctchange
+# - 407155_logpctchange
+# - 407157_logpctchange
+# - 407161_logpctchange
+# - 407165_logpctchange
+# - 407172_logpctchange
+# - 407173_logpctchange
+# - 407174_logpctchange
+# - 407176_logpctchange
+# - 407177_logpctchange
+# - 407179_logpctchange
+# - 407180_logpctchange
+# - 407181_logpctchange
+# - 407184_logpctchange
+# - 407185_logpctchange
+# - 407186_logpctchange
+# - 407187_logpctchange
+# - 407190_logpctchange
+# - 407191_logpctchange
+# - 407194_logpctchange
+# - 407200_logpctchange
+# - 407202_logpctchange
+# - 407204_logpctchange
+# - 407206_logpctchange
+# - 407207_logpctchange
+# - 407321_logpctchange
+# - 407323_logpctchange
+# - 407325_logpctchange
+# - 407328_logpctchange
+# - 407331_logpctchange
+# - 407332_logpctchange
+# - 407335_logpctchange
+# - 407336_logpctchange
+# - 407337_logpctchange
+# - 407339_logpctchange
+# - 407341_logpctchange
+# - 407342_logpctchange
+# - 407344_logpctchange
+# - 407348_logpctchange
+# - 407352_logpctchange
+# - 407359_logpctchange
+# - 407360_logpctchange
+# - 407361_logpctchange
+# - 407364_logpctchange
+# - 407367_logpctchange
+# - 407370_logpctchange
+# - 407372_logpctchange
+# - 407373_logpctchange
+# - 407374_logpctchange
+# - 407710_logpctchange
+# - 407711_logpctchange
+# - 408907_logpctchange
+# - 408911_logpctchange
+# - 409524_logpctchange
+# - 409525_logpctchange
+# - 409526_logpctchange
+# - 409528_logpctchange
+# - 409529_logpctchange
+# - 413026_logpctchange
+# - 413845_logpctchange
+# - 413877_logpctchange
+# - 413878_logpctchange
+# - 414284_logpctchange
+# - 414694_logpctchange
+d_ff: 4096 #2048
+d_model: 512
+data_path: PEMSBAY.csv
+# date_end: '2020-01-01'
+# date_start: '2012-01-01'
+# date_test: '2019-06-01'
+des: full_1h
+distil: false
+dont_shuffle_train: false
+dropout: 0.5
+dropout_emb: 0.0
+e_layers: 12
+emb_t2v_app_dim: 16
+enc_in: 10 #325
+factor: 5
+features: MS
+final_mode: mode3
+freq: m
+inverse_output: false
+inverse_pred: true
+label_len: 0
+learning_rate: 1.0e-05
+ln_mode: post
+loss: stock_tanhv1
+lradj: null
+max_epochs: 30
+mix: false
+model: stockformer
+n_heads: 512
+no_early_stop: false
+no_scale_mean: true
+optim: Adam #Ranger
+output_attention: false
+patience: 1000
+pred_len: 1
+pre_loss: null #stock_tanhv4
+pre_epochs: 0 #15
+root_path: ./data/other/
+scale: true
+seed: 2
+seq_len: 16
+t_embed: time2vec_add
+target: 400001_logpctchange

d.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+mkdir -p stock_transformer_back
+cd stock_transformer_back
+
+
+STUDIO="hallucination/slow-rose-kq8y"
+
+########################
+# 1. 目录：先 mkdir，再下载
+########################
+
+# configs/
+mkdir -p configs
+lightning download folder stock_transformer_back/configs \
+  --studio "$STUDIO" \
+  --local-path ./configs
+
+# data/
+mkdir -p data
+lightning download folder stock_transformer_back/data \
+  --studio "$STUDIO" \
+  --local-path ./data
+
+# data_provider/
+mkdir -p data_provider
+lightning download folder stock_transformer_back/data_provider \
+  --studio "$STUDIO" \
+  --local-path ./data_provider
+
+# exp/
+mkdir -p exp
+lightning download folder stock_transformer_back/exp \
+  --studio "$STUDIO" \
+  --local-path ./exp
+
+# layers/
+mkdir -p layers
+lightning download folder stock_transformer_back/layers \
+  --studio "$STUDIO" \
+  --local-path ./layers
+
+# models/
+mkdir -p models
+lightning download folder stock_transformer_back/models \
+  --studio "$STUDIO" \
+  --local-path ./models
+
+# old_stuff/
+mkdir -p old_stuff
+lightning download folder stock_transformer_back/old_stuff \
+  --studio "$STUDIO" \
+  --local-path ./old_stuff
+
+# utils/
+mkdir -p utils
+lightning download folder stock_transformer_back/utils \
+  --studio "$STUDIO" \
+  --local-path ./utils
+
+# 注意：不下 lightning_logs，所以这里不写它
+
+
+########################
+# 2. 单个文件：直接下到当前目录
+########################
+
+lightning download file stock_transformer_back/.gitignore     --studio "$STUDIO"
+lightning download file stock_transformer_back/.tmux.conf     --studio "$STUDIO"
+lightning download file stock_transformer_back/LICENSE        --studio "$STUDIO"
+lightning download file stock_transformer_back/README.md      --studio "$STUDIO"
+lightning download file stock_transformer_back/Stockformer.py --studio "$STUDIO"
+lightning download file stock_transformer_back/clean_ipynb.sh --studio "$STUDIO"
+
+lightning download file stock_transformer_back/data_collect.ipynb  --studio "$STUDIO"
+lightning download file stock_transformer_back/data_collect.py     --studio "$STUDIO"
+lightning download file stock_transformer_back/data_loader.py      --studio "$STUDIO"
+lightning download file stock_transformer_back/data_prepare.ipynb  --studio "$STUDIO"
+lightning download file stock_transformer_back/data_prepare.py     --studio "$STUDIO"
+
+lightning download file stock_transformer_back/embed.py        --studio "$STUDIO"
+lightning download file stock_transformer_back/exp_timeseries.py --studio "$STUDIO"
+
+lightning download file stock_transformer_back/run_bbtest.py   --studio "$STUDIO"
+lightning download file stock_transformer_back/run_hypopt.py   --studio "$STUDIO"
+lightning download file stock_transformer_back/run_once.py     --studio "$STUDIO"
+
+lightning download file stock_transformer_back/stock_metrics.py --studio "$STUDIO"
+lightning download file stock_transformer_back/tools.py         --studio "$STUDIO"
+
+lightning download file stock_transformer_back/vis_data.ipynb    --studio "$STUDIO"
+lightning download file stock_transformer_back/vis_results.ipynb --studio "$STUDIO"
+
+lightning download file stock_transformer_back/wavelet.py        --studio "$STUDIO"
+

data_collect.ipynb

@@ -0,0 +1,536 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from time import sleep\n",
+    "import datetime\n",
+    "import os\n",
+    "from utils.ipynb_helpers import read_data, write_df, convert_tz, add_tz\n",
+    "from dotenv import load_dotenv\n",
+    "\n",
+    "# Create a .env file and add your keys\n",
+    "load_dotenv()\n",
+    "\n",
+    "# Location to save raw data from data providers\n",
+    "DATA_RAW = \"data/raw\"\n",
+    "\n",
+    "\n",
+    "equities = [\"XOM\", \"CVX\", \"COP\", \"BP\", \"PBR\", \"WTI\", \"TTE\", \"EQNR\", \"EOG\", \"ENB\", \"SLB\"]\n",
+    "more_equities = []\n",
+    "\n",
+    "crude_oil = [\"CL=F\", \"BZ=F\"]  # wti, brent,\n",
+    "random = [\"TSLA\", \"AAPL\"]\n",
+    "\n",
+    "materials_equities = [\"BHP\", \"LIN\", \"RIO\", \"VALE\", \"APD\", \"FCX\", \"SHW\", \"SCCO\", \"CTVA\", \"ECL\", \"NUE\", \"NTR\"]\n",
+    "\n",
+    "\n",
+    "# https://en.wikipedia.org/wiki/List_of_countries_by_oil_production\n",
+    "# https://www.weforum.org/agenda/2016/05/which-economies-are-most-reliant-on-oil/\n",
+    "# OPEC: Iran, Iraq, Kuwait, Saudi Arabia, Venezuela\n",
+    "# fx_opec = [_, \"C:USDIQD\", \"C:USDKWD\", \"C:USDSAR\", \"C:USDVEF\"]\n",
+    "\n",
+    "# OPEC+: Algeria, Angola, Congo, Equatorial Guinea, Gabon, Libya, Nigeria, United Arab Emirates\n",
+    "# fx_opec_pp = [\"C:USDDZD\",_, \"C:USDCDF\", \"C:USDGNF\", _, \"C:USDLYD\", \"C:USDNGN\", \"C:USDAED\"]\n",
+    "\n",
+    "# Large: US, Russia, China, Canada, Norway\n",
+    "# Other important: Qatar, Kazakhstan\n",
+    "# fx_other= [\"C:USDQAR\", \"C:USDKZT\"]\n",
+    "\n",
+    "fx = [\"C:USDSAR\", \"C:USDAED\"]\n",
+    "\n",
+    "tickers = equities  # + crude_oil"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Get Data From Data Provider"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Y Finance\n",
+    "\n",
+    "import yfinance as yf\n",
+    "\n",
+    "\n",
+    "def use_yfinance(\n",
+    "    tickers, out_file, timeframe=\"day\", start=\"2000-01-01\", end=\"2023-01-01\"\n",
+    "):\n",
+    "    assert timeframe == \"day\", \"Use day timeframe for day\"\n",
+    "\n",
+    "    data = yf.download(tickers, start=start, end=end, group_by=\"ticker\")\n",
+    "\n",
+    "    if len(tickers) == 1:\n",
+    "        data = pd.concat([data], axis=1, keys=[tickers[0]])\n",
+    "\n",
+    "    data.index.rename(\"date\", inplace=True)\n",
+    "    data.rename(columns=lambda x: str.lower(x), level=1, inplace=True)\n",
+    "\n",
+    "    if data.index.to_series().dt.tz is None:\n",
+    "        print(\"Adding time\")\n",
+    "        data = add_tz(data, time_zone=\"UTC\")\n",
+    "\n",
+    "    if out_file is not None:\n",
+    "        write_df(data, out_file)\n",
+    "\n",
+    "    return data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Alpha Vantage\n",
+    "\n",
+    "\n",
+    "def csv_str_to_df(decoded_content, ticker):\n",
+    "    \"\"\"CSV string to df\"\"\"\n",
+    "    print(decoded_content[:1000])\n",
+    "    lines = decoded_content.splitlines()\n",
+    "    print(len(lines), lines[0].split(\",\")[1:])\n",
+    "    print(lines[2])\n",
+    "    #while(1):pass\n",
+    "    data = pd.DataFrame(\n",
+    "        [row.split(\",\") for row in lines[1:]],\n",
+    "        columns=[\"date\", *lines[0].split(\",\")[1:]],\n",
+    "    )\n",
+    "    \n",
+    "\n",
+    "    data = data.reset_index(drop=True).set_index(\"date\")\n",
+    "    data.index = pd.to_datetime(data.index)\n",
+    "\n",
+    "    # Add timezome -- we assume it is sent in with unlabled eastern time\n",
+    "    if data.index.to_series().dt.tz is None:\n",
+    "        print(\"CONVERTING TIME\")\n",
+    "        data = add_tz(data, time_zone=\"US/Eastern\")\n",
+    "        data = convert_tz(data, time_zone=\"UTC\")\n",
+    "    data = pd.concat([data], axis=1, keys=[ticker])\n",
+    "    return data\n",
+    "\n",
+    "\n",
+    "def alpha_vantage_get_ticker_data(ticker, time=\"1min\", year=1, month=1):\n",
+    "    \"\"\"Function to get (ticker, year, month) data using alpha vantage's time series intraday extended API\"\"\"\n",
+    "    ALPHA_VANTAGE_API_KEY = os.environ.get(\"ALPHA_VANTAGE_API_KEY\")\n",
+    "    import requests\n",
+    "\n",
+    "    CSV_URL = f\"https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol={ticker}&interval={time}&month={2026-year}-{11-month:02d}&outputsize=full&apikey={ALPHA_VANTAGE_API_KEY}\"\n",
+    "\n",
+    "    while True:\n",
+    "        with requests.Session() as s:\n",
+    "            download = s.get(CSV_URL)\n",
+    "            decoded_content = download.content.decode(\"utf-8\")\n",
+    "            print(\n",
+    "                f\"ticker: {ticker}, y{year} m{month}; response length: {len(decoded_content)}\"\n",
+    "            )\n",
+    "\n",
+    "            if len(decoded_content) == 236:\n",
+    "                # API too many requests\n",
+    "                sleep(60)\n",
+    "            elif len(decoded_content) <= 243:\n",
+    "                # Token doesn't exist or something\n",
+    "                print(f\"Error getting {ticker}, y{year}, m{month}. We are skipping\")\n",
+    "                print(decoded_content)\n",
+    "                return None\n",
+    "            else:\n",
+    "                return csv_str_to_df(decoded_content, ticker)\n",
+    "\n",
+    "\n",
+    "def use_alpha_vantage(tickers, out_file, time=\"1min\"):\n",
+    "    \"\"\"Function to get multiple full tickers data using alpha vantage's time series intraday extended API\"\"\"\n",
+    "\n",
+    "    dfs = []\n",
+    "    for ticker in tickers:\n",
+    "        t_dfs = []\n",
+    "        for year in range(1, 3):\n",
+    "            for month in range(1, 13):\n",
+    "                df_temp = alpha_vantage_get_ticker_data(\n",
+    "                    ticker, time=time, year=year, month=month\n",
+    "                )\n",
+    "                if df_temp is not None:\n",
+    "                    t_dfs.append(df_temp)\n",
+    "\n",
+    "        if len(t_dfs):\n",
+    "            dfs.append(pd.concat(t_dfs, axis=0))\n",
+    "        else:\n",
+    "            print(f\"Skipped {ticker}.\")\n",
+    "    df = pd.concat(dfs, axis=1, sort=True)\n",
+    "    df.index.rename(\"date\", inplace=True)\n",
+    "\n",
+    "    write_df(df, out_file)\n",
+    "\n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Alpaca\n",
+    "\n",
+    "\n",
+    "def use_alpaca(tickers, out_file, timeframe=\"1Minute\", start=\"2017-01-01\"):\n",
+    "    APCA_API_BASE_URL = os.environ.get(\"APCA_API_BASE_URL\")\n",
+    "    APCA_API_KEY_ID = os.environ.get(\"APCA_API_KEY_ID\")\n",
+    "    APCA_API_SECRET_KEY = os.environ.get(\"APCA_API_SECRET_KEY\")\n",
+    "    import alpaca_trade_api as tradeapi\n",
+    "\n",
+    "    alpaca = tradeapi.REST(\n",
+    "        key_id=APCA_API_KEY_ID,\n",
+    "        secret_key=APCA_API_SECRET_KEY,\n",
+    "        base_url=APCA_API_BASE_URL,\n",
+    "    )\n",
+    "    account = alpaca.get_account()\n",
+    "    print(account.status)\n",
+    "\n",
+    "    dfs = []\n",
+    "    for ticker in tickers:\n",
+    "        print(\"Getting\", ticker)\n",
+    "        df = alpaca.get_bars(ticker, timeframe, start).df\n",
+    "        print(\"Recieved\", ticker)\n",
+    "        df.index.name = \"date\"\n",
+    "        df = pd.concat([df], axis=1, keys=[ticker])\n",
+    "        dfs.append(df)\n",
+    "    df = pd.concat(dfs, axis=1, sort=True)\n",
+    "    df.index.rename(\"date\", inplace=True)\n",
+    "\n",
+    "    if out_file is not None:\n",
+    "        write_df(df, out_file)\n",
+    "\n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Polygon\n",
+    "\n",
+    "\n",
+    "def use_polygon(tickers, out_file, multiplier=1, timespan=\"minute\", start=\"2000-01-01\"):\n",
+    "    POLYGON_API_KEY = os.environ.get(\"POLYGON_API_KEY\")\n",
+    "    from polygon import RESTClient\n",
+    "\n",
+    "    client = RESTClient(POLYGON_API_KEY)\n",
+    "    dfs = []\n",
+    "    end = datetime.datetime.utcnow()\n",
+    "    start_og = start\n",
+    "    for ticker in tickers:\n",
+    "        start = start_og\n",
+    "        df_agg = None\n",
+    "        response_len = None\n",
+    "        i = 0\n",
+    "        print(\"Getting\", ticker)\n",
+    "        while response_len != 1:\n",
+    "            i += 1\n",
+    "            aggs = client.get_aggs(\n",
+    "                ticker,\n",
+    "                multiplier,\n",
+    "                timespan,\n",
+    "                start,\n",
+    "                end,\n",
+    "                adjusted=True,\n",
+    "                sort=\"asc\",\n",
+    "                limit=50000,\n",
+    "            )\n",
+    "            df = pd.DataFrame(aggs)\n",
+    "            df.index = pd.DatetimeIndex(\n",
+    "                pd.to_datetime(df[\"timestamp\"], unit=\"ms\", utc=True)\n",
+    "            )\n",
+    "            df.index.name = \"date\"\n",
+    "            df = df.filter([\"open\", \"high\", \"low\", \"close\", \"volume\", \"vwap\"], axis=1)\n",
+    "            response_len = len(df.index)\n",
+    "            start = df.last_valid_index()\n",
+    "            print(i, response_len)\n",
+    "            if df_agg is not None:\n",
+    "                df_agg.drop(index=df_agg.index[-1], axis=0, inplace=True)\n",
+    "                df_agg = pd.merge(df_agg.reset_index(), df.reset_index(), how=\"outer\")\n",
+    "                df_agg = df_agg.set_index(\"date\")\n",
+    "            else:\n",
+    "                df_agg = df\n",
+    "            sleep(1)  # Attempt to be nice\n",
+    "        df_agg = pd.concat([df_agg], axis=1, keys=[ticker])\n",
+    "        dfs.append(df_agg)\n",
+    "        print(\"Recieved\", ticker)\n",
+    "\n",
+    "    df = pd.concat(dfs, axis=1, sort=True)\n",
+    "    df.index.rename(\"date\", inplace=True)\n",
+    "\n",
+    "    if out_file is not None:\n",
+    "        write_df(df, out_file)\n",
+    "\n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_7521/3255818553.py:11: FutureWarning: YF.download() has changed argument auto_adjust default to True\n",
+      "  data = yf.download(tickers, start=start, end=end, group_by=\"ticker\")\n",
+      "[*********************100%***********************]  2 of 2 completed"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Adding time\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Yahoo Finance\n",
+    "df = use_yfinance(\n",
+    "    [\"AAPL\", \"TSLA\"], os.path.join(DATA_RAW, \"aapl_day_full.csv\"), start=\"1970-01-01\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "ticker: XOM, y1 m1; response length: 2481441\n",
+      "{\n",
+      "    \"Meta Data\": {\n",
+      "        \"1. Information\": \"Intraday (1min) open, high, low, close prices and volume\",\n",
+      "        \"2. Symbol\": \"XOM\",\n",
+      "        \"3. Last Refreshed\": \"2025-10-22 19:59:00\",\n",
+      "        \"4. Interval\": \"1min\",\n",
+      "        \"5. Output Size\": \"Full size\",\n",
+      "        \"6. Time Zone\": \"US/Eastern\"\n",
+      "    },\n",
+      "    \"Time Series (1min)\": {\n",
+      "        \"2025-10-22 19:59:00\": {\n",
+      "            \"1. open\": \"115.2100\",\n",
+      "            \"2. high\": \"115.3900\",\n",
+      "            \"3. low\": \"115.2100\",\n",
+      "            \"4. close\": \"115.3900\",\n",
+      "            \"5. volume\": \"105\"\n",
+      "        },\n",
+      "        \"2025-10-22 19:58:00\": {\n",
+      "            \"1. open\": \"115.4800\",\n",
+      "            \"2. high\": \"115.4800\",\n",
+      "            \"3. low\": \"115.2000\",\n",
+      "            \"4. close\": \"115.2000\",\n",
+      "            \"5. volume\": \"6\"\n",
+      "        },\n",
+      "        \"2025-10-22 19:57:00\": {\n",
+      "            \"1. open\": \"115.4800\",\n",
+      "            \"2. high\": \"115.4800\",\n",
+      "            \"3. low\": \"115.3800\",\n",
+      "            \"4. close\": \"115.3800\",\n",
+      "            \"5. volume\": \"170\"\n",
+      "        },\n",
+      "        \"2025-10-22 19:56:00\n",
+      "80589 []\n",
+      "        \"1. Information\": \"Intraday (1min) open, high, low, close prices and volume\",\n"
+     ]
+    },
+    {
+     "ename": "ValueError",
+     "evalue": "1 columns passed, passed data had 5 columns",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
+      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/pandas/core/internals/construction.py:939\u001b[0m, in \u001b[0;36m_finalize_columns_and_data\u001b[0;34m(content, columns, dtype)\u001b[0m\n\u001b[1;32m    938\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 939\u001b[0m     columns \u001b[38;5;241m=\u001b[39m \u001b[43m_validate_or_indexify_columns\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcontents\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    940\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mAssertionError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m    941\u001b[0m     \u001b[38;5;66;03m# GH#26429 do not raise user-facing AssertionError\u001b[39;00m\n",
+      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/pandas/core/internals/construction.py:986\u001b[0m, in \u001b[0;36m_validate_or_indexify_columns\u001b[0;34m(content, columns)\u001b[0m\n\u001b[1;32m    984\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_mi_list \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(columns) \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(content):  \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m    985\u001b[0m     \u001b[38;5;66;03m# caller's responsibility to check for this...\u001b[39;00m\n\u001b[0;32m--> 986\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mAssertionError\u001b[39;00m(\n\u001b[1;32m    987\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(columns)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m columns passed, passed data had \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    988\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(content)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m columns\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    989\u001b[0m     )\n\u001b[1;32m    990\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_mi_list:\n\u001b[1;32m    991\u001b[0m     \u001b[38;5;66;03m# check if nested list column, length of each sub-list should be equal\u001b[39;00m\n",
+      "\u001b[0;31mAssertionError\u001b[0m: 1 columns passed, passed data had 5 columns",
+      "\nThe above exception was the direct cause of the following exception:\n",
+      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[25], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Alpha Vantage\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[43muse_alpha_vantage\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtickers\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpath\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m(\u001b[49m\u001b[43mDATA_RAW\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mrealdata.csv\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
+      "Cell \u001b[0;32mIn[22], line 64\u001b[0m, in \u001b[0;36muse_alpha_vantage\u001b[0;34m(tickers, out_file, time)\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m year \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m3\u001b[39m):\n\u001b[1;32m     63\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m month \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m13\u001b[39m):\n\u001b[0;32m---> 64\u001b[0m         df_temp \u001b[38;5;241m=\u001b[39m \u001b[43malpha_vantage_get_ticker_data\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     65\u001b[0m \u001b[43m            \u001b[49m\u001b[43mticker\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtime\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43myear\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43myear\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmonth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmonth\u001b[49m\n\u001b[1;32m     66\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     67\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m df_temp \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m     68\u001b[0m             t_dfs\u001b[38;5;241m.\u001b[39mappend(df_temp)\n",
+      "Cell \u001b[0;32mIn[22], line 53\u001b[0m, in \u001b[0;36malpha_vantage_get_ticker_data\u001b[0;34m(ticker, time, year, month)\u001b[0m\n\u001b[1;32m     51\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 53\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcsv_str_to_df\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdecoded_content\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mticker\u001b[49m\u001b[43m)\u001b[49m\n",
+      "Cell \u001b[0;32mIn[22], line 11\u001b[0m, in \u001b[0;36mcsv_str_to_df\u001b[0;34m(decoded_content, ticker)\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28mprint\u001b[39m(lines[\u001b[38;5;241m2\u001b[39m])\n\u001b[1;32m     10\u001b[0m \u001b[38;5;66;03m#while(1):pass\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mDataFrame\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     12\u001b[0m \u001b[43m    \u001b[49m\u001b[43m[\u001b[49m\u001b[43mrow\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msplit\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m,\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mrow\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mlines\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     13\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mdate\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mlines\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msplit\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m,\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     14\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     17\u001b[0m data \u001b[38;5;241m=\u001b[39m data\u001b[38;5;241m.\u001b[39mreset_index(drop\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\u001b[38;5;241m.\u001b[39mset_index(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdate\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     18\u001b[0m data\u001b[38;5;241m.\u001b[39mindex \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mto_datetime(data\u001b[38;5;241m.\u001b[39mindex)\n",
+      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/pandas/core/frame.py:851\u001b[0m, in \u001b[0;36mDataFrame.__init__\u001b[0;34m(self, data, index, columns, dtype, copy)\u001b[0m\n\u001b[1;32m    849\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m columns \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    850\u001b[0m         columns \u001b[38;5;241m=\u001b[39m ensure_index(columns)\n\u001b[0;32m--> 851\u001b[0m     arrays, columns, index \u001b[38;5;241m=\u001b[39m \u001b[43mnested_data_to_arrays\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    852\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;66;43;03m# error: Argument 3 to \"nested_data_to_arrays\" has incompatible\u001b[39;49;00m\n\u001b[1;32m    853\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;66;43;03m# type \"Optional[Collection[Any]]\"; expected \"Optional[Index]\"\u001b[39;49;00m\n\u001b[1;32m    854\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    855\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    856\u001b[0m \u001b[43m        \u001b[49m\u001b[43mindex\u001b[49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# type: ignore[arg-type]\u001b[39;49;00m\n\u001b[1;32m    857\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    858\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    859\u001b[0m     mgr \u001b[38;5;241m=\u001b[39m arrays_to_mgr(\n\u001b[1;32m    860\u001b[0m         arrays,\n\u001b[1;32m    861\u001b[0m         columns,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    864\u001b[0m         typ\u001b[38;5;241m=\u001b[39mmanager,\n\u001b[1;32m    865\u001b[0m     )\n\u001b[1;32m    866\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
+      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/pandas/core/internals/construction.py:520\u001b[0m, in \u001b[0;36mnested_data_to_arrays\u001b[0;34m(data, columns, index, dtype)\u001b[0m\n\u001b[1;32m    517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_named_tuple(data[\u001b[38;5;241m0\u001b[39m]) \u001b[38;5;129;01mand\u001b[39;00m columns \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    518\u001b[0m     columns \u001b[38;5;241m=\u001b[39m ensure_index(data[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39m_fields)\n\u001b[0;32m--> 520\u001b[0m arrays, columns \u001b[38;5;241m=\u001b[39m \u001b[43mto_arrays\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    521\u001b[0m columns \u001b[38;5;241m=\u001b[39m ensure_index(columns)\n\u001b[1;32m    523\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m index \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
+      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/pandas/core/internals/construction.py:845\u001b[0m, in \u001b[0;36mto_arrays\u001b[0;34m(data, columns, dtype)\u001b[0m\n\u001b[1;32m    842\u001b[0m     data \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mtuple\u001b[39m(x) \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m data]\n\u001b[1;32m    843\u001b[0m     arr \u001b[38;5;241m=\u001b[39m _list_to_arrays(data)\n\u001b[0;32m--> 845\u001b[0m content, columns \u001b[38;5;241m=\u001b[39m \u001b[43m_finalize_columns_and_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43marr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    846\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m content, columns\n",
+      "File \u001b[0;32m~/.local/lib/python3.10/site-packages/pandas/core/internals/construction.py:942\u001b[0m, in \u001b[0;36m_finalize_columns_and_data\u001b[0;34m(content, columns, dtype)\u001b[0m\n\u001b[1;32m    939\u001b[0m     columns \u001b[38;5;241m=\u001b[39m _validate_or_indexify_columns(contents, columns)\n\u001b[1;32m    940\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mAssertionError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m    941\u001b[0m     \u001b[38;5;66;03m# GH#26429 do not raise user-facing AssertionError\u001b[39;00m\n\u001b[0;32m--> 942\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(err) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n\u001b[1;32m    944\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(contents) \u001b[38;5;129;01mand\u001b[39;00m contents[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mdtype \u001b[38;5;241m==\u001b[39m np\u001b[38;5;241m.\u001b[39mobject_:\n\u001b[1;32m    945\u001b[0m     contents \u001b[38;5;241m=\u001b[39m convert_object_array(contents, dtype\u001b[38;5;241m=\u001b[39mdtype)\n",
+      "\u001b[0;31mValueError\u001b[0m: 1 columns passed, passed data had 5 columns"
+     ]
+    }
+   ],
+   "source": [
+    "# Alpha Vantage\n",
+    "df = use_alpha_vantage(tickers, os.path.join(DATA_RAW, \"realdata.csv\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Alpaca\n",
+    "df = use_alpaca(\n",
+    "    tickers + random, os.path.join(DATA_RAW, \"realdata_alp_1h.csv\"), timeframe=\"1Hour\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Polygon\n",
+    "df = use_polygon(\n",
+    "    materials_equities,\n",
+    "    os.path.join(DATA_RAW, \"materials_1h.csv\"),\n",
+    "    multiplier=1,\n",
+    "    timespan=\"hour\",\n",
+    "    start=\"2000-01-01\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Extras"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Read Data From All-Data CSV (Multi Index Columns)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_all = read_data(os.path.join(DATA_RAW, \"realdata.csv\"))\n",
+    "# df = read_data(\"tsla_aapl.csv\")\n",
+    "print(df_all.head())\n",
+    "print(df.head())\n",
+    "print(df_all.columns)\n",
+    "print(df.columns)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Concatenate two datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "run = False\n",
+    "if run and not df.columns.equals(df_all.columns):\n",
+    "    df_new = write_df(\n",
+    "        pd.concat([df_all, df], axis=1), os.path.join(DATA_RAW, \"realdata.csv\")\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Remove rows with a lot of NANs\n",
+    "This is important when using FX data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_f = df.copy()\n",
+    "df_f = df_f.dropna(axis=0, thresh=50) #80\n",
+    "write_df(df_f, os.path.join(DATA_RAW, \"realdata_pol_1h.csv\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.tail(80)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.14"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "51980e48e269f7c05efac26b22569386591d7f1d45336266d53ed7fc3ab7efc6"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

data_collect.py

@@ -0,0 +1,369 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+import pandas as pd
+from time import sleep
+import datetime
+import os
+from utils.ipynb_helpers import read_data, write_df, convert_tz, add_tz
+from dotenv import load_dotenv
+import traceback
+
+# Create a .env file and add your keys
+load_dotenv()
+
+# Location to save raw data from data providers
+DATA_RAW = "data/raw"
+
+
+equities = ["XOM", "CVX", "COP", "BP", "PBR", "WTI", "TTE", "EQNR", "EOG", "ENB", "SLB"]
+more_equities = []
+
+crude_oil = ["CL=F", "BZ=F"]  # wti, brent,
+random = ["TSLA", "AAPL"]
+
+materials_equities = ["BHP", "LIN", "RIO", "VALE", "APD", "FCX", "SHW", "SCCO", "CTVA", "ECL", "NUE", "NTR"]
+
+
+# https://en.wikipedia.org/wiki/List_of_countries_by_oil_production
+# https://www.weforum.org/agenda/2016/05/which-economies-are-most-reliant-on-oil/
+# OPEC: Iran, Iraq, Kuwait, Saudi Arabia, Venezuela
+# fx_opec = [_, "C:USDIQD", "C:USDKWD", "C:USDSAR", "C:USDVEF"]
+
+# OPEC+: Algeria, Angola, Congo, Equatorial Guinea, Gabon, Libya, Nigeria, United Arab Emirates
+# fx_opec_pp = ["C:USDDZD",_, "C:USDCDF", "C:USDGNF", _, "C:USDLYD", "C:USDNGN", "C:USDAED"]
+
+# Large: US, Russia, China, Canada, Norway
+# Other important: Qatar, Kazakhstan
+# fx_other= ["C:USDQAR", "C:USDKZT"]
+
+fx = ["C:USDSAR", "C:USDAED"]
+
+tickers = equities  # + crude_oil
+
+
+# ##### Get Data From Data Provider
+
+# In[2]:
+
+
+# Y Finance
+
+import yfinance as yf
+
+
+def use_yfinance(
+    tickers, out_file, timeframe="day", start="2000-01-01", end="2023-01-01"
+):
+    assert timeframe == "day", "Use day timeframe for day"
+
+    data = yf.download(tickers, start=start, end=end, group_by="ticker", threads=False)
+
+    if len(tickers) == 1:
+        data = pd.concat([data], axis=1, keys=[tickers[0]])
+
+    data.index.rename("date", inplace=True)
+    data.rename(columns=lambda x: str.lower(x), level=1, inplace=True)
+
+    if data.index.to_series().dt.tz is None:
+        print("Adding time")
+        data = add_tz(data, time_zone="UTC")
+
+    if out_file is not None:
+        write_df(data, out_file)
+
+    return data
+
+
+# In[22]:
+
+
+# Alpha Vantage
+
+
+def csv_str_to_df(decoded_content, ticker):
+    """CSV string to df"""
+    lines = decoded_content.splitlines()
+    print(lines[-20:])
+    lines = [ "".join([ lines[i+j][8:-3] if j//6==0 else lines[i+j][12:-1] for j in range(6) ]) for i in range(10, len(lines), 6)]
+    print(len(lines))
+    print(lines[-20:])
+    while(1):pass
+    data = pd.DataFrame(
+        [row.split(",") for row in lines[1:]],
+        columns=["date", "open", "high", "low", "close", "volume"],
+    )
+
+
+    data = data.reset_index(drop=True).set_index("date")
+    data.index = pd.to_datetime(data.index)
+
+    # Add timezome -- we assume it is sent in with unlabled eastern time
+    if data.index.to_series().dt.tz is None:
+        print("CONVERTING TIME")
+        data = add_tz(data, time_zone="US/Eastern")
+        data = convert_tz(data, time_zone="UTC")
+    data = pd.concat([data], axis=1, keys=[ticker])
+    return data
+
+
+def alpha_vantage_get_ticker_data(ticker, time="1min", year=1, month=1):
+    """Function to get (ticker, year, month) data using alpha vantage's time series intraday extended API"""
+    ALPHA_VANTAGE_API_KEY = "VGRS7MNEHU6K8FAZ"
+    import requests
+
+    CSV_URL = f"https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY&symbol={ticker}&interval={time}&month={2026-year}-{11-month:02d}&outputsize=full&apikey={ALPHA_VANTAGE_API_KEY}"
+
+    while True:
+        with requests.Session() as s:
+            download = s.get(CSV_URL)
+            # save to local file
+            decoded_content = download.content.decode("utf-8")
+            print(
+                f"ticker: {ticker}, y{year} m{month}; response length: {len(decoded_content)}"
+            )
+
+            if len(decoded_content) == 236:
+                # API too many requests
+                sleep(60)
+            elif len(decoded_content) <= 243:
+                # Token doesn't exist or something
+                print(f"Error getting {ticker}, y{year}, m{month}. We are skipping")
+                print(decoded_content)
+                return None
+            else:
+                return csv_str_to_df(decoded_content, ticker)
+
+
+def use_alpha_vantage(tickers, out_file, time="1min"):
+    """Function to get multiple full tickers data using alpha vantage's time series intraday extended API"""
+
+    dfs = []
+    for ticker in tickers:
+        t_dfs = []
+        for year in range(1, 3):
+            for month in range(1, 13):
+                df_temp = alpha_vantage_get_ticker_data(
+                    ticker, time=time, year=year, month=month
+                )
+                if df_temp is not None:
+                    t_dfs.append(df_temp)
+
+        if len(t_dfs):
+            dfs.append(pd.concat(t_dfs, axis=0))
+        else:
+            print(f"Skipped {ticker}.")
+    df = pd.concat(dfs, axis=1, sort=True)
+    while(1):pass
+    df.index.rename("date", inplace=True)
+
+    write_df(df, out_file)
+
+    return df
+
+
+# In[23]:
+
+
+# Alpaca
+
+
+def use_alpaca(tickers, out_file, timeframe="1Minute", start="2017-01-01"):
+    APCA_API_BASE_URL = os.environ.get("APCA_API_BASE_URL")
+    APCA_API_KEY_ID = os.environ.get("APCA_API_KEY_ID")
+    APCA_API_SECRET_KEY = os.environ.get("APCA_API_SECRET_KEY")
+    import alpaca_trade_api as tradeapi
+
+    alpaca = tradeapi.REST(
+        key_id=APCA_API_KEY_ID,
+        secret_key=APCA_API_SECRET_KEY,
+        base_url=APCA_API_BASE_URL,
+    )
+    account = alpaca.get_account()
+    print(account.status)
+
+    dfs = []
+    for ticker in tickers:
+        print("Getting", ticker)
+        df = alpaca.get_bars(ticker, timeframe, start).df
+        print("Recieved", ticker)
+        df.index.name = "date"
+        df = pd.concat([df], axis=1, keys=[ticker])
+        dfs.append(df)
+    df = pd.concat(dfs, axis=1, sort=True)
+    df.index.rename("date", inplace=True)
+
+    if out_file is not None:
+        write_df(df, out_file)
+
+    return df
+
+
+# In[24]:
+
+
+# Polygon
+
+def try_until_suc(request_func, *args, **kwargs):
+    while True:
+        try:
+            res = request_func(*args, **kwargs)
+        except Exception as e:
+            print("Error Message:", e)
+            print(f"Traceback Details: {traceback.format_exc()}")  # Get full traceback as a string
+            print("retry sending request...")
+            sleep(5)
+        else:
+            break
+    return res
+
+
+def use_polygon(tickers, out_file, multiplier=1, timespan="minute", start="2000-01-01"):
+    POLYGON_API_KEY = "i0tmf9psII0FV_W7cAHs5PSKSVlqns72"
+    from polygon import RESTClient
+
+    client = RESTClient(POLYGON_API_KEY)
+    #  aggs = client.get_aggs("AAPL", 1, "day", "2000-01-01", "2001-01-01")
+    #  print(aggs[0].timestamp)
+    #  while(1):pass
+    dfs = []
+    end = datetime.datetime.utcnow()
+    start_og = start
+    for ticker in tickers:
+        start = start_og
+        df_agg = None
+        response_len = None
+        i = 0
+        print("Getting", ticker)
+        while response_len != 1:
+            i += 1
+            aggs = try_until_suc(
+                client.get_aggs,
+                ticker,
+                multiplier,
+                timespan,
+                start,
+                end,
+                adjusted=True,
+                sort="asc",
+                limit=50000,
+            )
+            df = pd.DataFrame(aggs)
+            df.index = pd.DatetimeIndex(
+                pd.to_datetime(df["timestamp"], unit="ms", utc=True)
+            )
+            df.index.name = "date"
+            df = df.filter(["open", "high", "low", "close", "volume", "vwap"], axis=1)
+            response_len = len(df.index)
+            start = df.last_valid_index()
+            print(i, response_len)
+            if df_agg is not None:
+                df_agg.drop(index=df_agg.index[-1], axis=0, inplace=True)
+                df_agg = pd.merge(df_agg.reset_index(), df.reset_index(), how="outer")
+                df_agg = df_agg.set_index("date")
+            else:
+                df_agg = df
+            sleep(12)  # Attempt to be nice
+        df_agg = pd.concat([df_agg], axis=1, keys=[ticker])
+        dfs.append(df_agg)
+        print("Recieved", ticker)
+
+    df = pd.concat(dfs, axis=1, sort=True)
+    df.index.rename("date", inplace=True)
+
+    if out_file is not None:
+        write_df(df, out_file)
+
+    return df
+
+
+# In[6]:
+
+
+# Yahoo Finance
+#  df = use_yfinance(
+   #  random, os.path.join(DATA_RAW, "aapl_day_full.csv"), start="1970-01-01",
+#  )
+
+
+# In[25]:
+
+
+# Alpha Vantage
+#  df = use_alpha_vantage(tickers, os.path.join(DATA_RAW, "realdata.csv"), time="1h")
+
+
+# In[ ]:
+
+
+#  # Alpaca
+#  df = use_alpaca(
+    #  tickers + random, os.path.join(DATA_RAW, "realdata_alp_1h.csv"), timeframe="1Hour"
+#  )
+
+
+#  # In[ ]:
+
+
+# Polygon
+df = use_polygon(
+    tickers,
+    os.path.join(DATA_RAW, "realdata.csv"),
+    multiplier=1,
+    timespan="hour",
+    start="2000-01-01",
+)
+
+
+# In[ ]:
+
+
+df.head()
+
+
+# ## Extras
+
+# ##### Read Data From All-Data CSV (Multi Index Columns)
+
+# In[ ]:
+
+
+df_all = read_data(os.path.join(DATA_RAW, "realdata.csv"))
+# df = read_data("tsla_aapl.csv")
+print(df_all.head())
+print(df.head())
+print(df_all.columns)
+print(df.columns)
+
+
+# ##### Concatenate two datasets
+
+# In[ ]:
+
+
+run = False
+if run and not df.columns.equals(df_all.columns):
+    df_new = write_df(
+        pd.concat([df_all, df], axis=1), os.path.join(DATA_RAW, "realdata.csv")
+    )
+
+
+# ### Remove rows with a lot of NANs
+# This is important when using FX data
+
+# In[ ]:
+
+
+df_f = df.copy()
+df_f = df_f.dropna(axis=0, thresh=50) #80
+write_df(df_f, os.path.join(DATA_RAW, "realdata_pol_1h.csv"))
+
+
+# In[ ]:
+
+
+df.tail(80)
+

data_loader.py

@@ -0,0 +1,652 @@
+import os
+import numpy as np
+import pandas as pd
+
+import torch
+from torch.utils.data import Dataset
+
+# from sklearn.preprocessing import StandardScaler
+
+from utils.tools import StandardScaler, dotdict
+from utils.timefeatures import time_features
+
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+class Dataset_ETT_hour(Dataset):
+    def __init__(
+        self,
+        root_path,
+        flag="train",
+        size=None,
+        features="S",
+        data_path="ETTh1.csv",
+        target="OT",
+        scale=True,
+        inverse=False,
+        timeenc=0,
+        freq="h",
+        cols=None,
+    ):
+        # size [seq_len, label_len, pred_len]
+        # info
+        if size == None:
+            self.seq_len = 24 * 4 * 4
+            self.label_len = 24 * 4
+            self.pred_len = 24 * 4
+        else:
+            self.seq_len = size[0]
+            self.label_len = size[1]
+            self.pred_len = size[2]
+        # init
+        assert flag in ["train", "test", "val"]
+        type_map = {"train": 0, "val": 1, "test": 2}
+        self.set_type = type_map[flag]
+
+        self.features = features
+        self.target = target
+        self.scale = scale
+        self.inverse = inverse
+        self.timeenc = timeenc
+        self.freq = freq
+
+        self.root_path = root_path
+        self.data_path = data_path
+        self.__read_data__()
+
+    def __read_data__(self):
+        self.scaler = StandardScaler()
+        df_raw = pd.read_csv(os.path.join(self.root_path, self.data_path))
+
+        border1s = [
+            0,
+            12 * 30 * 24 - self.seq_len,
+            12 * 30 * 24 + 4 * 30 * 24 - self.seq_len,
+        ]
+        border2s = [
+            12 * 30 * 24,
+            12 * 30 * 24 + 4 * 30 * 24,
+            12 * 30 * 24 + 8 * 30 * 24,
+        ]
+        border1 = border1s[self.set_type]
+        border2 = border2s[self.set_type]
+
+        if self.features == "M" or self.features == "MS":
+            cols_data = df_raw.columns[1:]
+            df_data = df_raw[cols_data]
+        elif self.features == "S":
+            df_data = df_raw[[self.target]]
+
+        if self.scale:
+            train_data = df_data[border1s[0] : border2s[0]]
+            self.scaler.fit(train_data.values, scale_mean=not self.config.no_scale_mean)
+            data = self.scaler.transform(df_data.values)
+        else:
+            data = df_data.values
+
+        df_stamp = df_raw[["date"]][border1:border2]
+        df_stamp["date"] = pd.to_datetime(df_stamp.date)
+        self.raw_dates = df_stamp.date.to_numpy(dtype=np.datetime64)
+        data_stamp = time_features(df_stamp, timeenc=self.timeenc, freq=self.freq)
+
+        self.data_x = data[border1:border2]
+        if self.inverse:
+            self.data_y = df_data.values[border1:border2]
+        else:
+            self.data_y = data[border1:border2]
+        self.data_stamp = data_stamp
+
+    def __getitem__(self, index):
+        s_begin = index
+        s_end = s_begin + self.seq_len
+        r_begin = s_end - self.label_len
+        r_end = r_begin + self.label_len + self.pred_len
+
+        seq_x = self.data_x[s_begin:s_end]
+        if self.inverse:
+            seq_y = np.concatenate(
+                [
+                    self.data_x[r_begin : r_begin + self.label_len],
+                    self.data_y[r_begin + self.label_len : r_end],
+                ],
+                0,
+            )
+        else:
+            seq_y = self.data_y[r_begin:r_end]
+        seq_x_mark = self.data_stamp[s_begin:s_end]
+        seq_y_mark = self.data_stamp[r_begin:r_end]
+
+        return seq_x, seq_y, seq_x_mark, seq_y_mark, index
+
+    def index_to_dates(self, index):
+        # index is of length batch_size
+        s_begin = index
+        s_end = s_begin + self.config.seq_len
+        r_begin = s_end - self.config.label_len
+        r_end = r_begin + self.config.label_len + self.config.pred_len
+
+        seq_x_raw_dates = self.raw_dates[
+            np.add.outer(s_begin, np.arange(self.config.seq_len))
+        ]
+        seq_y_raw_dates = self.raw_dates[
+            np.add.outer(
+                r_begin, np.arange(self.config.label_len + self.config.pred_len)
+            )
+        ]
+
+        return seq_x_raw_dates, seq_y_raw_dates
+
+    def __len__(self):
+        return len(self.data_x) - self.seq_len - self.pred_len + 1
+
+    def inverse_transform(self, data):
+        return self.scaler.inverse_transform(data)
+
+
+class Dataset_ETT_minute(Dataset):
+    def __init__(
+        self,
+        root_path,
+        flag="train",
+        size=None,
+        features="S",
+        data_path="ETTm1.csv",
+        target="OT",
+        scale=True,
+        inverse=False,
+        timeenc=0,
+        freq="t",
+        cols=None,
+    ):
+        # size [seq_len, label_len, pred_len]
+        # info
+        if size == None:
+            self.seq_len = 24 * 4 * 4
+            self.label_len = 24 * 4
+            self.pred_len = 24 * 4
+        else:
+            self.seq_len = size[0]
+            self.label_len = size[1]
+            self.pred_len = size[2]
+        # init
+        assert flag in ["train", "test", "val"]
+        type_map = {"train": 0, "val": 1, "test": 2}
+        self.set_type = type_map[flag]
+
+        self.features = features
+        self.target = target
+        self.scale = scale
+        self.inverse = inverse
+        self.timeenc = timeenc
+        self.freq = freq
+
+        self.root_path = root_path
+        self.data_path = data_path
+        self.__read_data__()
+
+    def __read_data__(self):
+        self.scaler = StandardScaler()
+        df_raw = pd.read_csv(os.path.join(self.root_path, self.data_path))
+
+        border1s = [
+            0,
+            12 * 30 * 24 * 4 - self.seq_len,
+            12 * 30 * 24 * 4 + 4 * 30 * 24 * 4 - self.seq_len,
+        ]
+        border2s = [
+            12 * 30 * 24 * 4,
+            12 * 30 * 24 * 4 + 4 * 30 * 24 * 4,
+            12 * 30 * 24 * 4 + 8 * 30 * 24 * 4,
+        ]
+        border1 = border1s[self.set_type]
+        border2 = border2s[self.set_type]
+
+        if self.features == "M" or self.features == "MS":
+            cols_data = df_raw.columns[1:]
+            df_data = df_raw[cols_data]
+        elif self.features == "S":
+            df_data = df_raw[[self.target]]
+
+        if self.scale:
+            train_data = df_data[border1s[0] : border2s[0]]
+            self.scaler.fit(train_data.values, scale_mean=not self.config.no_scale_mean)
+            data = self.scaler.transform(df_data.values)
+        else:
+            data = df_data.values
+
+        df_stamp = df_raw[["date"]][border1:border2]
+        df_stamp["date"] = pd.to_datetime(df_stamp.date)
+        self.raw_dates = df_stamp.date.to_numpy(dtype=np.datetime64)
+        data_stamp = time_features(df_stamp, timeenc=self.timeenc, freq=self.freq)
+
+        self.data_x = data[border1:border2]
+        if self.inverse:
+            self.data_y = df_data.values[border1:border2]
+        else:
+            self.data_y = data[border1:border2]
+        self.data_stamp = data_stamp
+
+    def __getitem__(self, index):
+        s_begin = index
+        s_end = s_begin + self.seq_len
+        r_begin = s_end - self.label_len
+        r_end = r_begin + self.label_len + self.pred_len
+
+        seq_x = self.data_x[s_begin:s_end]
+        if self.inverse:
+            seq_y = np.concatenate(
+                [
+                    self.data_x[r_begin : r_begin + self.label_len],
+                    self.data_y[r_begin + self.label_len : r_end],
+                ],
+                0,
+            )
+        else:
+            seq_y = self.data_y[r_begin:r_end]
+        seq_x_mark = self.data_stamp[s_begin:s_end]
+        seq_y_mark = self.data_stamp[r_begin:r_end]
+
+        return seq_x, seq_y, seq_x_mark, seq_y_mark, index
+
+    def index_to_dates(self, index):
+        # index is of length batch_size
+        s_begin = index
+        s_end = s_begin + self.config.seq_len
+        r_begin = s_end - self.config.label_len
+        r_end = r_begin + self.config.label_len + self.config.pred_len
+
+        seq_x_raw_dates = self.raw_dates[
+            np.add.outer(s_begin, np.arange(self.config.seq_len))
+        ]
+        seq_y_raw_dates = self.raw_dates[
+            np.add.outer(
+                r_begin, np.arange(self.config.label_len + self.config.pred_len)
+            )
+        ]
+
+        return seq_x_raw_dates, seq_y_raw_dates
+
+    def __len__(self):
+        return len(self.data_x) - self.seq_len - self.pred_len + 1
+
+    def inverse_transform(self, data):
+        return self.scaler.inverse_transform(data)
+
+
+class Dataset_Custom(Dataset):
+    def __init__(self, config, flag="train"):
+        # Default values
+        defaults = {
+            "size": None,
+            "features": "S",
+            "target": "OT",
+            "scale": True,
+            "inverse_pred": False,
+            "inverse_output": False,
+            "cols": None,
+            "date_start": None,
+            "date_end": None,
+            "date_test": None,
+            "date_val": None,
+            "t_embed": None,
+        }
+        config = dotdict({**defaults, **config})
+
+        assert config.seq_len is not None
+        assert config.label_len is not None
+        assert config.pred_len is not None
+        assert flag in ["train", "test", "val"]
+        assert config.freq is not None
+        assert config.root_path is not None
+        assert config.data_path is not None
+        assert (
+            (config.date_start is None)
+            or (config.date_end is None)
+            or (config.date_start < config.date_end)
+        ), "date_start isn't before date_end"
+
+        assert (
+            (config.date_test is None)
+            or (config.date_end is None)
+            or (config.date_test < config.date_end)
+        ), "date_test isn't before date_end"
+        assert (
+            (config.date_test is None)
+            or (config.date_start is None)
+            or (config.date_test > config.date_start)
+        ), "date_test isn't after date_start"
+
+        assert (config.date_val is None) or (
+            config.date_test is not None
+        ), "date_val is used without date_test"
+        assert (
+            (config.date_val is None)
+            or (config.date_test is None)
+            or (config.date_val < config.date_test)
+        ), "date_val isn't before date_test"
+
+        assert (
+            (config.date_val is None)
+            or (config.date_end is None)
+            or (config.date_val < config.date_end)
+        ), "date_val isn't before date_end"
+        assert (
+            (config.date_val is None)
+            or (config.date_start is None)
+            or (config.date_val > config.date_start)
+        ), "date_val isn't after date_start"
+
+        assert (config.label_len == 0) or (
+            config.inverse_output == config.inverse_pred
+        ), "If label length is non-zero then inverse_pred and inverse_output should be the same"
+
+        self.config = config
+        self.flag = flag
+
+        # self.timeenc = 0 if config.t_embed != "timeF" else 1
+        if config.t_embed == "timeF":
+            self.timeenc = 1
+        elif config.t_embed == "time2vec_add" or config.t_embed == "time2vec_app":
+            self.timeenc = 2
+        else:
+            self.timeenc = 0
+
+        type_map = {"train": 0, "val": 1, "test": 2}
+        self.set_type = type_map[flag]
+
+        self.__read_data__()
+
+    def __read_data__(self):
+        self.scaler = StandardScaler()
+        df_raw = pd.read_csv(os.path.join(self.config.root_path, self.config.data_path))
+        df_raw = df_raw.astype(
+            {c: np.float32 for c in df_raw.select_dtypes(include="float64").columns}
+        )
+        df_raw["date"] = pd.to_datetime(df_raw["date"])
+
+        if np.isinf(df_raw[df_raw.columns[1:]].to_numpy()).any():
+            raise Exception("There are inf's in the dataset")
+        if np.isnan(df_raw[df_raw.columns[1:]].to_numpy()).any():
+            raise Exception("There are nan's in the dataset")
+        """
+        df_raw.columns: ['date', ...(other features), target feature]
+        """
+        # Filter to datapoints in [date_start, date_end]
+        if self.config.date_start is not None:
+            df_raw = df_raw.loc[(df_raw["date"] >= self.config.date_start)]
+        if self.config.date_end is not None:
+            df_raw = df_raw.loc[(df_raw["date"] <= self.config.date_end)]
+
+        if self.config.cols:
+            cols = self.config.cols.copy()
+            assert self.config.target in cols, "Target not in cols"
+            cols.remove(self.config.target)
+        else:
+            cols = list(df_raw.columns)
+            assert self.config.target in cols, "Target not in data"
+            cols.remove(self.config.target)
+            assert "date" in cols, "`date` not in data"
+            cols.remove("date")
+        df_raw = df_raw[["date"] + cols + [self.config.target]]
+
+        # Define lengths of train, val, and test
+        if self.config.date_test is not None and self.config.date_val is not None:
+            # num_test and num_val are specified
+            num_test = len(df_raw.loc[df_raw["date"] >= self.config.date_test])
+            num_vali = len(
+                df_raw.loc[
+                    (df_raw["date"] >= self.config.date_val)
+                    & (df_raw["date"] < self.config.date_test)
+                ]
+            )
+            num_train = len(df_raw) - num_vali - num_test
+        elif self.config.date_test is not None:
+            # num_val is half of num_test which is specified
+            num_test = len(df_raw.loc[(df_raw["date"] >= self.config.date_test)])
+            num_vali = num_test // 2
+            num_train = len(df_raw) - num_vali - num_test
+        else:
+            # Default split
+            print("Warning: using default dataset split")
+            num_train = int(len(df_raw) * 0.9)
+            num_test = int(len(df_raw) * 0.05)
+            num_vali = len(df_raw) - num_train - num_test
+
+        if num_test == 0:
+            raise Exception("Dataset loading issue: num_test==0, check date settings")
+        elif num_vali == 0:
+            raise Exception("Dataset loading issue: num_vali==0, check date settings")
+        elif num_train == 0:
+            raise Exception("Dataset loading issue: num_train==0, check date settings")
+
+        border1s = [
+            0,
+            num_train - self.config.seq_len,
+            len(df_raw) - num_test - self.config.seq_len,
+        ]
+        border2s = [num_train, num_train + num_vali, len(df_raw)]
+        border1 = border1s[self.set_type]
+        border2 = border2s[self.set_type]
+
+        if self.config.features == "M" or self.config.features == "MS":
+            cols_data = df_raw.columns[1:]
+            df_data = df_raw[cols_data]
+        elif self.config.features == "S":
+            df_data = df_raw[[self.config.target]]
+
+        if self.config.scale:
+            train_data = df_data[border1s[0] : border2s[0]]
+            self.scaler.fit(train_data.values, scale_mean=not self.config.no_scale_mean)
+            data = torch.from_numpy(self.scaler.transform(df_data.values))
+        else:
+            data = torch.from_numpy(df_data.values)
+
+        df_stamp = df_raw[["date"]][border1:border2]
+        df_stamp["date"] = pd.to_datetime(df_stamp.date)
+        self.raw_dates = df_stamp.date.to_numpy(dtype=np.datetime64)
+        self.data_stamp = np.float32(
+            time_features(df_stamp, timeenc=self.timeenc, freq=self.config.freq)
+        )
+
+        self.data_x = data[border1:border2]
+        if self.config.inverse_pred:
+            self.data_y = torch.from_numpy(df_data.values[border1:border2])
+        else:
+            self.data_y = data[border1:border2]
+
+    def __getitem__(self, index):
+        s_begin = index
+        s_end = s_begin + self.config.seq_len
+        r_begin = s_end - self.config.label_len
+        r_end = r_begin + self.config.label_len + self.config.pred_len
+
+        seq_x = self.data_x[s_begin:s_end]
+        if self.config.inverse_pred:
+            # this is where inverse_pred != inverse output gets wonky if label_len != 0
+            # its because the label doesn't get inversed
+            seq_y = np.concatenate(
+                [
+                    self.data_x[
+                        r_begin : r_begin + self.config.label_len
+                    ],  # Use non-scaled data_x
+                    self.data_y[r_begin + self.config.label_len : r_end],
+                ],
+                axis=0,
+            )
+        else:
+            seq_y = self.data_y[r_begin:r_end]
+        seq_x_mark = self.data_stamp[s_begin:s_end]
+        seq_y_mark = self.data_stamp[r_begin:r_end]
+
+        return seq_x, seq_y, seq_x_mark, seq_y_mark, index
+
+    def index_to_dates(self, index):
+        # index is of length batch_size
+        s_begin = index
+        s_end = s_begin + self.config.seq_len
+        r_begin = s_end - self.config.label_len
+        r_end = r_begin + self.config.label_len + self.config.pred_len
+
+        seq_x_raw_dates = self.raw_dates[
+            np.add.outer(s_begin, np.arange(self.config.seq_len))
+        ]
+        seq_y_raw_dates = self.raw_dates[
+            np.add.outer(
+                r_begin, np.arange(self.config.label_len + self.config.pred_len)
+            )
+        ]
+        # seq_x_raw_dates = self.raw_dates[np.r_[s_begin,s_end-1].reshape(-1, index.shape[0]).T]# self.raw_dates.iloc[np.r_[s_begin,s_end]]
+        # seq_y_raw_dates = self.raw_dates[np.r_[r_begin,r_end-1].reshape(-1, index.shape[0]).T]# self.raw_dates.iloc[np.r_[r_begin,r_end]]
+
+        return seq_x_raw_dates, seq_y_raw_dates
+
+    def __len__(self):
+        return len(self.data_x) - self.config.seq_len - self.config.pred_len + 1
+
+    def inverse_transform(self, data):
+        return self.scaler.inverse_transform(data)
+
+
+class Dataset_Pred(Dataset):
+    def __init__(self, config, flag="pred"):
+        # Default values
+        defaults = {
+            "size": None,
+            "features": "S",
+            "target": "OT",
+            "scale": True,
+            "inverse": False,
+            "cols": None,
+            "date_start": None,
+            "date_end": None,
+            "t_embed": None,
+        }
+        config = dotdict({**defaults, **config})
+
+        assert config.seq_len is not None
+        assert config.label_len is not None
+        assert config.pred_len is not None
+        assert flag in ["pred"]
+        assert config.freq is not None
+        assert config.root_path is not None
+        assert config.data_path is not None
+        assert (
+            (config.date_start is None)
+            or (config.date_end is None)
+            or (config.date_start < config.date_end)
+        ), "date_start isn't before date_end"
+
+        self.config = config
+        self.flag = flag
+        # self.timeenc = 0 if config.t_embed != "timeF" else 1
+        if config.t_embed == "timeF":
+            self.timeenc = 1
+        elif config.t_embed == "time2vec_add" or config.t_embed == "time2vec_app":
+            self.timeenc = 2
+        else:
+            self.timeenc = 0
+
+        self.__read_data__()
+
+    def __read_data__(self):
+        self.scaler = StandardScaler()
+        df_raw = pd.read_csv(os.path.join(self.config.root_path, self.config.data_path))
+        """
+        df_raw.columns: ['date', ...(other features), target feature]
+        """
+
+        # Filter to datapoints in [date_start, date_end]
+        if self.config.date_start is not None:
+            df_raw = df_raw.loc[(df_raw["date"] >= self.config.date_start)]
+        if self.config.date_end is not None:
+            df_raw = df_raw.loc[(df_raw["date"] <= self.config.date_end)]
+
+        if self.config.cols:
+            cols = self.config.cols.copy()
+            cols.remove(self.config.target)
+        else:
+            cols = list(df_raw.columns)
+            cols.remove(self.config.target)
+            cols.remove("date")
+        df_raw = df_raw[["date"] + cols + [self.config.target]]
+
+        border1 = len(df_raw) - self.config.seq_len
+        border2 = len(df_raw)
+
+        if self.config.features == "M" or self.config.features == "MS":
+            cols_data = df_raw.columns[1:]
+            df_data = df_raw[cols_data]
+        elif self.config.features == "S":
+            df_data = df_raw[[self.config.target]]
+
+        if self.config.scale:
+            self.scaler.fit(df_data.values, scale_mean=not self.config.no_scale_mean)
+            data = self.scaler.transform(df_data.values)
+        else:
+            data = df_data.values
+
+        tmp_stamp = df_raw[["date"]][border1:border2]
+        tmp_stamp["date"] = pd.to_datetime(tmp_stamp.date)
+        pred_dates = pd.date_range(
+            tmp_stamp.date.values[-1],
+            periods=self.config.pred_len + 1,
+            freq=self.config.freq,
+        )
+
+        df_stamp = pd.DataFrame(columns=["date"])
+        df_stamp.date = pd.to_datetime(
+            list(tmp_stamp.date.values) + list(pred_dates[1:]), utc=True
+        )
+        self.raw_dates = df_stamp.date.to_numpy(dtype=np.datetime64)
+        # TODO: What is the deal with .freq[-1:]
+        self.data_stamp = np.float32(
+            time_features(df_stamp, timeenc=self.timeenc, freq=self.config.freq[-1:])
+        )
+
+        self.data_x = data[border1:border2]
+        if self.config.inverse:
+            self.data_y = df_data.values[border1:border2]
+        else:
+            self.data_y = data[border1:border2]
+
+    def __getitem__(self, index):
+        s_begin = index
+        s_end = s_begin + self.config.seq_len
+        r_begin = s_end - self.config.label_len
+        r_end = r_begin + self.config.label_len + self.config.pred_len
+
+        seq_x = self.data_x[s_begin:s_end]
+        if self.config.inverse:
+            seq_y = self.data_x[r_begin : r_begin + self.config.label_len]
+        else:
+            seq_y = self.data_y[r_begin : r_begin + self.config.label_len]
+        seq_x_mark = self.data_stamp[s_begin:s_end]
+        seq_y_mark = self.data_stamp[r_begin:r_end]
+
+        return seq_x, seq_y, seq_x_mark, seq_y_mark, index
+
+    def index_to_dates(self, index):
+        # index is of length batch_size
+        s_begin = index
+        s_end = s_begin + self.config.seq_len
+        r_begin = s_end - self.config.label_len
+        r_end = r_begin + self.config.label_len + self.config.pred_len
+
+        seq_x_raw_dates = self.raw_dates[
+            np.add.outer(s_begin, np.arange(self.config.seq_len))
+        ]
+        seq_y_raw_dates = self.raw_dates[
+            np.add.outer(
+                r_begin, np.arange(self.config.label_len + self.config.pred_len)
+            )
+        ]
+
+        return seq_x_raw_dates, seq_y_raw_dates
+
+    def __len__(self):
+        return len(self.data_x) - self.config.seq_len + 1
+
+    def inverse_transform(self, data):
+        return self.scaler.inverse_transform(data)

data_prepare.ipynb

@@ -0,0 +1,311 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import os\n",
+    "import datetime\n",
+    "import pytz\n",
+    "import numpy as np\n",
+    "from utils.ipynb_helpers import read_data, write_df, convert_tz\n",
+    "\n",
+    "\n",
+    "# Location to open raw data from data providers\n",
+    "DATA_RAW = \"data/raw\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Read Data From All-Data CSV (Multi Index Columns)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stock=True\n",
+    "df_all = read_data(os.path.join(DATA_RAW, \"realdata_pol_1h.csv\"), stock=stock)\n",
+    "# df_all = read_data(os.path.join(DATA_RAW, \"other/PEMSBAY.csv\"), stock=stock)\n",
+    "\n",
+    "df_all = df_all[df_all.columns[:-12]]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Filtering & Processing the Master Dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def percentage_nans(data, sort=True):\n",
+    "    percent_missing = data.isnull().sum() * 100 / len(data)\n",
+    "    missing_value_df = pd.DataFrame(\n",
+    "        {\"percent_missing\": percent_missing}  #'column_name': data.columns,\n",
+    "    )\n",
+    "    if sort:\n",
+    "        missing_value_df.sort_values(\"percent_missing\", inplace=True)\n",
+    "    return missing_value_df\n",
+    "\n",
+    "\n",
+    "def filter_percentage_nans(data, thresh=0.1):\n",
+    "    thresh *= 100\n",
+    "    per_nans = percentage_nans(data, sort=False)\n",
+    "    return data.loc[:, per_nans[per_nans[\"percent_missing\"] < thresh].index]\n",
+    "\n",
+    "\n",
+    "def filter_intra_ticker(data, cols=[\"close\"]):\n",
+    "    if cols is None:\n",
+    "        return data\n",
+    "    return data.iloc[\n",
+    "        :, data.columns.get_level_values(1).isin(cols)\n",
+    "    ]  # data.xs(\"close\",level=1, axis=1)\n",
+    "\n",
+    "\n",
+    "def no_premarket_after_hours(data):\n",
+    "    mkt_start = datetime.time(hour=9, minute=30, tzinfo=pytz.timezone(\"US/Eastern\"))\n",
+    "    mkt_end = datetime.time(hour=15, minute=59, tzinfo=pytz.timezone(\"US/Eastern\"))\n",
+    "    data = convert_tz(data, time_zone=\"US/Eastern\")\n",
+    "    data = data.between_time(mkt_start, mkt_end)\n",
+    "    data = convert_tz(data, time_zone=\"UTC\")\n",
+    "    return data\n",
+    "\n",
+    "\n",
+    "def add_technical(data):\n",
+    "    for ticker in data.columns.get_level_values(0).unique():\n",
+    "        # Assumption: close/open values are positive and a zero value means that datapoint is missing so we say no change\n",
+    "        data[ticker, \"pctchange\"] =  (\n",
+    "            data[ticker, \"close\"] / data[ticker, \"open\"] - 1\n",
+    "        ).fillna(0.0).replace([np.inf, -np.inf, -1], 0.0)\n",
+    "        data[ticker, \"logpctchange\"] = np.log(\n",
+    "            data[ticker, \"close\"] / data[ticker, \"open\"]\n",
+    "        ).fillna(0.0).replace([np.inf, -np.inf], 0.0)\n",
+    "\n",
+    "\n",
+    "        # data[ticker, \"pctchange-1\"] = data[ticker, \"pctchange\"].shift(1,fill_value=0.0)\n",
+    "        # data[ticker, \"pctchange-2\"] = data[ticker, \"pctchange\"].shift(2,fill_value=0.0)\n",
+    "\n",
+    "        data[ticker, \"shortsma\"] = (\n",
+    "            data[ticker, \"close\"].rolling(5).mean().fillna(data[ticker, \"close\"])\n",
+    "        )\n",
+    "        # data[ticker,'shortma-1'] = data[ticker,'shortsma'].shift(1)\n",
+    "        # data[ticker,'shortma-2'] = data[ticker,'shortsma'].shift(2)\n",
+    "    # print(data.columns.sort_values())\n",
+    "    data = data.reindex(sorted(data.columns), axis=1)\n",
+    "    # data.reindex(columns=data.columns.sort_values().get_level_values(0).unique(), level=0)\n",
+    "    return data\n",
+    "\n",
+    "if stock:\n",
+    "    # Filter df_all to normal hours\n",
+    "    df_all = no_premarket_after_hours(df_all)\n",
+    "\n",
+    "percentage_nans(df_all).tail(40)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = filter_percentage_nans(df_all, 0.08) #0.40\n",
+    "print(df.columns.get_level_values(0).unique())\n",
+    "df.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Add & filter columns\n",
+    "df = add_technical(df)\n",
+    "\n",
+    "# None\n",
+    "# [\"close\"]\n",
+    "# [\"pctchange\"]\n",
+    "# [\"open\", \"high\", \"low\", \"close\", \"volume\", 'pctchange', \"shortsma\"]\n",
+    "df = filter_intra_ticker(\n",
+    "    df, cols=[\"open\", \"close\", \"pctchange\", \"logpctchange\", \"shortsma\"]\n",
+    ")\n",
+    "\n",
+    "df.head(20)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "df_t = df[\"WTI\", \"pctchange\"]\n",
+    "start_date = \"2022-10-01\"\n",
+    "end_date = \"2022-11-01\"\n",
+    "f1 = df_t[df.index > start_date]\n",
+    "f2 = f1[f1.index < end_date]\n",
+    "print(f2)\n",
+    "# f = plt.figure()\n",
+    "# f.set_figwidth(60)\n",
+    "# f.set_figheight(20)\n",
+    "plt.figure(figsize=(24,4))\n",
+    "plt.plot(np.arange(f2.index.to_numpy().shape[0]), 3.3*  np.cumprod(f2.to_numpy()+1))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Fill NaNs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def ffill_nans(data):\n",
+    "    data = data.fillna(method=\"ffill\")\n",
+    "    data = data.dropna()\n",
+    "    return data\n",
+    "\n",
+    "\n",
+    "def del_nans_ffill(data, thresh):\n",
+    "    data = data.dropna(thresh=thresh)\n",
+    "    data = ffill_nans(data)\n",
+    "    return data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = ffill_nans(df)\n",
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Clip Outliers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def clip_outliers(data, p=0.005):\n",
+    "    lower = data.quantile(p)\n",
+    "    upper = data.quantile(1 - p)\n",
+    "\n",
+    "    return data.clip(lower=lower, upper=upper, axis=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if stock:\n",
+    "    df = clip_outliers(df)\n",
+    "\n",
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Save Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Sometimes it errors bc the path doesn't exist but just run it again\n",
+    "write_df(df, \"data/stock/material_1h.csv\")\n",
+    "# write_df(df, \"data/other/PEMSBAY.csv\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Extras"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "##### Read data and convert to percent delta"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# df_new = read_data(\"data/stock/close_1h.csv\")\n",
+    "\n",
+    "# print(\"Before:\\n\", df_new.head())\n",
+    "# df_new = df_new.pct_change()\n",
+    "# df_new.iloc[0] = 0\n",
+    "\n",
+    "# print(\"After:\\n\",df_new.head())\n",
+    "# write_df(df_new, \"data/stock/close_1h_pct_change.csv\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "51980e48e269f7c05efac26b22569386591d7f1d45336266d53ed7fc3ab7efc6"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

data_prepare.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# In[ ]:
+
+
+import pandas as pd
+import os
+import datetime
+import pytz
+import numpy as np
+from utils.ipynb_helpers import read_data, write_df, convert_tz
+
+
+# Location to open raw data from data providers
+DATA_RAW = "data/raw"
+
+
+# ##### Read Data From All-Data CSV (Multi Index Columns)
+
+# In[ ]:
+
+
+stock=True
+df_all = read_data(os.path.join(DATA_RAW, "realdata_pol_1h.csv"), stock=stock)
+# df_all = read_data(os.path.join(DATA_RAW, "other/PEMSBAY.csv"), stock=stock)
+
+df_all = df_all[df_all.columns[:-12]]
+
+
+# # Filtering & Processing the Master Dataset
+
+# In[ ]:
+
+
+def percentage_nans(data, sort=True):
+    percent_missing = data.isnull().sum() * 100 / len(data)
+    missing_value_df = pd.DataFrame(
+        {"percent_missing": percent_missing}  #'column_name': data.columns,
+    )
+    if sort:
+        missing_value_df.sort_values("percent_missing", inplace=True)
+    return missing_value_df
+
+
+def filter_percentage_nans(data, thresh=0.1):
+    thresh *= 100
+    per_nans = percentage_nans(data, sort=False)
+    return data.loc[:, per_nans[per_nans["percent_missing"] < thresh].index]
+
+
+def filter_intra_ticker(data, cols=["close"]):
+    if cols is None:
+        return data
+    return data.iloc[
+        :, data.columns.get_level_values(1).isin(cols)
+    ]  # data.xs("close",level=1, axis=1)
+
+
+def no_premarket_after_hours(data):
+    mkt_start = datetime.time(hour=9, minute=30, tzinfo=pytz.timezone("US/Eastern"))
+    mkt_end = datetime.time(hour=15, minute=59, tzinfo=pytz.timezone("US/Eastern"))
+    data = convert_tz(data, time_zone="US/Eastern")
+    data = data.between_time(mkt_start, mkt_end)
+    data = convert_tz(data, time_zone="UTC")
+    return data
+
+
+def add_technical(data):
+    for ticker in data.columns.get_level_values(0).unique():
+        # Assumption: close/open values are positive and a zero value means that datapoint is missing so we say no change
+        data[ticker, "pctchange"] =  (
+            data[ticker, "close"] / data[ticker, "open"] - 1
+        ).fillna(0.0).replace([np.inf, -np.inf, -1], 0.0)
+        data[ticker, "logpctchange"] = np.log(
+            data[ticker, "close"] / data[ticker, "open"]
+        ).fillna(0.0).replace([np.inf, -np.inf], 0.0)
+
+
+        # data[ticker, "pctchange-1"] = data[ticker, "pctchange"].shift(1,fill_value=0.0)
+        # data[ticker, "pctchange-2"] = data[ticker, "pctchange"].shift(2,fill_value=0.0)
+
+        data[ticker, "shortsma"] = (
+            data[ticker, "close"].rolling(5).mean().fillna(data[ticker, "close"])
+        )
+        # data[ticker,'shortma-1'] = data[ticker,'shortsma'].shift(1)
+        # data[ticker,'shortma-2'] = data[ticker,'shortsma'].shift(2)
+    # print(data.columns.sort_values())
+    data = data.reindex(sorted(data.columns), axis=1)
+    # data.reindex(columns=data.columns.sort_values().get_level_values(0).unique(), level=0)
+    return data
+
+if stock:
+    # Filter df_all to normal hours
+    df_all = no_premarket_after_hours(df_all)
+
+percentage_nans(df_all).tail(40)
+
+
+# In[ ]:
+
+
+df = filter_percentage_nans(df_all, 0.08) #0.40
+print(df.columns.get_level_values(0).unique())
+df.columns
+
+
+# In[ ]:
+
+
+# Add & filter columns
+df = add_technical(df)
+
+# None
+# ["close"]
+# ["pctchange"]
+# ["open", "high", "low", "close", "volume", 'pctchange', "shortsma"]
+df = filter_intra_ticker(
+    df, cols=["open", "close", "pctchange", "logpctchange", "shortsma"]
+)
+
+df.head(20)
+
+
+# In[ ]:
+
+
+import matplotlib.pyplot as plt
+df_t = df["WTI", "pctchange"]
+start_date = "2022-10-01"
+end_date = "2022-11-01"
+f1 = df_t[df.index > start_date]
+f2 = f1[f1.index < end_date]
+print(f2)
+# f = plt.figure()
+# f.set_figwidth(60)
+# f.set_figheight(20)
+plt.figure(figsize=(24,4))
+plt.plot(np.arange(f2.index.to_numpy().shape[0]), 3.3*  np.cumprod(f2.to_numpy()+1))
+
+
+# ##### Fill NaNs
+
+# In[ ]:
+
+
+def ffill_nans(data):
+    data = data.ffill()
+    #  data = data.fillna(method="ffill")
+    data = data.dropna()
+    return data
+
+
+def del_nans_ffill(data, thresh):
+    data = data.dropna(thresh=thresh)
+    data = ffill_nans(data)
+    return data
+
+
+# In[ ]:
+
+
+df = ffill_nans(df)
+df.head()
+
+
+# #### Clip Outliers
+
+# In[ ]:
+
+
+def clip_outliers(data, p=0.005):
+    lower = data.quantile(p)
+    upper = data.quantile(1 - p)
+
+    return data.clip(lower=lower, upper=upper, axis=1)
+
+
+# In[ ]:
+
+
+if stock:
+    df = clip_outliers(df)
+
+df.head()
+
+
+# ##### Save Data
+
+# In[ ]:
+
+
+# Sometimes it errors bc the path doesn't exist but just run it again
+write_df(df, "data/stock/material_1h.csv")
+# write_df(df, "data/other/PEMSBAY.csv")
+
+
+# ## Extras
+
+# ##### Read data and convert to percent delta
+
+# In[ ]:
+
+
+# df_new = read_data("data/stock/close_1h.csv")
+
+# print("Before:\n", df_new.head())
+# df_new = df_new.pct_change()
+# df_new.iloc[0] = 0
+
+# print("After:\n",df_new.head())
+# write_df(df_new, "data/stock/close_1h_pct_change.csv")
+plt.show()
+

data_provider/data_factory.py

@@ -0,0 +1,55 @@
+from data_provider.data_loader import (
+    Dataset_ETT_hour,
+    Dataset_ETT_minute,
+    Dataset_Custom,
+    Dataset_Pred,
+)
+from torch.utils.data import DataLoader
+
+data_dict = {
+    "ETTh1": Dataset_ETT_hour,
+    "ETTh2": Dataset_ETT_hour,
+    "ETTm1": Dataset_ETT_minute,
+    "ETTm2": Dataset_ETT_minute,
+    "WTH": Dataset_Custom,
+    "ECL": Dataset_Custom,
+    "Solar": Dataset_Custom,
+    "custom": Dataset_Custom,
+}
+
+
+def data_provider(args, flag):
+    Data = data_dict[args.data]
+
+    assert (
+        not args.inverse
+    ) or args.scale, "Can't enable inverse without enabling scale"
+
+    if flag == "test":
+        shuffle_flag = False
+        drop_last = True
+        batch_size = args.batch_size
+        # freq = args.freq
+    elif flag == "pred":
+        shuffle_flag = False
+        drop_last = False
+        batch_size = 1
+        # freq = args.detail_freq
+        Data = Dataset_Pred
+    else:
+        shuffle_flag = True
+        drop_last = True
+        batch_size = args.batch_size
+        # freq = args.freq
+
+    data_set = Data(args, flag=flag)
+
+    print(flag, len(data_set))
+    data_loader = DataLoader(
+        data_set,
+        batch_size=batch_size,
+        shuffle=shuffle_flag,
+        num_workers=args.num_workers,
+        drop_last=drop_last,
+    )
+    return data_set, data_loader

data_provider/data_loader.py

@@ -0,0 +1,652 @@
+import os
+import numpy as np
+import pandas as pd
+
+import torch
+from torch.utils.data import Dataset
+
+# from sklearn.preprocessing import StandardScaler
+
+from utils.tools import StandardScaler, dotdict
+from utils.timefeatures import time_features
+
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+class Dataset_ETT_hour(Dataset):
+    def __init__(
+        self,
+        root_path,
+        flag="train",
+        size=None,
+        features="S",
+        data_path="ETTh1.csv",
+        target="OT",
+        scale=True,
+        inverse=False,
+        timeenc=0,
+        freq="h",
+        cols=None,
+    ):
+        # size [seq_len, label_len, pred_len]
+        # info
+        if size == None:
+            self.seq_len = 24 * 4 * 4
+            self.label_len = 24 * 4
+            self.pred_len = 24 * 4
+        else:
+            self.seq_len = size[0]
+            self.label_len = size[1]
+            self.pred_len = size[2]
+        # init
+        assert flag in ["train", "test", "val"]
+        type_map = {"train": 0, "val": 1, "test": 2}
+        self.set_type = type_map[flag]
+
+        self.features = features
+        self.target = target
+        self.scale = scale
+        self.inverse = inverse
+        self.timeenc = timeenc
+        self.freq = freq
+
+        self.root_path = root_path
+        self.data_path = data_path
+        self.__read_data__()
+
+    def __read_data__(self):
+        self.scaler = StandardScaler()
+        df_raw = pd.read_csv(os.path.join(self.root_path, self.data_path))
+
+        border1s = [
+            0,
+            12 * 30 * 24 - self.seq_len,
+            12 * 30 * 24 + 4 * 30 * 24 - self.seq_len,
+        ]
+        border2s = [
+            12 * 30 * 24,
+            12 * 30 * 24 + 4 * 30 * 24,
+            12 * 30 * 24 + 8 * 30 * 24,
+        ]
+        border1 = border1s[self.set_type]
+        border2 = border2s[self.set_type]
+
+        if self.features == "M" or self.features == "MS":
+            cols_data = df_raw.columns[1:]
+            df_data = df_raw[cols_data]
+        elif self.features == "S":
+            df_data = df_raw[[self.target]]
+
+        if self.scale:
+            train_data = df_data[border1s[0] : border2s[0]]
+            self.scaler.fit(train_data.values, scale_mean=not self.config.no_scale_mean)
+            data = self.scaler.transform(df_data.values)
+        else:
+            data = df_data.values
+
+        df_stamp = df_raw[["date"]][border1:border2]
+        df_stamp["date"] = pd.to_datetime(df_stamp.date)
+        self.raw_dates = df_stamp.date.to_numpy(dtype=np.datetime64)
+        data_stamp = time_features(df_stamp, timeenc=self.timeenc, freq=self.freq)
+
+        self.data_x = data[border1:border2]
+        if self.inverse:
+            self.data_y = df_data.values[border1:border2]
+        else:
+            self.data_y = data[border1:border2]
+        self.data_stamp = data_stamp
+
+    def __getitem__(self, index):
+        s_begin = index
+        s_end = s_begin + self.seq_len
+        r_begin = s_end - self.label_len
+        r_end = r_begin + self.label_len + self.pred_len
+
+        seq_x = self.data_x[s_begin:s_end]
+        if self.inverse:
+            seq_y = np.concatenate(
+                [
+                    self.data_x[r_begin : r_begin + self.label_len],
+                    self.data_y[r_begin + self.label_len : r_end],
+                ],
+                0,
+            )
+        else:
+            seq_y = self.data_y[r_begin:r_end]
+        seq_x_mark = self.data_stamp[s_begin:s_end]
+        seq_y_mark = self.data_stamp[r_begin:r_end]
+
+        return seq_x, seq_y, seq_x_mark, seq_y_mark, index
+
+    def index_to_dates(self, index):
+        # index is of length batch_size
+        s_begin = index
+        s_end = s_begin + self.config.seq_len
+        r_begin = s_end - self.config.label_len
+        r_end = r_begin + self.config.label_len + self.config.pred_len
+
+        seq_x_raw_dates = self.raw_dates[
+            np.add.outer(s_begin, np.arange(self.config.seq_len))
+        ]
+        seq_y_raw_dates = self.raw_dates[
+            np.add.outer(
+                r_begin, np.arange(self.config.label_len + self.config.pred_len)
+            )
+        ]
+
+        return seq_x_raw_dates, seq_y_raw_dates
+
+    def __len__(self):
+        return len(self.data_x) - self.seq_len - self.pred_len + 1
+
+    def inverse_transform(self, data):
+        return self.scaler.inverse_transform(data)
+
+
+class Dataset_ETT_minute(Dataset):
+    def __init__(
+        self,
+        root_path,
+        flag="train",
+        size=None,
+        features="S",
+        data_path="ETTm1.csv",
+        target="OT",
+        scale=True,
+        inverse=False,
+        timeenc=0,
+        freq="t",
+        cols=None,
+    ):
+        # size [seq_len, label_len, pred_len]
+        # info
+        if size == None:
+            self.seq_len = 24 * 4 * 4
+            self.label_len = 24 * 4
+            self.pred_len = 24 * 4
+        else:
+            self.seq_len = size[0]
+            self.label_len = size[1]
+            self.pred_len = size[2]
+        # init
+        assert flag in ["train", "test", "val"]
+        type_map = {"train": 0, "val": 1, "test": 2}
+        self.set_type = type_map[flag]
+
+        self.features = features
+        self.target = target
+        self.scale = scale
+        self.inverse = inverse
+        self.timeenc = timeenc
+        self.freq = freq
+
+        self.root_path = root_path
+        self.data_path = data_path
+        self.__read_data__()
+
+    def __read_data__(self):
+        self.scaler = StandardScaler()
+        df_raw = pd.read_csv(os.path.join(self.root_path, self.data_path))
+
+        border1s = [
+            0,
+            12 * 30 * 24 * 4 - self.seq_len,
+            12 * 30 * 24 * 4 + 4 * 30 * 24 * 4 - self.seq_len,
+        ]
+        border2s = [
+            12 * 30 * 24 * 4,
+            12 * 30 * 24 * 4 + 4 * 30 * 24 * 4,
+            12 * 30 * 24 * 4 + 8 * 30 * 24 * 4,
+        ]
+        border1 = border1s[self.set_type]
+        border2 = border2s[self.set_type]
+
+        if self.features == "M" or self.features == "MS":
+            cols_data = df_raw.columns[1:]
+            df_data = df_raw[cols_data]
+        elif self.features == "S":
+            df_data = df_raw[[self.target]]
+
+        if self.scale:
+            train_data = df_data[border1s[0] : border2s[0]]
+            self.scaler.fit(train_data.values, scale_mean=not self.config.no_scale_mean)
+            data = self.scaler.transform(df_data.values)
+        else:
+            data = df_data.values
+
+        df_stamp = df_raw[["date"]][border1:border2]
+        df_stamp["date"] = pd.to_datetime(df_stamp.date)
+        self.raw_dates = df_stamp.date.to_numpy(dtype=np.datetime64)
+        data_stamp = time_features(df_stamp, timeenc=self.timeenc, freq=self.freq)
+
+        self.data_x = data[border1:border2]
+        if self.inverse:
+            self.data_y = df_data.values[border1:border2]
+        else:
+            self.data_y = data[border1:border2]
+        self.data_stamp = data_stamp
+
+    def __getitem__(self, index):
+        s_begin = index
+        s_end = s_begin + self.seq_len
+        r_begin = s_end - self.label_len
+        r_end = r_begin + self.label_len + self.pred_len
+
+        seq_x = self.data_x[s_begin:s_end]
+        if self.inverse:
+            seq_y = np.concatenate(
+                [
+                    self.data_x[r_begin : r_begin + self.label_len],
+                    self.data_y[r_begin + self.label_len : r_end],
+                ],
+                0,
+            )
+        else:
+            seq_y = self.data_y[r_begin:r_end]
+        seq_x_mark = self.data_stamp[s_begin:s_end]
+        seq_y_mark = self.data_stamp[r_begin:r_end]
+
+        return seq_x, seq_y, seq_x_mark, seq_y_mark, index
+
+    def index_to_dates(self, index):
+        # index is of length batch_size
+        s_begin = index
+        s_end = s_begin + self.config.seq_len
+        r_begin = s_end - self.config.label_len
+        r_end = r_begin + self.config.label_len + self.config.pred_len
+
+        seq_x_raw_dates = self.raw_dates[
+            np.add.outer(s_begin, np.arange(self.config.seq_len))
+        ]
+        seq_y_raw_dates = self.raw_dates[
+            np.add.outer(
+                r_begin, np.arange(self.config.label_len + self.config.pred_len)
+            )
+        ]
+
+        return seq_x_raw_dates, seq_y_raw_dates
+
+    def __len__(self):
+        return len(self.data_x) - self.seq_len - self.pred_len + 1
+
+    def inverse_transform(self, data):
+        return self.scaler.inverse_transform(data)
+
+
+class Dataset_Custom(Dataset):
+    def __init__(self, config, flag="train"):
+        # Default values
+        defaults = {
+            "size": None,
+            "features": "S",
+            "target": "OT",
+            "scale": True,
+            "inverse_pred": False,
+            "inverse_output": False,
+            "cols": None,
+            "date_start": None,
+            "date_end": None,
+            "date_test": None,
+            "date_val": None,
+            "t_embed": None,
+        }
+        config = dotdict({**defaults, **config})
+
+        assert config.seq_len is not None
+        assert config.label_len is not None
+        assert config.pred_len is not None
+        assert flag in ["train", "test", "val"]
+        assert config.freq is not None
+        assert config.root_path is not None
+        assert config.data_path is not None
+        assert (
+            (config.date_start is None)
+            or (config.date_end is None)
+            or (config.date_start < config.date_end)
+        ), "date_start isn't before date_end"
+
+        assert (
+            (config.date_test is None)
+            or (config.date_end is None)
+            or (config.date_test < config.date_end)
+        ), "date_test isn't before date_end"
+        assert (
+            (config.date_test is None)
+            or (config.date_start is None)
+            or (config.date_test > config.date_start)
+        ), "date_test isn't after date_start"
+
+        assert (config.date_val is None) or (
+            config.date_test is not None
+        ), "date_val is used without date_test"
+        assert (
+            (config.date_val is None)
+            or (config.date_test is None)
+            or (config.date_val < config.date_test)
+        ), "date_val isn't before date_test"
+
+        assert (
+            (config.date_val is None)
+            or (config.date_end is None)
+            or (config.date_val < config.date_end)
+        ), "date_val isn't before date_end"
+        assert (
+            (config.date_val is None)
+            or (config.date_start is None)
+            or (config.date_val > config.date_start)
+        ), "date_val isn't after date_start"
+
+        assert (config.label_len == 0) or (
+            config.inverse_output == config.inverse_pred
+        ), "If label length is non-zero then inverse_pred and inverse_output should be the same"
+
+        self.config = config
+        self.flag = flag
+
+        # self.timeenc = 0 if config.t_embed != "timeF" else 1
+        if config.t_embed == "timeF":
+            self.timeenc = 1
+        elif config.t_embed == "time2vec_add" or config.t_embed == "time2vec_app":
+            self.timeenc = 2
+        else:
+            self.timeenc = 0
+
+        type_map = {"train": 0, "val": 1, "test": 2}
+        self.set_type = type_map[flag]
+
+        self.__read_data__()
+
+    def __read_data__(self):
+        self.scaler = StandardScaler()
+        df_raw = pd.read_csv(os.path.join(self.config.root_path, self.config.data_path))
+        df_raw = df_raw.astype(
+            {c: np.float32 for c in df_raw.select_dtypes(include="float64").columns}
+        )
+        df_raw["date"] = pd.to_datetime(df_raw["date"])
+
+        if np.isinf(df_raw[df_raw.columns[1:]].to_numpy()).any():
+            raise Exception("There are inf's in the dataset")
+        if np.isnan(df_raw[df_raw.columns[1:]].to_numpy()).any():
+            raise Exception("There are nan's in the dataset")
+        """
+        df_raw.columns: ['date', ...(other features), target feature]
+        """
+        # Filter to datapoints in [date_start, date_end]
+        if self.config.date_start is not None:
+            df_raw = df_raw.loc[(df_raw["date"] >= self.config.date_start)]
+        if self.config.date_end is not None:
+            df_raw = df_raw.loc[(df_raw["date"] <= self.config.date_end)]
+
+        if self.config.cols:
+            cols = self.config.cols.copy()
+            assert self.config.target in cols, "Target not in cols"
+            cols.remove(self.config.target)
+        else:
+            cols = list(df_raw.columns)
+            assert self.config.target in cols, "Target not in data"
+            cols.remove(self.config.target)
+            assert "date" in cols, "`date` not in data"
+            cols.remove("date")
+        df_raw = df_raw[["date"] + cols + [self.config.target]]
+
+        # Define lengths of train, val, and test
+        if self.config.date_test is not None and self.config.date_val is not None:
+            # num_test and num_val are specified
+            num_test = len(df_raw.loc[df_raw["date"] >= self.config.date_test])
+            num_vali = len(
+                df_raw.loc[
+                    (df_raw["date"] >= self.config.date_val)
+                    & (df_raw["date"] < self.config.date_test)
+                ]
+            )
+            num_train = len(df_raw) - num_vali - num_test
+        elif self.config.date_test is not None:
+            # num_val is half of num_test which is specified
+            num_test = len(df_raw.loc[(df_raw["date"] >= self.config.date_test)])
+            num_vali = num_test // 2
+            num_train = len(df_raw) - num_vali - num_test
+        else:
+            # Default split
+            print("Warning: using default dataset split")
+            num_train = int(len(df_raw) * 0.7)
+            num_test = int(len(df_raw) * 0.2)
+            num_vali = len(df_raw) - num_train - num_test
+
+        if num_test == 0:
+            raise Exception("Dataset loading issue: num_test==0, check date settings")
+        elif num_vali == 0:
+            raise Exception("Dataset loading issue: num_vali==0, check date settings")
+        elif num_train == 0:
+            raise Exception("Dataset loading issue: num_train==0, check date settings")
+
+        border1s = [
+            0,
+            num_train - self.config.seq_len,
+            len(df_raw) - num_test - self.config.seq_len,
+        ]
+        border2s = [num_train, num_train + num_vali, len(df_raw)]
+        border1 = border1s[self.set_type]
+        border2 = border2s[self.set_type]
+
+        if self.config.features == "M" or self.config.features == "MS":
+            cols_data = df_raw.columns[1:]
+            df_data = df_raw[cols_data]
+        elif self.config.features == "S":
+            df_data = df_raw[[self.config.target]]
+
+        if self.config.scale:
+            train_data = df_data[border1s[0] : border2s[0]]
+            self.scaler.fit(train_data.values, scale_mean=not self.config.no_scale_mean)
+            data = torch.from_numpy(self.scaler.transform(df_data.values))
+        else:
+            data = torch.from_numpy(df_data.values)
+
+        df_stamp = df_raw[["date"]][border1:border2]
+        df_stamp["date"] = pd.to_datetime(df_stamp.date)
+        self.raw_dates = df_stamp.date.to_numpy(dtype=np.datetime64)
+        self.data_stamp = np.float32(
+            time_features(df_stamp, timeenc=self.timeenc, freq=self.config.freq)
+        )
+
+        self.data_x = data[border1:border2]
+        if self.config.inverse_pred:
+            self.data_y = torch.from_numpy(df_data.values[border1:border2])
+        else:
+            self.data_y = data[border1:border2]
+
+    def __getitem__(self, index):
+        s_begin = index
+        s_end = s_begin + self.config.seq_len
+        r_begin = s_end - self.config.label_len
+        r_end = r_begin + self.config.label_len + self.config.pred_len
+
+        seq_x = self.data_x[s_begin:s_end]
+        if self.config.inverse_pred:
+            # this is where inverse_pred != inverse output gets wonky if label_len != 0
+            # its because the label doesn't get inversed
+            seq_y = np.concatenate(
+                [
+                    self.data_x[
+                        r_begin : r_begin + self.config.label_len
+                    ],  # Use non-scaled data_x
+                    self.data_y[r_begin + self.config.label_len : r_end],
+                ],
+                axis=0,
+            )
+        else:
+            seq_y = self.data_y[r_begin:r_end]
+        seq_x_mark = self.data_stamp[s_begin:s_end]
+        seq_y_mark = self.data_stamp[r_begin:r_end]
+
+        return seq_x, seq_y, seq_x_mark, seq_y_mark, index
+
+    def index_to_dates(self, index):
+        # index is of length batch_size
+        s_begin = index
+        s_end = s_begin + self.config.seq_len
+        r_begin = s_end - self.config.label_len
+        r_end = r_begin + self.config.label_len + self.config.pred_len
+
+        seq_x_raw_dates = self.raw_dates[
+            np.add.outer(s_begin, np.arange(self.config.seq_len))
+        ]
+        seq_y_raw_dates = self.raw_dates[
+            np.add.outer(
+                r_begin, np.arange(self.config.label_len + self.config.pred_len)
+            )
+        ]
+        # seq_x_raw_dates = self.raw_dates[np.r_[s_begin,s_end-1].reshape(-1, index.shape[0]).T]# self.raw_dates.iloc[np.r_[s_begin,s_end]]
+        # seq_y_raw_dates = self.raw_dates[np.r_[r_begin,r_end-1].reshape(-1, index.shape[0]).T]# self.raw_dates.iloc[np.r_[r_begin,r_end]]
+
+        return seq_x_raw_dates, seq_y_raw_dates
+
+    def __len__(self):
+        return len(self.data_x) - self.config.seq_len - self.config.pred_len + 1
+
+    def inverse_transform(self, data):
+        return self.scaler.inverse_transform(data)
+
+
+class Dataset_Pred(Dataset):
+    def __init__(self, config, flag="pred"):
+        # Default values
+        defaults = {
+            "size": None,
+            "features": "S",
+            "target": "OT",
+            "scale": True,
+            "inverse": False,
+            "cols": None,
+            "date_start": None,
+            "date_end": None,
+            "t_embed": None,
+        }
+        config = dotdict({**defaults, **config})
+
+        assert config.seq_len is not None
+        assert config.label_len is not None
+        assert config.pred_len is not None
+        assert flag in ["pred"]
+        assert config.freq is not None
+        assert config.root_path is not None
+        assert config.data_path is not None
+        assert (
+            (config.date_start is None)
+            or (config.date_end is None)
+            or (config.date_start < config.date_end)
+        ), "date_start isn't before date_end"
+
+        self.config = config
+        self.flag = flag
+        # self.timeenc = 0 if config.t_embed != "timeF" else 1
+        if config.t_embed == "timeF":
+            self.timeenc = 1
+        elif config.t_embed == "time2vec_add" or config.t_embed == "time2vec_app":
+            self.timeenc = 2
+        else:
+            self.timeenc = 0
+
+        self.__read_data__()
+
+    def __read_data__(self):
+        self.scaler = StandardScaler()
+        df_raw = pd.read_csv(os.path.join(self.config.root_path, self.config.data_path))
+        """
+        df_raw.columns: ['date', ...(other features), target feature]
+        """
+
+        # Filter to datapoints in [date_start, date_end]
+        if self.config.date_start is not None:
+            df_raw = df_raw.loc[(df_raw["date"] >= self.config.date_start)]
+        if self.config.date_end is not None:
+            df_raw = df_raw.loc[(df_raw["date"] <= self.config.date_end)]
+
+        if self.config.cols:
+            cols = self.config.cols.copy()
+            cols.remove(self.config.target)
+        else:
+            cols = list(df_raw.columns)
+            cols.remove(self.config.target)
+            cols.remove("date")
+        df_raw = df_raw[["date"] + cols + [self.config.target]]
+
+        border1 = len(df_raw) - self.config.seq_len
+        border2 = len(df_raw)
+
+        if self.config.features == "M" or self.config.features == "MS":
+            cols_data = df_raw.columns[1:]
+            df_data = df_raw[cols_data]
+        elif self.config.features == "S":
+            df_data = df_raw[[self.config.target]]
+
+        if self.config.scale:
+            self.scaler.fit(df_data.values, scale_mean=not self.config.no_scale_mean)
+            data = self.scaler.transform(df_data.values)
+        else:
+            data = df_data.values
+
+        tmp_stamp = df_raw[["date"]][border1:border2]
+        tmp_stamp["date"] = pd.to_datetime(tmp_stamp.date)
+        pred_dates = pd.date_range(
+            tmp_stamp.date.values[-1],
+            periods=self.config.pred_len + 1,
+            freq=self.config.freq,
+        )
+
+        df_stamp = pd.DataFrame(columns=["date"])
+        df_stamp.date = pd.to_datetime(
+            list(tmp_stamp.date.values) + list(pred_dates[1:]), utc=True
+        )
+        self.raw_dates = df_stamp.date.to_numpy(dtype=np.datetime64)
+        # TODO: What is the deal with .freq[-1:]
+        self.data_stamp = np.float32(
+            time_features(df_stamp, timeenc=self.timeenc, freq=self.config.freq[-1:])
+        )
+
+        self.data_x = data[border1:border2]
+        if self.config.inverse:
+            self.data_y = df_data.values[border1:border2]
+        else:
+            self.data_y = data[border1:border2]
+
+    def __getitem__(self, index):
+        s_begin = index
+        s_end = s_begin + self.config.seq_len
+        r_begin = s_end - self.config.label_len
+        r_end = r_begin + self.config.label_len + self.config.pred_len
+
+        seq_x = self.data_x[s_begin:s_end]
+        if self.config.inverse:
+            seq_y = self.data_x[r_begin : r_begin + self.config.label_len]
+        else:
+            seq_y = self.data_y[r_begin : r_begin + self.config.label_len]
+        seq_x_mark = self.data_stamp[s_begin:s_end]
+        seq_y_mark = self.data_stamp[r_begin:r_end]
+
+        return seq_x, seq_y, seq_x_mark, seq_y_mark, index
+
+    def index_to_dates(self, index):
+        # index is of length batch_size
+        s_begin = index
+        s_end = s_begin + self.config.seq_len
+        r_begin = s_end - self.config.label_len
+        r_end = r_begin + self.config.label_len + self.config.pred_len
+
+        seq_x_raw_dates = self.raw_dates[
+            np.add.outer(s_begin, np.arange(self.config.seq_len))
+        ]
+        seq_y_raw_dates = self.raw_dates[
+            np.add.outer(
+                r_begin, np.arange(self.config.label_len + self.config.pred_len)
+            )
+        ]
+
+        return seq_x_raw_dates, seq_y_raw_dates
+
+    def __len__(self):
+        return len(self.data_x) - self.config.seq_len + 1
+
+    def inverse_transform(self, data):
+        return self.scaler.inverse_transform(data)

data_provider/data_module.py

@@ -0,0 +1,135 @@
+import numpy as np
+import pandas as pd
+
+from torch.utils.data import Dataset, DataLoader
+from data_provider.data_loader import (
+    Dataset_Custom,
+    Dataset_Pred,
+    # Dataset_ETT_hour,
+    # Dataset_ETT_minute,
+)
+from utils.tools import dotdict
+import pytorch_lightning as pl
+
+
+class CustomDataModule(pl.LightningDataModule):
+    def __init__(self, config: dotdict, num_workers: int = 0):
+        super().__init__()
+        self.data_train: Dataset | None = None
+        self.data_val: Dataset | None = None
+        self.data_test: Dataset | None = None
+        self.config = config
+
+        # pl makes self.batch_size special
+        self.batch_size = config.batch_size
+        self.num_workers = num_workers
+
+        assert (
+            not config.inverse
+        ) or config.scale, "Can't enable inverse without enabling scale"
+
+    def prepare_data(self):
+        """Download data if needed. This method is called only from a single GPU.
+        Do not use it to assign state (self.x = y)."""
+        pass
+
+    def setup(self, stage: str | None = None):
+        """Load data. Set variables: `self.data_train`, `self.data_val`, `self.data_test`.
+        This method is called by lightning twice for `trainer.fit()` and `trainer.test()`, so be careful if you do a random split!
+        The `stage` can be used to differentiate whether it's called before trainer.fit()` or `trainer.test()`.
+        """
+
+        self.data_train = Dataset_Custom(self.config, flag="train")
+        self.data_val = Dataset_Custom(self.config, flag="val")
+        self.data_test = Dataset_Custom(self.config, flag="test")
+        # self.data_pred = Dataset_Pred(self.config, flag="pred")
+        print(
+            f"LOADED DATASETS for {stage}: train: {len(self.data_train)}\tval: {len(self.data_val)}\ttest: {len(self.data_test)}"
+        )
+
+    def train_dataloader(self):
+        return DataLoader(
+            self.data_train,
+            batch_size=self.batch_size,
+            shuffle=not self.config.dont_shuffle_train,
+            num_workers=self.num_workers,
+            drop_last=True,
+        )
+
+    def val_dataloader(self):
+        # assert self.batch_size <= len(
+        #     self.data_val
+        # ), f"Batch size larger than val data set, batch size: {self.batch_size}, val size: {len(self.data_val)}"
+        return [
+            DataLoader(
+                self.data_val,
+                batch_size=self.batch_size,
+                shuffle=False,
+                drop_last=False,
+                num_workers=self.num_workers,
+            ),
+            DataLoader(
+                self.data_test,
+                batch_size=self.batch_size,
+                shuffle=False,
+                drop_last=False,
+                num_workers=self.num_workers,
+            ),
+        ]
+
+    def test_dataloader(self):
+        return [
+            DataLoader(
+                self.data_train,
+                batch_size=self.config.batch_size,
+                shuffle=False,
+                drop_last=False,
+                num_workers=self.num_workers,
+            ),
+            DataLoader(
+                self.data_val,
+                batch_size=self.config.batch_size,
+                shuffle=False,
+                drop_last=False,
+                num_workers=self.num_workers,
+            ),
+            DataLoader(
+                self.data_test,
+                batch_size=self.config.batch_size,
+                shuffle=False,
+                drop_last=False,
+                num_workers=self.num_workers,
+            ),
+        ]
+
+    def predict_dataloader(self):
+        return (
+            DataLoader(
+                self.data_train,
+                batch_size=self.config.batch_size,
+                shuffle=False,
+                drop_last=False,
+                num_workers=self.num_workers,
+            ),
+            DataLoader(
+                self.data_val,
+                batch_size=self.config.batch_size,
+                shuffle=False,
+                drop_last=False,
+                num_workers=self.num_workers,
+            ),
+            DataLoader(
+                self.data_test,
+                batch_size=self.config.batch_size,
+                shuffle=False,
+                drop_last=False,
+                num_workers=self.num_workers,
+            ),
+            # DataLoader(
+            #     self.data_pred,
+            #     batch_size=self.config.batch_size,
+            #     shuffle=False,
+            #     drop_last=False,
+            #     num_workers=self.num_workers,
+            # ),
+        )

embed.py

@@ -0,0 +1,228 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+import math
+
+
+class PositionalEmbedding(nn.Module):
+    def __init__(self, d_model, max_len=5000):
+        super(PositionalEmbedding, self).__init__()
+        # Compute the positional encodings once in log space.
+        pe = torch.zeros(max_len, d_model).float()
+        pe.require_grad = False
+
+        position = torch.arange(0, max_len).float().unsqueeze(1)
+        div_term = (
+            torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)
+        ).exp()
+
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+
+        pe = pe.unsqueeze(0)
+        self.register_buffer("pe", pe)
+
+    def forward(self, x):
+        return self.pe[:, : x.size(1)]
+
+
+class TokenEmbedding(nn.Module):
+    def __init__(self, c_in, d_model):
+        super(TokenEmbedding, self).__init__()
+        padding = 1 if torch.__version__ >= "1.5.0" else 2
+        self.tokenConv = nn.Conv1d(
+            in_channels=c_in,
+            out_channels=d_model,
+            kernel_size=3,
+            padding=padding,
+            padding_mode="circular",
+        )
+        for m in self.modules():
+            if isinstance(m, nn.Conv1d):
+                nn.init.kaiming_normal_(
+                    m.weight, mode="fan_in", nonlinearity="leaky_relu"
+                )
+
+    def forward(self, x):
+        x = self.tokenConv(x.permute(0, 2, 1)).transpose(1, 2)
+        return x
+
+
+class TokenEmbeddingBasic(nn.Module):
+    def __init__(self, c_in, d_model):
+        super(TokenEmbeddingBasic, self).__init__()
+        self.linear = nn.Linear(c_in, d_model)
+
+    def forward(self, x):
+        x = self.linear(x)
+        return x
+
+
+class FixedEmbedding(nn.Module):
+    def __init__(self, c_in, d_model):
+        super(FixedEmbedding, self).__init__()
+
+        w = torch.zeros(c_in, d_model).float()
+        w.require_grad = False
+
+        position = torch.arange(0, c_in).float().unsqueeze(1)
+        div_term = (
+            torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)
+        ).exp()
+
+        w[:, 0::2] = torch.sin(position * div_term)
+        w[:, 1::2] = torch.cos(position * div_term)
+
+        self.emb = nn.Embedding(c_in, d_model)
+        self.emb.weight = nn.Parameter(w, requires_grad=False)
+
+    def forward(self, x):
+        return self.emb(x).detach()
+
+
+class TemporalEmbedding(nn.Module):
+    def __init__(self, d_model, t_embed="fixed", freq="h"):
+        super(TemporalEmbedding, self).__init__()
+
+        minute_size = 4
+        hour_size = 24
+        weekday_size = 7
+        day_size = 32
+        month_size = 13
+
+        Embed = FixedEmbedding if t_embed == "fixed" else nn.Embedding
+        if freq == "t":
+            self.minute_embed = Embed(minute_size, d_model)
+        self.hour_embed = Embed(hour_size, d_model)
+        self.weekday_embed = Embed(weekday_size, d_model)
+        self.day_embed = Embed(day_size, d_model)
+        self.month_embed = Embed(month_size, d_model)
+
+    def forward(self, x):
+        x = x.long()
+
+        minute_x = (
+            self.minute_embed(x[:, :, 4]) if hasattr(self, "minute_embed") else 0.0
+        )
+        hour_x = self.hour_embed(x[:, :, 3])
+        weekday_x = self.weekday_embed(x[:, :, 2])
+        day_x = self.day_embed(x[:, :, 1])
+        month_x = self.month_embed(x[:, :, 0])
+
+        return hour_x + weekday_x + day_x + month_x + minute_x
+
+
+class TimeFeatureEmbedding(nn.Module):
+    def __init__(self, d_model, t_embed="timeF", freq="h"):
+        super(TimeFeatureEmbedding, self).__init__()
+
+        freq_map = {"h": 4, "t": 5, "s": 6, "m": 1, "a": 1, "w": 2, "d": 3, "b": 3}
+        d_inp = freq_map[freq]
+        self.embed = nn.Linear(d_inp, d_model)
+
+    def forward(self, x):
+        return self.embed(x)
+
+
+class Time2Vec(nn.Module):
+    def __init__(self, time_emb_dim, freq="h"):
+        super(Time2Vec, self).__init__()
+        freq_map = {"h": 4, "t": 5, "s": 6, "m": 1, "a": 1, "w": 2, "d": 3, "b": 3}
+        time_feat_dim = freq_map[freq]
+
+        self.output_dim = time_emb_dim
+
+        self.out_features = time_emb_dim
+
+        # TODO: Initialize uniform
+        self.linear_periodic = nn.Linear(time_feat_dim, time_emb_dim - 1)
+        self.linear_non_periodic = nn.Linear(time_feat_dim, 1)
+
+    def forward(self, x):
+        non_periodic = self.linear_non_periodic(x.float())
+        periodic = torch.sin(self.linear_periodic(x.float()))
+        out = torch.cat([non_periodic, periodic], -1)
+        return out
+
+
+class DataEmbedding(nn.Module):
+    def __init__(
+        self,
+        c_in,
+        d_model,
+        t_embed="fixed",
+        freq="h",
+        dropout_emb=0.01,
+        position_embedding=True,
+        emb_t2v_app_dim=32,
+        tok_emb="default",
+    ):
+        super(DataEmbedding, self).__init__()
+
+        self.append_time_emb = t_embed == "time2vec_app"
+
+        # For the temporal embedding
+        if t_embed is not None:
+            assert t_embed in [
+                "fixed",
+                "learned",
+                "timeF",
+                "time2vec_add",
+                "time2vec_app",
+            ], "Invalid t_embed"
+            if t_embed == "fixed" or t_embed == "learned":
+                self.temporal_embedding = TemporalEmbedding(
+                    d_model=d_model, t_embed=t_embed, freq=freq
+                )
+            elif t_embed == "timeF":
+                self.temporal_embedding = TimeFeatureEmbedding(
+                    d_model=d_model, t_embed=t_embed, freq=freq
+                )
+            elif t_embed == "time2vec_add":
+                # Time2Vec time embedding add elementwise
+                self.temporal_embedding = Time2Vec(time_emb_dim=d_model, freq=freq)
+            elif t_embed == "time2vec_app":
+                # Time2Vec time embedding appended
+                assert (
+                    emb_t2v_app_dim is not None
+                ), "Need to provide the emb_t2v_app_dim argument"
+                assert emb_t2v_app_dim > 0 and emb_t2v_app_dim < d_model
+                self.temporal_embedding = Time2Vec(
+                    time_emb_dim=emb_t2v_app_dim, freq=freq
+                )
+                d_model -= emb_t2v_app_dim
+        else:
+            self.temporal_embedding = lambda _: 0
+
+        # For the value embedding
+        if tok_emb == "basic":
+            self.value_embedding = TokenEmbeddingBasic(c_in=c_in, d_model=d_model)
+        elif tok_emb == "raw":
+            self.value_embedding = lambda x: x
+            assert c_in == d_model, "c_in and d_model must be equal for raw embedding"
+            assert (
+                t_embed != "time2vec_app"
+            ), "time2vec_app not supported for raw embedding"
+        else:
+            self.value_embedding = TokenEmbedding(c_in=c_in, d_model=d_model)
+
+        self.position_embedding = (
+            PositionalEmbedding(d_model=d_model) if position_embedding else lambda x: 0
+        )
+
+        self.dropout = nn.Dropout(p=dropout_emb)
+
+    def forward(self, x, x_mark):
+        if self.append_time_emb:
+            x = self.value_embedding(x) + self.position_embedding(x)
+            x_drop = self.dropout(x)
+            time_emb = self.temporal_embedding(x_mark)
+            return torch.concat([x_drop, time_emb], -1)
+        else:
+            x = (
+                self.value_embedding(x)
+                + self.position_embedding(x)
+                + self.temporal_embedding(x_mark)
+            )
+            return self.dropout(x)

exp/exp_basic.py

@@ -0,0 +1,38 @@
+import os
+import torch
+import numpy as np
+
+
+class Exp_Basic(object):
+    def __init__(self, args):
+        self.args = args
+        self.device = self._acquire_device()
+        self.model = self._build_model().to(self.device)
+
+    def _build_model(self):
+        raise NotImplementedError
+        return None
+
+    def _acquire_device(self):
+        if self.args.use_gpu:
+            os.environ["CUDA_VISIBLE_DEVICES"] = (
+                str(self.args.gpu) if not self.args.use_multi_gpu else self.args.devices
+            )
+            device = torch.device(f"cuda:{self.args.gpu}")
+            print(f"Use GPU: cuda:{self.args.gpu}")
+        else:
+            device = torch.device("cpu")
+            print("Use CPU")
+        return device
+
+    def _get_data(self, *args, **kwargs):
+        pass
+
+    def vali(self, *args, **kwargs):
+        pass
+
+    def train(self, *args, **kwargs):
+        pass
+
+    def test(self, *args, **kwargs):
+        pass

exp/exp_informer.py

@@ -0,0 +1,370 @@
+from data_provider.data_factory import data_provider
+from exp.exp_basic import Exp_Basic
+from models.Informer import Informer, InformerStack
+from models.Basic import NLinear, MLP
+from models.Stockformer import Stockformer
+
+from utils.tools import EarlyStopping, adjust_learning_rate
+from utils.metrics import metric
+from utils.stock_metrics import stock_loss
+
+import numpy as np
+
+import torch
+import torch.nn as nn
+from torch import optim
+
+import os
+import time
+import json
+
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+class Exp_Informer(Exp_Basic):
+    def __init__(self, args):
+        super(Exp_Informer, self).__init__(args)
+
+    def _build_model(self):
+        model_dict = {
+            "informer": Informer,
+            "informerstack": InformerStack,
+            "mlp": MLP,
+            "stockformer": Stockformer,
+            "nlinear": NLinear,
+        }
+
+        # Use stack layers for encoder layers if using informerstack
+        self.args.e_layers = (
+            self.args.s_layers
+            if self.args.model == "informerstack"
+            else self.args.e_layers
+        )
+
+        assert (
+            self.args.model in model_dict
+        ), f"Invalid args.model: {self.args.model}, options: {list(model_dict.keys())}"
+        model = model_dict[self.args.model](self.args).float()
+
+        if self.args.use_multi_gpu and self.args.use_gpu:
+            model = nn.DataParallel(model, device_ids=self.args.device_ids)
+        return model
+
+    def _get_data(self, flag):
+        data_set, data_loader = data_provider(self.args, flag)
+        return data_set, data_loader
+
+    def _select_optimizer(self):
+        model_optim = optim.Adam(self.model.parameters(), lr=self.args.learning_rate)
+        return model_optim
+
+    def _select_criterion(self):
+        if "stock" in self.args.loss:
+            _, stock_loss_mode = self.args.loss.split("_")
+            assert (
+                self.args.target.split("_")[1] == "pctchange"
+            ), "Can't use stock loss unless target is pctchange"
+            assert not (
+                self.args.scale and not self.args.inverse
+            ), "Can't use stock loss when args.scale==True and args.inverse==False."
+            criterion = stock_loss(self.args, stock_loss_mode=stock_loss_mode)
+        else:
+            assert self.args.loss == "mse"
+            criterion = nn.MSELoss()
+        return criterion
+
+    def _select_scheduler(self, optimizer):
+        if self.args.lradj == "type1":
+            lmbda = lambda epoch: 0.5
+            scheduler = torch.optim.lr_scheduler.MultiplicativeLR(
+                optimizer, lr_lambda=lmbda, verbose=True
+            )
+        elif self.args.lradj == "type2":
+            scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
+                optimizer,
+                factor=0.5,
+                patience=2,
+                threshold=1e-2,
+                cooldown=0,
+                verbose=True,
+            )
+        else:
+            scheduler = None
+        return scheduler
+
+    def vali(self, vali_data, vali_loader, criterion):
+        self.model.eval()
+        total_loss = []
+        for i, (batch_x, batch_y, batch_x_mark, batch_y_mark, _) in enumerate(
+            vali_loader
+        ):
+            pred, true, _ = self._process_one_batch(
+                vali_data, batch_x, batch_y, batch_x_mark, batch_y_mark, ds_index=None
+            )
+            loss = criterion(pred.detach().cpu(), true.detach().cpu())
+            total_loss.append(loss)
+        total_loss = np.average(total_loss)
+        self.model.train()
+        return total_loss
+
+    def train(self, setting):
+        train_data, train_loader = self._get_data(flag="train")
+        vali_data, vali_loader = self._get_data(flag="val")
+        test_data, test_loader = self._get_data(flag="test")
+
+        path = os.path.join(self.args.checkpoints, setting)
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+        # Save args
+        with open(os.path.join(path, "args.json"), "w") as convert_file:
+            convert_file.write(json.dumps(self.args))
+
+        time_now = time.time()
+
+        train_steps = len(train_loader)
+
+        early_stopping = None
+        if not self.args.no_early_stop:
+            early_stopping = EarlyStopping(patience=self.args.patience, verbose=True)
+
+        model_optim = self._select_optimizer()
+        criterion = self._select_criterion()
+        scheduler = self._select_scheduler(model_optim)
+
+        if self.args.use_amp:
+            scaler = torch.cuda.amp.GradScaler()
+
+        for epoch in range(self.args.max_epochs):
+            if epoch == 0:
+                for param_group in model_optim.param_groups:
+                    param_group["lr"] = 1e-8
+            elif epoch == 1:
+                for param_group in model_optim.param_groups:
+                    param_group["lr"] = self.args.learning_rate
+            iter_count = 0
+            train_loss = []
+
+            self.model.train()
+            epoch_time = time.time()
+            for i, (batch_x, batch_y, batch_x_mark, batch_y_mark, _) in enumerate(
+                train_loader
+            ):
+                iter_count += 1
+
+                model_optim.zero_grad()
+                pred, true, _ = self._process_one_batch(
+                    train_data,
+                    batch_x,
+                    batch_y,
+                    batch_x_mark,
+                    batch_y_mark,
+                    ds_index=None,
+                )
+                loss = criterion(pred, true)
+                train_loss.append(loss.item())
+
+                if (i + 1) % 100 == 0:
+                    print(
+                        "\titers: {0}, epoch: {1} | loss: {2:.7f}".format(
+                            i + 1, epoch + 1, loss.item()
+                        )
+                    )
+                    speed = (time.time() - time_now) / iter_count
+                    left_time = speed * (
+                        (self.args.max_epochs - epoch) * train_steps - i
+                    )
+                    print(
+                        "\tspeed: {:.4f}s/iter; left time: {:.4f}s".format(
+                            speed, left_time
+                        )
+                    )
+                    iter_count = 0
+                    time_now = time.time()
+
+                if self.args.use_amp:
+                    scaler.scale(loss).backward()
+                    scaler.step(model_optim)
+                    scaler.update()
+                else:
+                    loss.backward()
+                    model_optim.step()
+
+            print(f"Epoch: {epoch+1} cost time: {time.time()-epoch_time}")
+            train_loss = np.average(train_loss)
+            vali_loss = self.vali(vali_data, vali_loader, criterion)
+            test_loss = self.vali(test_data, test_loader, criterion)
+
+            print(
+                "Epoch: {0}, Steps: {1} | Train Loss: {2:.7f} Vali Loss: {3:.7f} Test"
+                " Loss: {4:.7f}".format(
+                    epoch + 1, train_steps, train_loss, vali_loss, test_loss
+                )
+            )
+
+            if not self.args.no_early_stop:
+                early_stopping(vali_loss, self.model, path)
+                if early_stopping.early_stop:
+                    print("Early stopping")
+                    break
+
+            # adjust_learning_rate(model_optim, epoch+1, self.args)
+            if scheduler is not None:
+                scheduler.step(metrics=vali_loss)
+
+        if self.args.no_early_stop:
+            # This is only for debugging
+            print("Saving overfitted model")
+            # os.rename(os.path.join(path, 'checkpoint.pth'), os.path.join(path, 'checkpoint-real.pth'))
+            torch.save(self.model.state_dict(), os.path.join(path, "checkpoint.pth"))
+        else:
+            best_model_path = os.path.join(path, "checkpoint.pth")
+            self.model.load_state_dict(torch.load(best_model_path))
+
+        return self.model
+
+    def test(self, setting, flag="test", inverse=True):
+        # Enable inverse if scale
+        inverse_og = self.args.inverse
+        self.args.inverse = self.args.scale and inverse
+
+        data, loader = self._get_data(flag=flag)
+
+        self.model.eval()
+
+        preds = []
+        trues = []
+        raw_dates = []
+
+        for i, (batch_x, batch_y, batch_x_mark, batch_y_mark, ds_index) in enumerate(
+            loader
+        ):
+            pred, true, rdates = self._process_one_batch(
+                data, batch_x, batch_y, batch_x_mark, batch_y_mark, ds_index=ds_index
+            )
+            preds.append(pred.detach().cpu().numpy())
+            trues.append(true.detach().cpu().numpy())
+            raw_dates.append(rdates)
+
+        assert len(preds) == len(trues)
+        preds = np.array(preds)
+        trues = np.array(trues)
+        raw_dates = np.array(raw_dates)
+        print(flag, "shape:", preds.shape, trues.shape)
+        preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1])
+        trues = trues.reshape(-1, trues.shape[-2], trues.shape[-1])
+        raw_dates = raw_dates.reshape(-1, raw_dates.shape[-1])
+        print(flag, "shape:", preds.shape, trues.shape)
+
+        # Result save
+        folder_path = os.path.join("./results/", setting)
+        if not os.path.exists(folder_path):
+            os.makedirs(folder_path)
+
+        # Save args
+        with open(os.path.join(folder_path, "args.json"), "w") as convert_file:
+            convert_file.write(json.dumps(self.args))
+
+        mae, mse, rmse, mape, mspe = metric(preds, trues)
+        print(f"{flag} mse:{mse}, mae:{mae}")
+
+        # Save metrics
+        with open(os.path.join(folder_path, "results.txt"), "a") as f:
+            f.write(f"{setting}\t{flag}\nmse:{mse}, mae:{mae}\n\n")
+        np.save(
+            os.path.join(folder_path, f"metrics_{flag}.npy"),
+            np.array([mae, mse, rmse, mape, mspe]),
+        )
+
+        # Save pred & true & raw dates
+        np.save(os.path.join(folder_path, f"pred_{flag}.npy"), preds)
+        np.save(os.path.join(folder_path, f"true_{flag}.npy"), trues)
+        np.save(os.path.join(folder_path, f"date_{flag}.npy"), raw_dates)
+        self.args.inverse = inverse_og
+        return
+
+    def predict(self, setting, load=False):
+        pred_data, pred_loader = self._get_data(flag="pred")
+
+        if load:
+            path = os.path.join(self.args.checkpoints, setting)
+            best_model_path = os.path.join(path, "checkpoint.pth")
+            self.model.load_state_dict(torch.load(best_model_path))
+
+        self.model.eval()
+
+        preds = []
+        # pred_trues = []
+        for i, (batch_x, batch_y, batch_x_mark, batch_y_mark, _) in enumerate(
+            pred_loader
+        ):
+            pred, true, _ = self._process_one_batch(
+                pred_data, batch_x, batch_y, batch_x_mark, batch_y_mark, ds_index=None
+            )
+            preds.append(pred.detach().cpu().numpy())
+            # pred_trues.append(true.detach().cpu().numpy())
+
+        preds = np.array(preds)
+        preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1])
+
+        # result save
+        folder_path = os.path.join("./results/", setting)
+        if not os.path.exists(folder_path):
+            os.makedirs(folder_path)
+
+        np.save(os.path.join(folder_path, "real_prediction.npy"), preds)
+
+        return
+
+    def _process_one_batch(
+        self,
+        dataset_object,
+        batch_x,
+        batch_y,
+        batch_x_mark,
+        batch_y_mark,
+        ds_index=None,
+    ):
+        batch_x = batch_x.float().to(self.device)
+        batch_y = batch_y.float()
+
+        batch_x_mark = batch_x_mark.float().to(self.device)
+        batch_y_mark = batch_y_mark.float().to(self.device)
+
+        # Decoder input if self.args.dec_in
+        dec_inp = None
+        if self.args.dec_in and (self.args.padding == 0 or self.args.padding == 1):
+            # FF: dec_inp = torch.zeros_like(batch_y[:, -self.args.pred_len:, :]).float()
+            dec_inp = torch.full(
+                [batch_y.shape[0], self.args.pred_len, batch_y.shape[-1]],
+                self.args.padding,
+            ).float()
+            dec_inp = (
+                torch.cat([batch_y[:, : self.args.label_len, :], dec_inp], dim=1)
+                .float()
+                .to(self.device)
+            )
+
+        # Encoder - Decoder
+        with torch.cuda.amp.autocast(enabled=self.args.use_amp):
+            if self.args.output_attention:
+                outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)[0]
+            else:
+                outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)
+        if self.args.inverse:
+            outputs = dataset_object.inverse_transform(outputs)
+        f_dim = -1 if self.args.features == "MS" else 0
+
+        if ds_index is None:
+            batch_y = batch_y[:, -self.args.pred_len :, f_dim:].to(self.device)
+            return outputs, batch_y, None
+        else:
+            batch_x_raw_dates, batch_y_raw_dates = dataset_object.index_to_dates(
+                ds_index
+            )
+            assert batch_y_raw_dates.shape == batch_y.shape[0:2]
+            batch_y = batch_y[:, -self.args.pred_len :, f_dim:].to(self.device)
+            batch_y_raw_dates = batch_y_raw_dates[:, -self.args.pred_len :]
+            return outputs, batch_y, batch_y_raw_dates

exp/exp_timeseries.py

@@ -0,0 +1,368 @@
+import torch
+import pytorch_lightning as pl
+
+from models.Basic import MLP
+from models.Lstm import LSTM
+from models.Informer import Informer, InformerStack
+from models.Stockformer import Stockformer
+from utils.stock_metrics import get_stock_algo, pct_direction_torch
+from torchmetrics import MeanSquaredError, MeanAbsoluteError
+from torch_optimizer import Ranger
+
+
+class ExpTimeseries(pl.LightningModule):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+
+        # pl makes self.learning_rate special
+        self.learning_rate = config.learning_rate
+
+        # Torch metrics has a state that resets but val and train can be called in unison so we split
+        # If pre_loss isn't supplied (ie: pre_loss is None) it will default to config.loss
+        self.train_criterion = self._select_criterion(
+            loss_override=self.config.pre_loss
+        )
+        self.other_criterion = self._select_criterion(
+            loss_override=self.config.pre_loss
+        )
+        self.loss_switched = False
+
+        self._build_model()
+        # self.save_hyperparameters()
+
+    def _build_model(self):
+        model_dict = {
+            "informer": Informer,
+            "informerstack": InformerStack,
+            "mlp": MLP,
+            "stockformer": Stockformer,
+            "lstm": LSTM,
+        }
+        assert (
+            self.config.model in model_dict
+        ), f"Invalid config.model: {self.config.model}, options: {list(model_dict.keys())}"
+        self.model = model_dict[self.config.model](self.config).float()
+
+        # Load model
+        if self.config.load_model_path is not None:
+            self.load_from_checkpoint(self.config.load_model_path)
+
+    def _select_criterion(self, loss_override=None):
+        loss = self.config.loss
+        if loss_override is not None:
+            loss = loss_override
+
+        def combine_loss(loss, weights=None):
+            if weights is None:
+                weights = [1.0] * len(loss)
+            def combined(pred, target, inv_pred):
+                #  print(pred.shape, target.shape)
+                #  while(1):pass
+                return sum(w*l(inv_pred, target) if "Mean" in l.__class__.__name__ else w*l(pred, target) for w,l in zip(weights, loss))
+            return combined
+
+        def loss_lib(loss: str):
+            if "stock" in loss:
+                # Using Stock Loss
+                _, stock_loss_mode = loss.split("_")
+                target_type = self.config.target.split("_")[1]
+                assert (
+                    target_type == "pctchange" or target_type == "logpctchange"
+                ), "Can't use stock loss unless target is pctchange or logpctchange"
+                assert (
+                    self.config.scale
+                    and self.config.inverse_pred
+                    #  and not self.config.inverse_output
+                ), "Can't use stock loss without scale, inverse pred, and not inverse output"
+
+                criterion = get_stock_algo(target_type, stock_loss_mode)
+                print("criterion:", criterion)
+                return lambda x, y: -1 * criterion.loss(x, y).mean()
+                # return lambda x, y: -LogPctProfitTanhV1.loss(x, y).mean()
+                # return get_stock_loss(target_type, stock_loss_mode, threshold=0.0)
+            elif loss == "mae":
+                assert (
+                    self.config.scale
+                    and self.config.inverse_pred
+                    #  and self.config.inverse_output
+                ), "Can't use mae loss without scale, inverse pred, and inverse output"
+                return MeanAbsoluteError().cuda()
+            elif loss == "mse":
+                assert (
+                    self.config.scale
+                    and self.config.inverse_pred
+                    #  and self.config.inverse_output
+                ), "Can't use mse loss without scale, inverse pred, and inverse output"
+                return MeanSquaredError().cuda()
+        loss_list = [ loss_lib(loss_type) for loss_type in loss.split('+') ]
+        weights = [1.0] if '+' not in loss else [10.0, 1.0]
+        return combine_loss(loss_list, weights)
+
+        raise Exception(f"Invalid loss: {loss}")
+
+    def forward(self, x):
+        # in lightning, forward defines the prediction/inference actions
+        return self.model(x)
+
+    def training_step(self, batch, batch_idx):
+        # training_step defines the train loop. It is independent of forward
+        batch_x, batch_y, batch_x_mark, batch_y_mark, _ = batch
+
+        pred, true, inv_pred = self._process_one_batch(
+            self.trainer.datamodule.data_train,
+            batch_x,
+            batch_y,
+            batch_x_mark,
+            batch_y_mark,
+            ds_index=None,
+        )
+        loss = self.train_criterion(pred, true, inv_pred)
+
+        self.log("train_loss", loss, prog_bar=True, on_step=True, on_epoch=True)
+
+        self.log(
+            "train_pct_dir",
+            pct_direction_torch(pred, true),
+            prog_bar=True,
+            on_step=False,
+            on_epoch=True,
+        )
+        self.log(
+            "train_mag",
+            torch.linalg.norm(pred),  # torch.mean(torch.abs(pred))
+            prog_bar=False,
+            on_step=False,
+            on_epoch=True,
+        )
+
+        if (
+            self.config.pre_epochs is not None
+            and self.config.pre_loss is not None
+            and self.current_epoch == self.config.pre_epochs
+            and not self.loss_switched
+        ):
+            # Revert to default loss
+            self.train_criterion = self._select_criterion(
+                loss_override=self.config.loss
+            )
+            self.other_criterion = self._select_criterion(
+                loss_override=self.config.loss
+            )
+            self.loss_switched = True
+
+        return loss
+
+    def validation_step(self, batch, batch_idx, dataloader_idx=0):
+        # validation_step defines the validation loop. It is independent of forward
+        batch_x, batch_y, batch_x_mark, batch_y_mark, _ = batch
+
+        pred, true, inv_pred = self._process_one_batch(
+            self.trainer.datamodule.data_val,
+            batch_x,
+            batch_y,
+            batch_x_mark,
+            batch_y_mark,
+            ds_index=None,
+        )
+
+        if dataloader_idx == 0:
+            # Actual val dataset
+            assert self.trainer.val_dataloaders[0].dataset.flag == "val"
+            loss = self.other_criterion(pred, true, inv_pred)
+            self.log(
+                "val_loss",
+                loss,
+                prog_bar=True,
+                on_step=False,
+                on_epoch=True,
+                sync_dist=False,
+                add_dataloader_idx=False,
+            )
+
+            self.log(
+                "val_pct_dir",
+                pct_direction_torch(pred, true),
+                prog_bar=False,
+                on_step=False,
+                on_epoch=True,
+                add_dataloader_idx=False,
+            )
+            return loss
+        elif dataloader_idx == 1:
+            # TODO: If we are using torch metrics we should create an additional loss function
+            # Test dataset
+            assert self.trainer.val_dataloaders[1].dataset.flag == "test"
+            loss = self.other_criterion(pred, true, inv_pred)
+            self.log(
+                "test_loss",
+                loss,
+                prog_bar=True,
+                on_step=False,
+                on_epoch=True,
+                sync_dist=False,
+                add_dataloader_idx=False,
+            )
+            self.log(
+                "test_pct_dir",
+                pct_direction_torch(pred, true),
+                prog_bar=False,
+                on_step=False,
+                on_epoch=True,
+                add_dataloader_idx=False,
+            )
+            return loss
+
+    def test_step(self, batch, batch_idx, dataloader_idx=0):
+        # test_step defines the test loop. It is independent of forward
+        batch_x, batch_y, batch_x_mark, batch_y_mark, _ = batch
+
+        data_sets = [
+            self.trainer.datamodule.data_train,
+            self.trainer.datamodule.data_val,
+            self.trainer.datamodule.data_test,
+        ]
+
+        pred, true, inv_pred = self._process_one_batch(
+            data_sets[dataloader_idx],
+            batch_x,
+            batch_y,
+            batch_x_mark,
+            batch_y_mark,
+            ds_index=None,
+        )
+        loss = self.other_criterion(pred, true, inv_pred)
+
+        # if dataloader_idx == 0:
+        self.log(
+            "test_loss",
+            loss,
+            sync_dist=False,
+        )
+
+    def predict_step(self, batch, batch_idx, dataloader_idx=0):
+        batch_x, batch_y, batch_x_mark, batch_y_mark, _ = batch
+
+        data_sets = [
+            self.trainer.datamodule.data_train,
+            self.trainer.datamodule.data_val,
+            self.trainer.datamodule.data_test,
+        ]
+
+        pred, true, inv_pred = self._process_one_batch(
+            data_sets[dataloader_idx],
+            batch_x,
+            batch_y,
+            batch_x_mark,
+            batch_y_mark,
+            ds_index=None,
+        )
+
+        # dataset = self.trainer.predict_dataloaders[dataloader_idx].dataset
+        # batch_x_raw_date, batch_y_raw_date = dataset.index_to_dates(batch_idx)
+
+        if "mse" in self.config.loss or "mae" in self.config.loss:
+            pred = inv_pred
+        return {
+            "pred": pred,
+            "true": true,
+        }
+
+    # def on_predict_epoch_end(self, results):
+    #     pass
+
+    # def on_predict_end(self):
+    #     pass
+
+    def _process_one_batch(
+        self,
+        dataset_object,
+        batch_x,
+        batch_y,
+        batch_x_mark,
+        batch_y_mark,
+        ds_index=None,
+    ):
+        # Decoder input if self.config.dec_in
+        dec_inp = None
+        # if self.config.dec_in and (
+        #     self.config.padding == 0 or self.config.padding == 1
+        # ):
+        #     # FF: dec_inp = torch.zeros_like(batch_y[:, -self.config.pred_len:, :]).float()
+        #     dec_inp = torch.full(
+        #         [batch_y.shape[0], self.config.pred_len, batch_y.shape[-1]],
+        #         self.config.padding,
+        #     ).float()
+        #     dec_inp = (
+        #         torch.cat([batch_y[:, : self.config.label_len, :], dec_inp], dim=1)
+        #         .float()
+        #         .to(self.device)
+        #     )
+
+        # Encoder - Decoder
+        if self.config.output_attention:
+            outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)[0]
+        else:
+            outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)
+        #  if self.config.inverse_output:
+        inv_outputs = dataset_object.inverse_transform(outputs)
+        f_dim = -1 if self.config.features == "MS" else 0
+
+        # if ds_index is None:
+        batch_y = batch_y[:, -self.config.pred_len :, f_dim:]
+        return outputs, batch_y, inv_outputs
+        # else:
+        #     batch_x_raw_dates, batch_y_raw_dates = dataset_object.index_to_dates(
+        #         ds_index
+        #     )
+        #     assert batch_y_raw_dates.shape == batch_y.shape[0:2]
+        #     batch_y = batch_y[:, -self.config.pred_len :, f_dim:].to(self.device)
+        #     batch_y_raw_dates = batch_y_raw_dates[:, -self.config.pred_len :]
+        #     return outputs, batch_y, batch_y_raw_dates
+
+    def configure_optimizers(self):
+        if self.config.optim == "AdamW":
+            optimizer = torch.optim.AdamW(self.parameters(), lr=self.learning_rate)
+        elif self.config.optim == "Ranger":
+            optimizer = Ranger(self.parameters(), lr=self.learning_rate)
+        else:
+            optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)
+        # optimizer = torch.optim.SGD(self.parameters(), lr=self.learning_rate)
+
+        # Learning rate scheduler
+        if self.config.lradj == "type1":
+            lmbda = lambda epoch: 0.5
+            scheduler = torch.optim.lr_scheduler.MultiplicativeLR(
+                optimizer, lr_lambda=lmbda, verbose=True
+            )
+        elif self.config.lradj == "type2":
+            scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
+                optimizer,
+                factor=0.5,
+                patience=10,
+                threshold=0,
+                cooldown=0,
+                verbose=True,
+                min_lr=1e-8,
+            )
+            scheduler = {
+                "scheduler": scheduler,
+                "interval": "epoch",  # called after each training epoch
+                "monitor": "val_loss",
+            }
+        elif self.config.lradj == "type3":
+            scheduler = torch.optim.lr_scheduler.OneCycleLR(
+                optimizer,
+                max_lr=self.config.learning_rate,
+                steps_per_epoch=len(self.trainer.datamodule.data_train)
+                // self.config.batch_size,  # Would be nicer to use self.trainer.train_dataloader.dataset but there is a pl bug
+                epochs=self.config.max_epochs,
+            )
+            scheduler = {
+                "scheduler": scheduler,
+                "interval": "step",  # called after each training step
+            }
+        else:
+            return optimizer
+
+        return [optimizer], [scheduler]

exp_timeseries.py

@@ -0,0 +1,466 @@
+import torch
+import pytorch_lightning as pl
+
+from collections import defaultdict
+from models.Basic import MLP
+from models.Lstm import LSTM
+from models.Informer import Informer, InformerStack
+from models.Stockformer import Stockformer
+from utils.stock_metrics import get_stock_algo, pct_direction_torch
+from torchmetrics import MeanSquaredError, MeanAbsoluteError
+from torch_optimizer import Ranger
+
+
+class ExpTimeseries(pl.LightningModule):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+
+        # pl makes self.learning_rate special
+        self.learning_rate = config.learning_rate
+
+        # Torch metrics has a state that resets but val and train can be called in unison so we split
+        # If pre_loss isn't supplied (ie: pre_loss is None) it will default to config.loss
+        self.train_criterion = self._select_criterion(
+            loss_override=self.config.pre_loss
+        )
+        self.other_criterion = self._select_criterion(
+            loss_override=self.config.pre_loss
+        )
+        self.metric = self._select_criterion(metric=True)
+        self.loss_switched = False
+
+        self._build_model()
+        # self.save_hyperparameters()
+        self.loss_reg = None
+        self.scale = None
+        self.val_log_growth_sum = None
+        self.val_log_growth_count = None
+        self.test_log_growth_sum = None
+        self.test_log_growth_count = None
+
+    def _build_model(self):
+        model_dict = {
+            "informer": Informer,
+            "informerstack": InformerStack,
+            "mlp": MLP,
+            "stockformer": Stockformer,
+            "lstm": LSTM,
+        }
+        assert (
+            self.config.model in model_dict
+        ), f"Invalid config.model: {self.config.model}, options: {list(model_dict.keys())}"
+        self.model = model_dict[self.config.model](self.config).float()
+
+        # Load model
+        if self.config.load_model_path is not None:
+            self.load_from_checkpoint(self.config.load_model_path)
+
+    def _select_criterion(self, loss_override=None, metric=False):
+        loss = self.config.loss
+        if loss_override is not None:
+            loss = loss_override
+
+        def combine_loss(loss, weights=None):
+            if weights is None:
+                weights = [1.0] * len(loss)
+            def combined(pred, target, inv_pred, input_scale=None):
+                return loss[0](pred, target, input_scale=input_scale)
+                #  return sum(w*l(inv_pred, target) if "Mean" in l.__class__.__name__ else w*l(pred, target) for w,l in zip(weights, loss))
+            return combined
+
+        def loss_lib(loss: str):
+            if "stock" in loss:
+                # Using Stock Loss
+                _, stock_loss_mode = loss.split("_")
+                target_type = self.config.target.split("_")[1]
+                assert (
+                    target_type == "pctchange" or target_type == "logpctchange"
+                ), "Can't use stock loss unless target is pctchange or logpctchange"
+                assert (
+                    self.config.scale and
+                    self.config.inverse_pred
+                    #  and not self.config.inverse_output
+                ), "Can't use stock loss without scale, inverse pred, and not inverse output"
+
+                criterion = get_stock_algo(target_type, stock_loss_mode)
+                print("criterion:", criterion)
+                if metric:
+                    def mt(x, y, input_scale):
+                        return criterion.metric(x, y, input_scale=input_scale)
+                    return mt
+                else:
+                    return lambda x, y, input_scale: [-1 * criterion.loss(x, y).mean(), criterion.sharpe(x, y).mean()]
+                # return lambda x, y: -LogPctProfitTanhV1.loss(x, y).mean()
+                # return get_stock_loss(target_type, stock_loss_mode, threshold=0.0)
+            elif loss == "mae":
+                assert (
+                    self.config.scale
+                    and self.config.inverse_pred
+                    #  and self.config.inverse_output
+                ), "Can't use mae loss without scale, inverse pred, and inverse output"
+                return MeanAbsoluteError().cuda()
+            elif loss == "mse":
+                assert (
+                    self.config.scale
+                    and self.config.inverse_pred
+                    #  and self.config.inverse_output
+                ), "Can't use mse loss without scale, inverse pred, and inverse output"
+                return MeanSquaredError().cuda()
+        loss_list = [ loss_lib(loss_type) for loss_type in loss.split('+') ]
+        weights = [1.0] if '+' not in loss else [1.0, 0.1]
+        return combine_loss(loss_list, weights)
+
+        raise Exception(f"Invalid loss: {loss}")
+
+    def forward(self, x):
+        # in lightning, forward defines the prediction/inference actions
+        return self.model(x)
+
+    def training_step(self, batch, batch_idx):
+        # training_step defines the train loop. It is independent of forward
+        batch_x, batch_y, batch_x_mark, batch_y_mark, _ = batch
+
+        sigma_x = 0.001
+        batch_x = batch_x + (torch.randn_like(batch_x)*2-1) * sigma_x
+        #  print(sigma_x.mean(), batch_x.mean(), batch_x.shape)
+        #  sigma_y = 0.01 * batch_y.std(dim=(1, 2), keepdim=True)
+        #  batch_y = batch_y + (torch.randn_like(batch_y)*2-1) * sigma_y
+
+        pred, true, inv_pred = self._process_one_batch(
+            self.trainer.datamodule.data_train,
+            batch_x,
+            batch_y,
+            batch_x_mark,
+            batch_y_mark,
+            ds_index=None,
+        )
+        #  print(self.loss_reg)
+        loss, sharpe = self.train_criterion(pred, true, inv_pred)
+        self.log("train_loss", loss, prog_bar=True, on_step=False, on_epoch=True)
+        self.log("train_sharpe", sharpe, prog_bar=True, on_step=False, on_epoch=True)
+        self.log("wavelet_loss", self.loss_reg, prog_bar=True, on_step=False, on_epoch=True)
+
+        #  self.log(
+            #  "tr_pct_dir",
+            #  pct_direction_torch(pred, true),
+            #  prog_bar=True,
+            #  on_step=False,
+            #  on_epoch=True,
+        #  )
+        #  self.log(
+            #  "tr_mag",
+            #  torch.linalg.norm(pred),  # torch.mean(torch.abs(pred))
+            #  prog_bar=False,
+            #  on_step=False,
+            #  on_epoch=True,
+        #  )
+
+        if (
+            self.config.pre_epochs is not None
+            and self.config.pre_loss is not None
+            and self.current_epoch == self.config.pre_epochs
+            and not self.loss_switched
+        ):
+            # Revert to default loss
+            self.train_criterion = self._select_criterion(
+                loss_override=self.config.loss
+            )
+            self.other_criterion = self._select_criterion(
+                loss_override=self.config.loss
+            )
+            self.loss_switched = True
+
+        return loss + torch.exp(-2.5*sharpe) + 1e0*self.loss_reg
+
+    def validation_step(self, batch, batch_idx, dataloader_idx=0):
+        # validation_step defines the validation loop. It is independent of forward
+        batch_x, batch_y, batch_x_mark, batch_y_mark, _ = batch
+
+        pred, true, inv_pred = self._process_one_batch(
+            self.trainer.datamodule.data_val,
+            batch_x,
+            batch_y,
+            batch_x_mark,
+            batch_y_mark,
+            ds_index=None,
+        )
+
+        if dataloader_idx == 0:
+            # Actual val dataset
+            #  assert self.trainer.val_dataloaders[0].dataset.flag == "val"
+            loss, sharpe = self.other_criterion(pred, true, inv_pred)
+            self.log(
+                "val_loss",
+                loss,
+                prog_bar=True,
+                on_step=False,
+                on_epoch=True,
+                sync_dist=False,
+                add_dataloader_idx=False,
+            )
+            self.log(
+                "val_sharpe",
+                sharpe,
+                prog_bar=True,
+                on_step=False,
+                on_epoch=True,
+                sync_dist=False,
+                add_dataloader_idx=False,
+            )
+            raw, self.scale = self.metric(pred, true, inv_pred)
+            self.val_log_growth_sum[0] += raw.detach().sum()
+            self.val_log_growth_count[0] += raw.numel()
+            #  self.log(
+                #  "val_pct_dir",
+                #  pct_direction_torch(pred, true),
+                #  prog_bar=False,
+                #  on_step=False,
+                #  on_epoch=True,
+                #  add_dataloader_idx=False,
+            #  )
+            return
+        elif dataloader_idx == 1:
+            # TODO: If we are using torch metrics we should create an additional loss function
+            # Test dataset
+            assert self.trainer.val_dataloaders[1].dataset.flag == "test"
+            loss, sharpe = self.other_criterion(pred, true, inv_pred)
+            self.log(
+                "test_loss",
+                loss,
+                prog_bar=True,
+                on_step=False,
+                on_epoch=True,
+                sync_dist=False,
+                add_dataloader_idx=False,
+            )
+            self.log(
+                "test_sharpe",
+                sharpe,
+                prog_bar=True,
+                on_step=False,
+                on_epoch=True,
+                sync_dist=False,
+                add_dataloader_idx=False,
+            )
+            raw, _ = self.metric(pred, true, inv_pred, self.scale)
+            self.val_log_growth_sum[1] += raw.detach().sum()
+            self.val_log_growth_count[1] += raw.numel()
+            #  self.log(
+                #  "test_pct_dir",
+                #  pct_direction_torch(pred, true),
+                #  prog_bar=False,
+                #  on_step=False,
+                #  on_epoch=True,
+                #  add_dataloader_idx=False,
+            #  )
+            return
+
+    def on_validation_epoch_start(self):
+        self.val_log_growth_sum = defaultdict(lambda: 0.0)
+        self.val_log_growth_count = defaultdict(int)
+
+    def on_validation_epoch_end(self):
+        for dl_idx, sum_log in self.val_log_growth_sum.items():
+            #  count = self.val_log_growth_count[dl_idx]
+            factor = torch.exp(sum_log)
+            roi = factor - 1
+            if dl_idx == 0:
+                name = "val_roi"
+            elif dl_idx == 1:
+                name = "test_roi"
+            else:
+                raise Exception
+            self.log(
+                name,
+                roi,
+                prog_bar=True,
+                on_step=False,
+                on_epoch=True,
+                sync_dist=False,
+                add_dataloader_idx=False,
+            )
+        #  # 或者用平均 log-growth 当 metric（和 T 无关，更稳）
+        #  mean_log_growth = self.val_log_growth_sum / self.val_log_growth_count
+        #  self.log("val_mean_log_growth", mean_log_growth, prog_bar=False)
+
+    def test_step(self, batch, batch_idx, dataloader_idx=0):
+        # test_step defines the test loop. It is independent of forward
+        batch_x, batch_y, batch_x_mark, batch_y_mark, _ = batch
+
+        data_sets = [
+            self.trainer.datamodule.data_train,
+            self.trainer.datamodule.data_val,
+            self.trainer.datamodule.data_test,
+        ]
+
+        pred, true, inv_pred = self._process_one_batch(
+            data_sets[dataloader_idx],
+            batch_x,
+            batch_y,
+            batch_x_mark,
+            batch_y_mark,
+            ds_index=None,
+        )
+        #  loss = self.other_criterion(pred, true, inv_pred)
+        #  # if dataloader_idx == 0:
+        #  self.log(
+            #  "test_loss",
+            #  loss,
+            #  sync_dist=False,
+        #  )
+
+        if dataloader_idx == 0:
+            raw, _ = self.metric(pred, true, inv_pred)
+        if dataloader_idx == 1:
+            raw, self.scale = self.metric(pred, true, inv_pred)
+        if dataloader_idx == 2:
+            raw, _ = self.metric(pred, true, inv_pred, self.scale)
+        self.test_log_growth_sum[dataloader_idx] += raw.detach().sum()
+        self.test_log_growth_count[dataloader_idx] += raw.numel()
+
+    def on_test_epoch_start(self):
+        self.test_log_growth_sum = defaultdict(lambda: 0.0)
+        self.test_log_growth_count = defaultdict(int)
+
+    def on_test_epoch_end(self):
+        for dl_idx, sum_log in self.test_log_growth_sum.items():
+            factor = torch.exp(sum_log)
+            roi = factor - 1
+            self.log(
+                "test_roi",
+                roi,
+                sync_dist=False,
+            )
+
+    def predict_step(self, batch, batch_idx, dataloader_idx=0):
+        batch_x, batch_y, batch_x_mark, batch_y_mark, _ = batch
+
+        data_sets = [
+            self.trainer.datamodule.data_train,
+            self.trainer.datamodule.data_val,
+            self.trainer.datamodule.data_test,
+        ]
+
+        pred, true, inv_pred = self._process_one_batch(
+            data_sets[dataloader_idx],
+            batch_x,
+            batch_y,
+            batch_x_mark,
+            batch_y_mark,
+            ds_index=None,
+        )
+
+        # dataset = self.trainer.predict_dataloaders[dataloader_idx].dataset
+        # batch_x_raw_date, batch_y_raw_date = dataset.index_to_dates(batch_idx)
+
+        if "mse" in self.config.loss or "mae" in self.config.loss:
+            pred = inv_pred
+        return {
+            "pred": pred.detach().to(torch.float32),
+            "true": true.detach().to(torch.float32),
+        }
+
+    # def on_predict_epoch_end(self, results):
+    #     pass
+
+    # def on_predict_end(self):
+    #     pass
+
+    def _process_one_batch(
+        self,
+        dataset_object,
+        batch_x,
+        batch_y,
+        batch_x_mark,
+        batch_y_mark,
+        ds_index=None,
+    ):
+        # Decoder input if self.config.dec_in
+        dec_inp = None
+        # if self.config.dec_in and (
+        #     self.config.padding == 0 or self.config.padding == 1
+        # ):
+        #     # FF: dec_inp = torch.zeros_like(batch_y[:, -self.config.pred_len:, :]).float()
+        #     dec_inp = torch.full(
+        #         [batch_y.shape[0], self.config.pred_len, batch_y.shape[-1]],
+        #         self.config.padding,
+        #     ).float()
+        #     dec_inp = (
+        #         torch.cat([batch_y[:, : self.config.label_len, :], dec_inp], dim=1)
+        #         .float()
+        #         .to(self.device)
+        #     )
+
+        # Encoder - Decoder
+        if self.config.output_attention:
+            outputs = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)[0]
+        else:
+            outputs, loss_reg = self.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)
+            self.loss_reg = loss_reg
+        #  if self.config.inverse_output:
+        f_dim = -1 if self.config.features == "MS" else 0
+
+        # if ds_index is None:
+        batch_y = batch_y[:, -self.config.pred_len :, f_dim:]
+        #  print(batch_y.std())
+        #  batch_y = dataset_object.inverse_transform(batch_y)
+        #  print(batch_y.std())
+        #  while 1:pass
+        inv_outputs = dataset_object.inverse_transform(outputs)
+        return outputs, batch_y, inv_outputs
+        # else:
+        #     batch_x_raw_dates, batch_y_raw_dates = dataset_object.index_to_dates(
+        #         ds_index
+        #     )
+        #     assert batch_y_raw_dates.shape == batch_y.shape[0:2]
+        #     batch_y = batch_y[:, -self.config.pred_len :, f_dim:].to(self.device)
+        #     batch_y_raw_dates = batch_y_raw_dates[:, -self.config.pred_len :]
+        #     return outputs, batch_y, batch_y_raw_dates
+
+    def configure_optimizers(self):
+        if self.config.optim == "AdamW":
+            optimizer = torch.optim.AdamW(self.parameters(), lr=self.learning_rate)
+        elif self.config.optim == "Ranger":
+            optimizer = Ranger(self.parameters(), lr=self.learning_rate)
+        else:
+            optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)
+        # optimizer = torch.optim.SGD(self.parameters(), lr=self.learning_rate)
+
+        # Learning rate scheduler
+        if self.config.lradj == "type1":
+            lmbda = lambda epoch: 0.5
+            scheduler = torch.optim.lr_scheduler.MultiplicativeLR(
+                optimizer, lr_lambda=lmbda, verbose=True
+            )
+        elif self.config.lradj == "type2":
+            scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
+                optimizer,
+                factor=0.5,
+                patience=10,
+                threshold=0,
+                cooldown=0,
+                verbose=True,
+                min_lr=1e-8,
+            )
+            scheduler = {
+                "scheduler": scheduler,
+                "interval": "epoch",  # called after each training epoch
+                "monitor": "val_loss",
+            }
+        elif self.config.lradj == "type3":
+            scheduler = torch.optim.lr_scheduler.OneCycleLR(
+                optimizer,
+                max_lr=self.config.learning_rate,
+                steps_per_epoch=len(self.trainer.datamodule.data_train)
+                // self.config.batch_size,  # Would be nicer to use self.trainer.train_dataloader.dataset but there is a pl bug
+                epochs=self.config.max_epochs,
+            )
+            scheduler = {
+                "scheduler": scheduler,
+                "interval": "step",  # called after each training step
+            }
+        else:
+            return optimizer
+
+        return [optimizer], [scheduler]

general_Banks_Diversified.yaml

@@ -0,0 +1,57 @@
+activation: gelu
+tok_emb: raw
+attn: full
+c_out: 1
+cols:
+- EWBC_logpctchange
+- SNV_logpctchange
+- FITB_logpctchange
+- TFC_logpctchange
+- FNB_logpctchange
+- MTB_logpctchange
+- FULT_logpctchange
+- ZION_logpctchange
+- KEY_logpctchange
+- CFG_logpctchange
+- HBAN_logpctchange
+data_path: material_Banks_Diversified_1h.csv
+date_end: '2025-10-29'
+date_start: '2020-10-29'
+# date_test: '2025-06-29'
+des: full_1h
+distil: false
+dont_shuffle_train: false
+dropout: 0.1
+dropout_emb: 0.0
+t_embed: time2vec_app
+enc_in: 11
+factor: 5
+features: MS
+freq: h
+inverse_pred: true
+loss: stock_tanhv1
+final_mode: mode3
+label_len: 0
+learning_rate: 1.0e-04
+batch_size: 256
+d_ff: 1024
+d_model: 512
+n_heads: 512
+seq_len: 100
+e_layers: 6
+emb_t2v_app_dim: 128
+ln_mode: pre
+lradj: null
+mix: false
+model: stockformer
+no_early_stop: false
+no_scale_mean: true
+optim: AdamW
+output_attention: false
+patience: 100
+pred_len: 1
+root_path: ./data/stock/
+scale: true
+seed: 4
+target: HBAN_logpctchange
+max_epochs: 1000

general_Life_Insurance.yaml

@@ -0,0 +1,53 @@
+activation: gelu
+tok_emb: raw
+attn: full
+c_out: 1
+cols:
+- CNO_logpctchange
+- PFG_logpctchange
+- LNC_logpctchange
+- MET_logpctchange
+- PRU_logpctchange
+- BHF_logpctchange
+- EQH_logpctchange
+data_path: material_Life_Insurance_1h.csv
+date_end: '2025-10-29'
+date_start: '2020-10-29'
+# date_test: '2025-06-29'
+des: full_1h
+distil: false
+dont_shuffle_train: false
+dropout: 0.1
+dropout_emb: 0.0
+t_embed: time2vec_app
+enc_in: 7
+factor: 5
+features: MS
+freq: h
+inverse_pred: true
+loss: stock_tanhv1
+final_mode: mode3
+label_len: 0
+learning_rate: 1.0e-04
+batch_size: 256
+d_ff: 1024
+d_model: 512
+n_heads: 512
+seq_len: 100
+e_layers: 6
+emb_t2v_app_dim: 128
+ln_mode: pre
+lradj: null
+mix: false
+model: stockformer
+no_early_stop: false
+no_scale_mean: true
+optim: AdamW
+output_attention: false
+patience: 100
+pred_len: 1
+root_path: ./data/stock/
+scale: true
+seed: 4
+target: EQH_logpctchange
+max_epochs: 1000

general_Semiconductors_Equipment.yaml

@@ -0,0 +1,48 @@
+activation: gelu
+attn: full
+c_out: 1
+cols:
+- AMAT_logpctchange
+- LRCX_logpctchange
+- KLAC_logpctchange
+data_path: material_Semiconductors_Equipment_1h.csv
+date_end: '2025-10-29'
+date_start: '2020-10-29'
+# date_test: '2025-06-29'
+des: full_1h
+distil: false
+dont_shuffle_train: false
+dropout: 0.0
+dropout_emb: 0.0
+t_embed: time2vec_app
+enc_in: 3
+factor: 5
+features: MS
+freq: h
+inverse_pred: true
+loss: stock_tanhv1
+final_mode: mode3
+label_len: 0
+learning_rate: 1.0e-04
+batch_size: 256
+d_ff: 1024
+d_model: 512
+n_heads: 512
+seq_len: 100
+e_layers: 12
+emb_t2v_app_dim: 256
+ln_mode: pre
+lradj: null
+mix: false
+model: stockformer
+no_early_stop: false
+no_scale_mean: true
+optim: AdamW
+output_attention: false
+patience: 100
+pred_len: 1
+root_path: ./data/stock/
+scale: true
+seed: 4
+target: KLAC_logpctchange
+max_epochs: 333

layers/attn.py

@@ -0,0 +1,184 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+import numpy as np
+
+from math import sqrt
+from utils.masking import TriangularCausalMask, ProbMask
+
+
+class FullAttention(nn.Module):
+    def __init__(
+        self,
+        mask_flag=True,
+        factor=5,
+        scale=None,
+        attention_dropout=0.1,
+        output_attention=False,
+    ):
+        super(FullAttention, self).__init__()
+        self.scale = scale
+        self.mask_flag = mask_flag
+        self.output_attention = output_attention
+        self.dropout = nn.Dropout(attention_dropout)
+
+    def forward(self, queries, keys, values, attn_mask):
+        B, L, H, E = queries.shape
+        _, S, _, D = values.shape
+        scale = self.scale or 1.0 / sqrt(E)
+
+        scores = torch.einsum("blhe,bshe->bhls", queries, keys)
+        if self.mask_flag:
+            if attn_mask is None:
+                attn_mask = TriangularCausalMask(B, L, device=queries.device)
+
+            scores.masked_fill_(attn_mask.mask, -np.inf)
+
+        A = self.dropout(torch.softmax(scale * scores, dim=-1))
+        V = torch.einsum("bhls,bshd->blhd", A, values)
+
+        if self.output_attention:
+            return (V.contiguous(), A)
+        else:
+            return (V.contiguous(), None)
+
+
+class ProbAttention(nn.Module):
+    def __init__(
+        self,
+        mask_flag=True,
+        factor=5,
+        scale=None,
+        attention_dropout=0.1,
+        output_attention=False,
+    ):
+        super(ProbAttention, self).__init__()
+        self.factor = factor
+        self.scale = scale
+        self.mask_flag = mask_flag
+        self.output_attention = output_attention
+        self.dropout = nn.Dropout(attention_dropout)
+
+    def _prob_QK(self, Q, K, sample_k, n_top):  # n_top: c*ln(L_q)
+        # Q [B, H, L, D]
+        B, H, L_K, E = K.shape
+        _, _, L_Q, _ = Q.shape
+
+        # calculate the sampled Q_K
+        K_expand = K.unsqueeze(-3).expand(B, H, L_Q, L_K, E)
+        index_sample = torch.randint(
+            L_K, (L_Q, sample_k)
+        )  # real U = U_part(factor*ln(L_k))*L_q
+        K_sample = K_expand[:, :, torch.arange(L_Q).unsqueeze(1), index_sample, :]
+        Q_K_sample = torch.matmul(Q.unsqueeze(-2), K_sample.transpose(-2, -1)).squeeze(
+            -2
+        )
+
+        # find the Top_k query with sparisty measurement
+        M = Q_K_sample.max(-1)[0] - torch.div(Q_K_sample.sum(-1), L_K)
+        M_top = M.topk(n_top, sorted=False)[1]
+
+        # use the reduced Q to calculate Q_K
+        Q_reduce = Q[
+            torch.arange(B)[:, None, None], torch.arange(H)[None, :, None], M_top, :
+        ]  # factor*ln(L_q)
+        Q_K = torch.matmul(Q_reduce, K.transpose(-2, -1))  # factor*ln(L_q)*L_k
+
+        return Q_K, M_top
+
+    def _get_initial_context(self, V, L_Q):
+        B, H, L_V, D = V.shape
+        if not self.mask_flag:
+            # V_sum = V.sum(dim=-2)
+            V_sum = V.mean(dim=-2)
+            contex = V_sum.unsqueeze(-2).expand(B, H, L_Q, V_sum.shape[-1]).clone()
+        else:  # use mask
+            assert L_Q == L_V  # requires that L_Q == L_V, i.e. for self-attention only
+            contex = V.cumsum(dim=-2)
+        return contex
+
+    def _update_context(self, context_in, V, scores, index, L_Q, attn_mask):
+        B, H, L_V, D = V.shape
+
+        if self.mask_flag:
+            attn_mask = ProbMask(B, H, L_Q, index, scores, device=V.device)
+            scores.masked_fill_(attn_mask.mask, -np.inf)
+
+        attn = torch.softmax(scores, dim=-1)  # nn.Softmax(dim=-1)(scores)
+
+        context_in[
+            torch.arange(B)[:, None, None], torch.arange(H)[None, :, None], index, :
+        ] = torch.matmul(attn, V).type_as(context_in)
+        if self.output_attention:
+            attns = (torch.ones([B, H, L_V, L_V]) / L_V).type_as(attn).to(attn.device)
+            attns[
+                torch.arange(B)[:, None, None], torch.arange(H)[None, :, None], index, :
+            ] = attn
+            return (context_in, attns)
+        else:
+            return (context_in, None)
+
+    def forward(self, queries, keys, values, attn_mask):
+        B, L_Q, H, D = queries.shape
+        _, L_K, _, _ = keys.shape
+
+        queries = queries.transpose(2, 1)
+        keys = keys.transpose(2, 1)
+        values = values.transpose(2, 1)
+
+        U_part = self.factor * np.ceil(np.log(L_K)).astype("int").item()  # c*ln(L_k)
+        u = self.factor * np.ceil(np.log(L_Q)).astype("int").item()  # c*ln(L_q)
+
+        U_part = U_part if U_part < L_K else L_K
+        u = u if u < L_Q else L_Q
+
+        scores_top, index = self._prob_QK(queries, keys, sample_k=U_part, n_top=u)
+
+        # add scale factor
+        scale = self.scale or 1.0 / sqrt(D)
+        if scale is not None:
+            scores_top = scores_top * scale
+        # get the context
+        context = self._get_initial_context(values, L_Q)
+        # update the context with selected top_k queries
+        context, attn = self._update_context(
+            context, values, scores_top, index, L_Q, attn_mask
+        )
+
+        return context.transpose(2, 1).contiguous(), attn
+
+
+class AttentionLayer(nn.Module):
+    def __init__(
+        self, attention, d_model, n_heads, d_keys=None, d_values=None, mix=False
+    ):
+        super(AttentionLayer, self).__init__()
+
+        d_keys = d_keys or (d_model // n_heads)
+        d_values = d_values or (d_model // n_heads)
+
+        self.inner_attention = attention
+        self.query_projection = nn.Linear(d_model, d_keys * n_heads)
+        self.key_projection = nn.Linear(d_model, d_keys * n_heads)
+        self.value_projection = nn.Linear(d_model, d_values * n_heads)
+        self.out_projection = nn.Linear(d_values * n_heads, d_model)
+        self.n_heads = n_heads
+        self.mix = mix
+
+    def forward(self, queries, keys, values, attn_mask):
+        B, L, _ = queries.shape
+        _, S, _ = keys.shape
+        H = self.n_heads
+
+        queries = self.query_projection(queries).view(B, L, H, -1)
+        keys = self.key_projection(keys).view(B, S, H, -1)
+        values = self.value_projection(values).view(B, S, H, -1)
+
+        out, attn = self.inner_attention(queries, keys, values, attn_mask)
+        if self.mix:
+            # https://arxiv.org/pdf/2109.02789.pdf
+            out = out.transpose(2, 1).contiguous()
+        out = out.view(B, L, -1)
+
+        return self.out_projection(out), attn

layers/decoder.py

@@ -0,0 +1,56 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class DecoderLayer(nn.Module):
+    def __init__(
+        self,
+        self_attention,
+        cross_attention,
+        d_model,
+        d_ff=None,
+        dropout=0.1,
+        activation="relu",
+    ):
+        super(DecoderLayer, self).__init__()
+        d_ff = d_ff or 4 * d_model
+        self.self_attention = self_attention
+        self.cross_attention = cross_attention
+        self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1)
+        self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1)
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.norm3 = nn.LayerNorm(d_model)
+        self.dropout = nn.Dropout(dropout)
+        self.activation = F.relu if activation == "relu" else F.gelu
+
+    def forward(self, x, cross, x_mask=None, cross_mask=None):
+        x = x + self.dropout(self.self_attention(x, x, x, attn_mask=x_mask)[0])
+        x = self.norm1(x)
+
+        x = x + self.dropout(
+            self.cross_attention(x, cross, cross, attn_mask=cross_mask)[0]
+        )
+
+        y = x = self.norm2(x)
+        y = self.dropout(self.activation(self.conv1(y.transpose(-1, 1))))
+        y = self.dropout(self.conv2(y).transpose(-1, 1))
+
+        return self.norm3(x + y)
+
+
+class Decoder(nn.Module):
+    def __init__(self, layers, norm_layer=None):
+        super(Decoder, self).__init__()
+        self.layers = nn.ModuleList(layers)
+        self.norm = norm_layer
+
+    def forward(self, x, cross, x_mask=None, cross_mask=None):
+        for layer in self.layers:
+            x = layer(x, cross, x_mask=x_mask, cross_mask=cross_mask)
+
+        if self.norm is not None:
+            x = self.norm(x)
+
+        return x

layers/embed.py

@@ -0,0 +1,228 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+import math
+
+
+class PositionalEmbedding(nn.Module):
+    def __init__(self, d_model, max_len=5000):
+        super(PositionalEmbedding, self).__init__()
+        # Compute the positional encodings once in log space.
+        pe = torch.zeros(max_len, d_model).float()
+        pe.require_grad = False
+
+        position = torch.arange(0, max_len).float().unsqueeze(1)
+        div_term = (
+            torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)
+        ).exp()
+
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+
+        pe = pe.unsqueeze(0)
+        self.register_buffer("pe", pe)
+
+    def forward(self, x):
+        return self.pe[:, : x.size(1)]
+
+
+class TokenEmbedding(nn.Module):
+    def __init__(self, c_in, d_model):
+        super(TokenEmbedding, self).__init__()
+        padding = 1 if torch.__version__ >= "1.5.0" else 2
+        self.tokenConv = nn.Conv1d(
+            in_channels=c_in,
+            out_channels=d_model,
+            kernel_size=3,
+            padding=padding,
+            padding_mode="circular",
+        )
+        for m in self.modules():
+            if isinstance(m, nn.Conv1d):
+                nn.init.kaiming_normal_(
+                    m.weight, mode="fan_in", nonlinearity="leaky_relu"
+                )
+
+    def forward(self, x):
+        x = self.tokenConv(x.permute(0, 2, 1)).transpose(1, 2)
+        return x
+
+
+class TokenEmbeddingBasic(nn.Module):
+    def __init__(self, c_in, d_model):
+        super(TokenEmbeddingBasic, self).__init__()
+        self.linear = nn.Linear(c_in, d_model)
+
+    def forward(self, x):
+        x = self.linear(x)
+        return x
+
+
+class FixedEmbedding(nn.Module):
+    def __init__(self, c_in, d_model):
+        super(FixedEmbedding, self).__init__()
+
+        w = torch.zeros(c_in, d_model).float()
+        w.require_grad = False
+
+        position = torch.arange(0, c_in).float().unsqueeze(1)
+        div_term = (
+            torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)
+        ).exp()
+
+        w[:, 0::2] = torch.sin(position * div_term)
+        w[:, 1::2] = torch.cos(position * div_term)
+
+        self.emb = nn.Embedding(c_in, d_model)
+        self.emb.weight = nn.Parameter(w, requires_grad=False)
+
+    def forward(self, x):
+        return self.emb(x).detach()
+
+
+class TemporalEmbedding(nn.Module):
+    def __init__(self, d_model, t_embed="fixed", freq="h"):
+        super(TemporalEmbedding, self).__init__()
+
+        minute_size = 4
+        hour_size = 24
+        weekday_size = 7
+        day_size = 32
+        month_size = 13
+
+        Embed = FixedEmbedding if t_embed == "fixed" else nn.Embedding
+        if freq == "t":
+            self.minute_embed = Embed(minute_size, d_model)
+        self.hour_embed = Embed(hour_size, d_model)
+        self.weekday_embed = Embed(weekday_size, d_model)
+        self.day_embed = Embed(day_size, d_model)
+        self.month_embed = Embed(month_size, d_model)
+
+    def forward(self, x):
+        x = x.long()
+
+        minute_x = (
+            self.minute_embed(x[:, :, 4]) if hasattr(self, "minute_embed") else 0.0
+        )
+        hour_x = self.hour_embed(x[:, :, 3])
+        weekday_x = self.weekday_embed(x[:, :, 2])
+        day_x = self.day_embed(x[:, :, 1])
+        month_x = self.month_embed(x[:, :, 0])
+
+        return hour_x + weekday_x + day_x + month_x + minute_x
+
+
+class TimeFeatureEmbedding(nn.Module):
+    def __init__(self, d_model, t_embed="timeF", freq="h"):
+        super(TimeFeatureEmbedding, self).__init__()
+
+        freq_map = {"h": 4, "t": 5, "s": 6, "m": 1, "a": 1, "w": 2, "d": 3, "b": 3}
+        d_inp = freq_map[freq]
+        self.embed = nn.Linear(d_inp, d_model)
+
+    def forward(self, x):
+        return self.embed(x)
+
+
+class Time2Vec(nn.Module):
+    def __init__(self, time_emb_dim, freq="h"):
+        super(Time2Vec, self).__init__()
+        freq_map = {"h": 4, "t": 5, "s": 6, "m": 1, "a": 1, "w": 2, "d": 3, "b": 3}
+        time_feat_dim = freq_map[freq]
+
+        self.output_dim = time_emb_dim
+
+        self.out_features = time_emb_dim
+
+        # TODO: Initialize uniform
+        self.linear_periodic = nn.Linear(time_feat_dim, time_emb_dim - 1)
+        self.linear_non_periodic = nn.Linear(time_feat_dim, 1)
+
+    def forward(self, x):
+        non_periodic = self.linear_non_periodic(x.float())
+        periodic = torch.sin(self.linear_periodic(x.float()))
+        out = torch.cat([non_periodic, periodic], -1)
+        return out
+
+
+class DataEmbedding(nn.Module):
+    def __init__(
+        self,
+        c_in,
+        d_model,
+        t_embed="fixed",
+        freq="h",
+        dropout_emb=0.01,
+        position_embedding=True,
+        emb_t2v_app_dim=32,
+        tok_emb="default",
+    ):
+        super(DataEmbedding, self).__init__()
+
+        self.append_time_emb = t_embed == "time2vec_app"
+
+        # For the temporal embedding
+        if t_embed is not None:
+            assert t_embed in [
+                "fixed",
+                "learned",
+                "timeF",
+                "time2vec_add",
+                "time2vec_app",
+            ], "Invalid t_embed"
+            if t_embed == "fixed" or t_embed == "learned":
+                self.temporal_embedding = TemporalEmbedding(
+                    d_model=d_model, t_embed=t_embed, freq=freq
+                )
+            elif t_embed == "timeF":
+                self.temporal_embedding = TimeFeatureEmbedding(
+                    d_model=d_model, t_embed=t_embed, freq=freq
+                )
+            elif t_embed == "time2vec_add":
+                # Time2Vec time embedding add elementwise
+                self.temporal_embedding = Time2Vec(time_emb_dim=d_model, freq=freq)
+            elif t_embed == "time2vec_app":
+                # Time2Vec time embedding appended
+                assert (
+                    emb_t2v_app_dim is not None
+                ), "Need to provide the emb_t2v_app_dim argument"
+                assert emb_t2v_app_dim > 0 and emb_t2v_app_dim < d_model
+                self.temporal_embedding = Time2Vec(
+                    time_emb_dim=emb_t2v_app_dim, freq=freq
+                )
+                d_model -= emb_t2v_app_dim
+        else:
+            self.temporal_embedding = lambda _: 0
+
+        # For the value embedding
+        if tok_emb == "basic":
+            self.value_embedding = TokenEmbeddingBasic(c_in=c_in, d_model=d_model)
+        elif tok_emb == "raw":
+            self.value_embedding = lambda x: x
+            assert c_in == d_model, "c_in and d_model must be equal for raw embedding"
+            assert (
+                t_embed != "time2vec_app"
+            ), "time2vec_app not supported for raw embedding"
+        else:
+            self.value_embedding = TokenEmbedding(c_in=c_in, d_model=d_model)
+
+        self.position_embedding = (
+            PositionalEmbedding(d_model=d_model) if position_embedding else lambda x: 0
+        )
+
+        self.dropout = nn.Dropout(p=dropout_emb)
+
+    def forward(self, x, x_mark):
+        if self.append_time_emb:
+            x = self.value_embedding(x) + self.position_embedding(x)
+            x_drop = self.dropout(x)
+            time_emb = self.temporal_embedding(x_mark)
+            return torch.concat([x_drop, time_emb], -1)
+        else:
+            x = (
+                self.value_embedding(x)
+                + self.position_embedding(x)
+                + self.temporal_embedding(x_mark)
+            )
+            return self.dropout(x)

layers/encoder.py

@@ -0,0 +1,216 @@
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+# --------------------------
+# Utilities / Norm / Activations
+# --------------------------
+
+class RMSNorm(nn.Module):
+    """RMSNorm with learnable weight. Drop-in for LayerNorm when using Pre-Norm."""
+    def __init__(self, d_model, eps=1e-8):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(d_model))
+
+    def forward(self, x):
+        # x: (..., D)
+        norm = x.pow(2).mean(dim=-1, keepdim=True).add(self.eps).rsqrt()
+        return self.weight * x * norm
+
+
+def get_activation(name: str):
+    name = (name or "relu").lower()
+    if name == "relu":
+        return nn.ReLU()
+    if name in ("gelu", "geglu"):
+        return nn.GELU()
+    if name in ("silu", "swish"):
+        return nn.SiLU()
+    return nn.ReLU()
+
+
+class SwiGLU(nn.Module):
+    """SwiGLU FFN: proj( SiLU(a) * b ), a,b from linear split."""
+    def __init__(self, d_model, d_ff):
+        super().__init__()
+        self.w12 = nn.Linear(d_model, 2 * d_ff, bias=True)
+        self.proj = nn.Linear(d_ff, d_model, bias=True)
+
+    def forward(self, x):
+        a, b = self.w12(x).chunk(2, dim=-1)
+        return self.proj(F.silu(a) * b)
+
+
+# --------------------------
+# Conv Layer (kept signature)
+# --------------------------
+
+class ConvLayer(nn.Module):
+    def __init__(self, c_in):
+        super(ConvLayer, self).__init__()
+        padding = 1 if torch.__version__ >= "1.5.0" else 2
+        self.downConv = nn.Conv1d(
+            in_channels=c_in,
+            out_channels=c_in,
+            kernel_size=3,
+            padding=padding,
+            padding_mode="circular",
+        )
+        self.norm = nn.BatchNorm1d(c_in)
+        self.activation = nn.ELU()
+        self.maxPool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1)
+
+    def forward(self, x):
+        # x: [B, L, D]
+        x = x.permute(0, 2, 1)                       # B, D, L
+        y = self.downConv(x)
+        y = self.norm(y)
+        y = self.activation(y)
+        y = self.maxPool(y)
+        y = y.transpose(1, 2).contiguous()           # B, L', D
+        return y
+
+
+# --------------------------
+# Encoder Layer (kept signature)
+# --------------------------
+
+class EncoderLayer(nn.Module):
+    """
+    Keep the same signature:
+      __init__(self, attention, d_model, d_ff=None, dropout=0.1, activation="relu", ln_mode="pre", conv_layer=False)
+      forward(self, x, attn_mask=None)
+    Internals:
+      - Pre-Norm by default (ln_mode="pre")
+      - RMSNorm (instead of LN) but callable doesn’t change
+      - Residual scaling 1/sqrt(2)
+      - FFN uses SwiGLU, dropout after branch
+      - Attention module is expected to have signature (q, k, v, attn_mask=None) -> (new_x, attn)
+        and internally do q *= 1/sqrt(d_head)
+    """
+    def __init__(
+        self,
+        attention,
+        d_model,
+        d_ff=None,
+        dropout=0.1,
+        activation="relu",
+        ln_mode="pre",
+        conv_layer=False,
+    ):
+        super(EncoderLayer, self).__init__()
+        self.attention = attention
+        self.conv_layer = ConvLayer(d_model) if conv_layer else None
+        self.dropout = nn.Dropout(dropout)
+        self.activation = get_activation(activation)
+        self.ln_mode = ln_mode  # will honor "pre" / "post" without changing signature
+
+        # Core hyperparams
+        self.d_model = d_model
+        self.d_ff = d_ff or 4 * d_model
+        self.res_scale = 1.0 / math.sqrt(2.0)
+
+        # Use RMSNorm but keep object names norm1/norm2 to avoid API change elsewhere
+        self.norm1 = RMSNorm(d_model)
+        self.norm2 = RMSNorm(d_model)
+
+        # FFN: use SwiGLU for better stability/accuracy
+        self.ff = SwiGLU(d_model, self.d_ff)
+
+        # In case some pipeline expects LayerNorm instance, we also keep a post-norm if ln_mode="post"
+        # (But the actual normalization used is RMSNorm above; this is just to respect the mode)
+        if self.ln_mode == "post":
+            self.post_ln1 = nn.LayerNorm(d_model)
+            self.post_ln2 = nn.LayerNorm(d_model)
+
+    def forward(self, x, attn_mask=None):
+        # x: [B, L, D]
+
+        if self.conv_layer is not None:
+            x = x + self.dropout(self.conv_layer(x)) * self.res_scale
+
+        if self.ln_mode == "post":
+            # -------- Post-LN path (kept behavior but more explicit/clean) --------
+            new_x, attn = self.attention(x, x, x, attn_mask=attn_mask)
+            x = x + self.dropout(new_x) * self.res_scale
+            x = self.post_ln1(x)
+
+            y = self.ff(x)
+            x = x + self.dropout(y) * self.res_scale
+            x = self.post_ln2(x)
+            return x, attn
+
+        # -------- Default: Pre-LN (recommended) --------
+        # Attention branch (Pre-Norm)
+        h, attn = self.attention(self.norm1(x), self.norm1(x), self.norm1(x), attn_mask=attn_mask)
+        x = x + self.dropout(h) * self.res_scale
+
+        # FFN branch (Pre-Norm)
+        y = self.ff(self.norm2(x))
+        x = x + self.dropout(y) * self.res_scale
+
+        return x, attn
+
+
+# --------------------------
+# Encoder (kept signature)
+# --------------------------
+
+class Encoder(nn.Module):
+    """
+    Keep the same signature:
+      __init__(self, attn_layers, conv_layers=None, norm_layer=None)
+      forward(self, x, attn_mask=None)
+    """
+    def __init__(self, attn_layers, conv_layers=None, norm_layer=None):
+        super(Encoder, self).__init__()
+        self.attn_layers = nn.ModuleList(attn_layers)
+        self.conv_layers = nn.ModuleList(conv_layers) if conv_layers is not None else None
+        self.norm = norm_layer  # can be None or nn.Module
+
+    def forward(self, x, attn_mask=None):
+        # x: [B, L, D]
+        attns = []
+        for i, attn_layer in enumerate(self.attn_layers):
+            x, attn = attn_layer(x, attn_mask=attn_mask)
+            attns.append(attn)
+            if self.conv_layers is not None and i < len(self.conv_layers):
+                x = self.conv_layers[i](x)
+
+        if self.norm is not None:
+            x = self.norm(x)
+        return x, attns
+
+
+# --------------------------
+# Encoder Stack (kept signature)
+# --------------------------
+
+class EncoderStack(nn.Module):
+    """
+    Keep the same signature:
+      __init__(self, encoders, inp_lens, d_model)
+      forward(self, x, attn_mask=None)
+    """
+    def __init__(self, encoders, inp_lens, d_model):
+        super(EncoderStack, self).__init__()
+        self.encoders = nn.ModuleList(encoders)
+        self.inp_lens = inp_lens
+        self.d_model = d_model
+
+    def forward(self, x, attn_mask=None):
+        # x: [B, L, D]
+        x_stack = []
+        attns = []
+        # For each pyramid level, take the tail part of the sequence
+        for i_len, encoder in zip(self.inp_lens, self.encoders):
+            inp_len = x.shape[1] // (2 ** i_len)
+            x_s, attn = encoder(x[:, -inp_len:, :], attn_mask=attn_mask)
+            x_stack.append(x_s)
+            attns.append(attn)
+        x_stack = torch.cat(x_stack, dim=-2)  # concat on sequence length axis
+        return x_stack, attns
+

models/Basic.py

@@ -0,0 +1,63 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+from layers.embed import Time2Vec
+
+
+class MLP(nn.Module):
+    """
+    Just your everyday neural net
+    """
+
+    def __init__(self, config):
+        super(MLP, self).__init__()
+        self.seq_len = config.seq_len
+        self.pred_len = config.pred_len
+        assert config.pred_len == 1
+
+        self.e_layers = config.e_layers
+        assert config.e_layers >= 1
+
+        self.enc_in = config.enc_in
+        self.d_model = config.d_model
+        self.c_out = config.c_out
+
+        # Time Embedding
+        self.app_time_emb = config.t_embed is not None
+        if self.app_time_emb:
+            if config.t_embed != "time2vec_app":
+                raise Exception(
+                    "The only options for t_embed with mlp are null and time2vec_app"
+                )
+            elif not (config.emb_t2v_app_dim > 0):
+                raise Exception("Need to specify a valid emb_t2v_app_dim")
+            self.enc_in += config.emb_t2v_app_dim
+            self.temporal_embedding = Time2Vec(
+                time_emb_dim=config.emb_t2v_app_dim, freq=config.freq
+            )
+
+        flattened_enc_in = self.seq_len * self.enc_in
+
+        if self.e_layers == 1:
+            layers = [nn.Linear(flattened_enc_in, self.c_out)]
+        else:
+            layers = [nn.Linear(flattened_enc_in, self.d_model), nn.GELU()]
+            for _ in range(self.e_layers - 2):
+                layers.append(nn.Dropout(config.dropout))
+                layers.append(nn.Linear(self.d_model, self.d_model))
+                layers.append(nn.GELU())
+
+            layers.append(nn.Linear(self.d_model, self.c_out))
+
+        self.model = nn.Sequential(*layers)
+
+    def forward(self, x, x_mark, *args):
+        # x: [Batch, Input length, Channel]
+        if self.app_time_emb:
+            time_emb = self.temporal_embedding(x_mark)
+            x = torch.concat([x, time_emb], dim=-1)
+
+        x_flat = x.reshape(x.shape[0], 1, -1)
+        return self.model(x_flat)

models/DLinear.py

@@ -0,0 +1,104 @@
+# Code is from https://github.com/cure-lab/LTSF-Linear
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+
+class moving_avg(nn.Module):
+    """
+    Moving average block to highlight the trend of time series
+    """
+
+    def __init__(self, kernel_size, stride):
+        super(moving_avg, self).__init__()
+        self.kernel_size = kernel_size
+        self.avg = nn.AvgPool1d(kernel_size=kernel_size, stride=stride, padding=0)
+
+    def forward(self, x):
+        # padding on the both ends of time series
+        front = x[:, 0:1, :].repeat(1, (self.kernel_size - 1) // 2, 1)
+        end = x[:, -1:, :].repeat(1, (self.kernel_size - 1) // 2, 1)
+        x = torch.cat([front, x, end], dim=1)
+        x = self.avg(x.permute(0, 2, 1))
+        x = x.permute(0, 2, 1)
+        return x
+
+
+class series_decomp(nn.Module):
+    """
+    Series decomposition block
+    """
+
+    def __init__(self, kernel_size):
+        super(series_decomp, self).__init__()
+        self.moving_avg = moving_avg(kernel_size, stride=1)
+
+    def forward(self, x):
+        moving_mean = self.moving_avg(x)
+        res = x - moving_mean
+        return res, moving_mean
+
+
+class Model(nn.Module):
+    """
+    Decomposition-Linear
+    """
+
+    def __init__(self, configs):
+        super(Model, self).__init__()
+        self.seq_len = configs.seq_len
+        self.pred_len = configs.pred_len
+
+        # Decompsition Kernel Size
+        kernel_size = 25
+        self.decompsition = series_decomp(kernel_size)
+        self.individual = configs.individual
+        self.channels = configs.enc_in
+
+        if self.individual:
+            self.Linear_Seasonal = nn.ModuleList()
+            self.Linear_Trend = nn.ModuleList()
+
+            for i in range(self.channels):
+                self.Linear_Seasonal.append(nn.Linear(self.seq_len, self.pred_len))
+                self.Linear_Trend.append(nn.Linear(self.seq_len, self.pred_len))
+
+                # Use this two lines if you want to visualize the weights
+                # self.Linear_Seasonal[i].weight = nn.Parameter((1/self.seq_len)*torch.ones([self.pred_len,self.seq_len]))
+                # self.Linear_Trend[i].weight = nn.Parameter((1/self.seq_len)*torch.ones([self.pred_len,self.seq_len]))
+        else:
+            self.Linear_Seasonal = nn.Linear(self.seq_len, self.pred_len)
+            self.Linear_Trend = nn.Linear(self.seq_len, self.pred_len)
+
+            # Use this two lines if you want to visualize the weights
+            # self.Linear_Seasonal.weight = nn.Parameter((1/self.seq_len)*torch.ones([self.pred_len,self.seq_len]))
+            # self.Linear_Trend.weight = nn.Parameter((1/self.seq_len)*torch.ones([self.pred_len,self.seq_len]))
+
+    def forward(self, x, *args):
+        # x: [Batch, Input length, Channel]
+        seasonal_init, trend_init = self.decompsition(x)
+        seasonal_init, trend_init = seasonal_init.permute(0, 2, 1), trend_init.permute(
+            0, 2, 1
+        )
+        if self.individual:
+            seasonal_output = torch.zeros(
+                [seasonal_init.size(0), seasonal_init.size(1), self.pred_len],
+                dtype=seasonal_init.dtype,
+            ).to(seasonal_init.device)
+            trend_output = torch.zeros(
+                [trend_init.size(0), trend_init.size(1), self.pred_len],
+                dtype=trend_init.dtype,
+            ).to(trend_init.device)
+            for i in range(self.channels):
+                seasonal_output[:, i, :] = self.Linear_Seasonal[i](
+                    seasonal_init[:, i, :]
+                )
+                trend_output[:, i, :] = self.Linear_Trend[i](trend_init[:, i, :])
+        else:
+            seasonal_output = self.Linear_Seasonal(seasonal_init)
+            trend_output = self.Linear_Trend(trend_init)
+
+        x = seasonal_output + trend_output
+        return x.permute(0, 2, 1)  # to [Batch, Output length, Channel]

models/Informer.py

@@ -0,0 +1,242 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from utils.masking import TriangularCausalMask, ProbMask
+from layers.encoder import Encoder, EncoderLayer, ConvLayer, EncoderStack
+from layers.decoder import Decoder, DecoderLayer
+from layers.attn import FullAttention, ProbAttention, AttentionLayer
+from layers.embed import DataEmbedding
+
+
+class Informer(nn.Module):
+    def __init__(self, config):
+        # enc_in, dec_in, c_out, seq_len, label_len, out_len,
+        #         factor=5, d_model=512, n_heads=8, e_layers=3, d_layers=2, d_ff=512,
+        #         dropout=0.0, attn='prob', t_embed='fixed', freq='h', activation='gelu',
+        #         output_attention = False, distil=True, mix=True,
+        #         device=torch.device('cuda:0')
+        super(Informer, self).__init__()
+        self.pred_len = config.pred_len
+        self.attn = config.attn
+        self.output_attention = config.output_attention
+
+        # Encoding
+        self.enc_embedding = DataEmbedding(
+            config.enc_in, config.d_model, config.t_embed, config.freq, config.dropout_emb
+        )
+        self.dec_embedding = DataEmbedding(
+            config.dec_in, config.d_model, config.t_embed, config.freq, config.dropout_emb
+        )
+        # Attention
+        Attn = ProbAttention if config.attn == "prob" else FullAttention
+        # Encoder
+        self.encoder = Encoder(
+            [
+                EncoderLayer(
+                    AttentionLayer(
+                        Attn(
+                            False,
+                            config.factor,
+                            attention_dropout=config.dropout,
+                            output_attention=config.output_attention,
+                        ),
+                        config.d_model,
+                        config.n_heads,
+                        mix=False,
+                    ),
+                    config.d_model,
+                    config.d_ff,
+                    dropout=config.dropout,
+                    activation=config.activation,
+                )
+                for l in range(config.e_layers)
+            ],
+            [ConvLayer(config.d_model) for l in range(config.e_layers - 1)]
+            if config.distil
+            else None,
+            norm_layer=torch.nn.LayerNorm(config.d_model),
+        )
+        # Decoder
+        self.decoder = Decoder(
+            [
+                DecoderLayer(
+                    AttentionLayer(
+                        Attn(
+                            True,
+                            config.factor,
+                            attention_dropout=config.dropout,
+                            output_attention=False,
+                        ),
+                        config.d_model,
+                        config.n_heads,
+                        mix=config.mix,
+                    ),
+                    AttentionLayer(
+                        FullAttention(
+                            False,
+                            config.factor,
+                            attention_dropout=config.dropout,
+                            output_attention=False,
+                        ),
+                        config.d_model,
+                        config.n_heads,
+                        mix=False,
+                    ),
+                    config.d_model,
+                    config.d_ff,
+                    dropout=config.dropout,
+                    activation=config.activation,
+                )
+                for l in range(config.d_layers)
+            ],
+            norm_layer=torch.nn.LayerNorm(config.d_model),
+        )
+        self.projection = nn.Linear(config.d_model, config.c_out, bias=True)
+
+    def forward(
+        self,
+        x_enc,
+        x_mark_enc,
+        x_dec,
+        x_mark_dec,
+        enc_self_mask=None,
+        dec_self_mask=None,
+        dec_enc_mask=None,
+    ):
+        enc_out = self.enc_embedding(x_enc, x_mark_enc)
+        enc_out, attns = self.encoder(enc_out, attn_mask=enc_self_mask)
+
+        dec_out = self.dec_embedding(x_dec, x_mark_dec)
+        dec_out = self.decoder(
+            dec_out, enc_out, x_mask=dec_self_mask, cross_mask=dec_enc_mask
+        )
+        dec_out = self.projection(dec_out)
+
+        if self.output_attention:
+            return dec_out[:, -self.pred_len :, :], attns
+        else:
+            return dec_out[:, -self.pred_len :, :]  # [B, L, D]
+
+
+class InformerStack(nn.Module):
+    def __init__(self, config):
+        # enc_in, dec_in, c_out, seq_len, label_len, out_len,
+        #         factor=5, d_model=512, n_heads=8, e_layers=[3,2,1], d_layers=2, d_ff=512,
+        #         dropout=0.0, attn='prob', t_embed='fixed', freq='h', activation='gelu',
+        #         output_attention = False, distil=True, mix=True,
+        #         device=torch.device('cuda:0'))
+        super(InformerStack, self).__init__()
+        self.pred_len = config.pred_len
+        self.attn = config.attn
+        self.output_attention = config.output_attention
+
+        assert (
+            type(config.e_layers) is list
+        ), "For Informer Stack e_layers must be a list"
+
+        # Encoding
+        self.enc_embedding = DataEmbedding(
+            config.enc_in, config.d_model, config.t_embed, config.freq, config.dropout_emb
+        )
+        self.dec_embedding = DataEmbedding(
+            config.dec_in, config.d_model, config.t_embed, config.freq, config.dropout_emb
+        )
+        # Attention
+        Attn = ProbAttention if config.attn == "prob" else FullAttention
+        # Encoder
+
+        inp_lens = list(
+            range(len(config.e_layers))
+        )  # [0,1,2,...] you can customize here
+        encoders = [
+            Encoder(
+                [
+                    EncoderLayer(
+                        AttentionLayer(
+                            Attn(
+                                False,
+                                config.factor,
+                                attention_dropout=config.dropout,
+                                output_attention=config.output_attention,
+                            ),
+                            config.d_model,
+                            config.n_heads,
+                            mix=False,
+                        ),
+                        config.d_model,
+                        config.d_ff,
+                        dropout=config.dropout,
+                        activation=config.activation,
+                    )
+                    for l in range(el)
+                ],
+                [ConvLayer(config.d_model) for l in range(el - 1)]
+                if config.distil
+                else None,
+                norm_layer=torch.nn.LayerNorm(config.d_model),
+            )
+            for el in config.e_layers
+        ]
+        self.encoder = EncoderStack(encoders, inp_lens)
+        # Decoder
+        self.decoder = Decoder(
+            [
+                DecoderLayer(
+                    AttentionLayer(
+                        Attn(
+                            True,
+                            config.factor,
+                            attention_dropout=config.dropout,
+                            output_attention=False,
+                        ),
+                        config.d_model,
+                        config.n_heads,
+                        mix=config.mix,
+                    ),
+                    AttentionLayer(
+                        FullAttention(
+                            False,
+                            config.factor,
+                            attention_dropout=config.dropout,
+                            output_attention=False,
+                        ),
+                        config.d_model,
+                        config.n_heads,
+                        mix=False,
+                    ),
+                    config.d_model,
+                    config.d_ff,
+                    dropout=config.dropout,
+                    activation=config.activation,
+                )
+                for l in range(config.d_layers)
+            ],
+            norm_layer=torch.nn.LayerNorm(config.d_model),
+        )
+
+        self.projection = nn.Linear(config.d_model, config.c_out, bias=True)
+
+    def forward(
+        self,
+        x_enc,
+        x_mark_enc,
+        x_dec,
+        x_mark_dec,
+        enc_self_mask=None,
+        dec_self_mask=None,
+        dec_enc_mask=None,
+    ):
+        enc_out = self.enc_embedding(x_enc, x_mark_enc)
+        enc_out, attns = self.encoder(enc_out, attn_mask=enc_self_mask)
+
+        dec_out = self.dec_embedding(x_dec, x_mark_dec)
+        dec_out = self.decoder(
+            dec_out, enc_out, x_mask=dec_self_mask, cross_mask=dec_enc_mask
+        )
+        dec_out = self.projection(dec_out)
+
+        if self.output_attention:
+            return dec_out[:, -self.pred_len :, :], attns
+        else:
+            return dec_out[:, -self.pred_len :, :]  # [B, L, D]

models/Lstm.py

@@ -0,0 +1,84 @@
+import torch
+from torch import nn
+
+from layers.embed import Time2Vec
+
+
+class LSTM(nn.Module):
+    def __init__(self, config):
+        super(LSTM, self).__init__()
+        assert config.pred_len == 1
+        assert config.label_len == 0
+        # Hidden dimensions
+        self.d_model = config.d_model
+
+        # Number of hidden layers
+        self.e_layers = config.e_layers
+
+        self.enc_in = config.enc_in
+
+        # Time Embedding
+
+        self.t_embed = config.t_embed
+        if self.t_embed is not None:
+            if config.t_embed == "time2vec_app":
+                if not (config.emb_t2v_app_dim > 0):
+                    raise Exception("Need to specify a valid emb_t2v_app_dim")
+                self.enc_in += config.emb_t2v_app_dim
+                self.temporal_embedding = Time2Vec(
+                    time_emb_dim=config.emb_t2v_app_dim, freq=config.freq
+                )
+            elif config.t_embed == "time2vec_add":
+                self.temporal_embedding = Time2Vec(
+                    time_emb_dim=self.enc_in, freq=config.freq
+                )
+            else:
+                raise Exception(
+                    "The only options for t_embed with mlp are null and time2vec_app"
+                )
+
+        # batch_first=True causes input/output tensors to be of shape
+        # (batch_dim, seq_dim, feature_dim)
+        self.lstm = nn.LSTM(
+            input_size=self.enc_in,
+            hidden_size=config.d_model,
+            num_layers=config.e_layers,
+            batch_first=True,
+            dropout=config.dropout,
+            bidirectional=False,
+        )
+
+        self.fc_1 = nn.Linear(config.d_model, config.d_ff)
+        self.relu = nn.ReLU()
+        # Readout layer
+        self.fc = nn.Linear(config.d_ff, config.c_out)
+
+    def forward(self, x, x_mark, *args, **kwargs):
+        if self.t_embed is not None:
+            if self.t_embed == "time2vec_app":
+                time_emb = self.temporal_embedding(x_mark)
+                x = torch.concat([x, time_emb], dim=-1)
+            elif self.t_embed == "time2vec_add":
+                time_emb = self.temporal_embedding(x_mark)
+                x = x + time_emb
+
+        # Initialize hidden state with zeros
+        h0 = torch.zeros(self.e_layers, x.size(0), self.d_model).to(x)
+
+        # Initialize cell state
+        c0 = torch.zeros(self.e_layers, x.size(0), self.d_model).to(x)
+
+        # We need to detach as we are doing truncated backpropagation through time (BPTT)
+        # If we don't, we'll backprop all the way to the start even after going through another batch
+        out, (hn, cn) = self.lstm(x, (h0, c0))
+
+        # Index hidden state of last time step
+        # out.size() --> 100, 32, 100
+        # out[:, -1, :] --> 100, 100 --> just want last time step hidden states!
+
+        # out = self.relu(self.fc_1(out[:, -1, :]))
+        out = self.relu(self.fc_1(self.relu(hn[-1])))
+
+        out = self.fc(out)
+        # out.size() --> 100, 10
+        return out[:, None]

models/Stockformer.py

@@ -0,0 +1,131 @@
+import os
+import torch
+import torch.nn as nn
+
+from layers.encoder import Encoder, EncoderLayer, ConvLayer
+from layers.attn import FullAttention, AttentionLayer, ProbAttention
+from layers.embed import DataEmbedding
+from utils.masking import QuestionMask
+
+
+class Stockformer(nn.Module):
+    def __init__(self, config):
+        super(Stockformer, self).__init__()
+        self.pred_len = config.pred_len
+        assert self.pred_len == 1, "Stockformer needs pred_len to be 1"
+        self.attn = config.attn
+        self.output_attention = config.output_attention
+
+        self.seq_len = config.seq_len
+
+        self.final_mode = config.final_mode
+
+        # Embedding
+        self.enc_embedding = DataEmbedding(
+            config.enc_in,
+            config.d_model,
+            config.t_embed,
+            config.freq,
+            config.dropout_emb,
+            emb_t2v_app_dim=config.emb_t2v_app_dim,
+            tok_emb=config.tok_emb,
+        )
+        # Attention
+        Attn = ProbAttention if config.attn == "prob" else FullAttention
+        # Encoder
+        self.encoder = Encoder(
+            [
+                EncoderLayer(
+                    AttentionLayer(
+                        Attn(
+                            True if config.final_mode == "mode3" else False,
+                            config.factor,
+                            attention_dropout=config.dropout,
+                            output_attention=config.output_attention,
+                        ),
+                        config.d_model,
+                        config.n_heads,
+                        mix=False,
+                    ),
+                    config.d_model,
+                    config.d_ff,
+                    dropout=config.dropout,
+                    activation=config.activation,
+                    ln_mode=config.ln_mode,
+                )
+                for l in range(config.e_layers)
+            ],
+            [ConvLayer(config.d_model) for l in range(config.e_layers - 1)]
+            if config.distil
+            else None,
+            #  norm_layer=torch.nn.LayerNorm(config.d_model),
+            norm_layer=torch.nn.RMSNorm(config.d_model),
+        )
+
+        if config.final_mode == "mode1":
+            self.final = nn.Linear(
+                config.d_model * config.seq_len, config.c_out, bias=True
+            )
+        elif config.final_mode == "mode2" or config.final_mode == "mode3":
+            self.final = nn.Linear(config.d_model, config.c_out, bias=True)
+        else:
+            raise Exception(f"Invalid final_mode: {config.final_mode}")
+        # nn.init.xavier_normal_(self.final.weight, gain=nn.init.calculate_gain("tanh"))
+
+        # self.final = nn.Sequential(*[
+        #     nn.Linear(config.d_model * config.seq_len, config.d_model * 4, bias=True),
+        #     nn.GELU(),
+        #     nn.Linear(config.d_model * 4, config.c_out, bias=True)
+        # ])
+
+        # Load pre-trained model
+        if config.load_model_path is not None:
+            path = os.path.join(config.checkpoints, config.load_model_path)
+            print(f"Loading Model from {path}")
+            self.load_state_dict(torch.load(path))
+
+    def forward(
+        self,
+        x_enc,
+        x_mark_enc,
+        x_dec,
+        x_mark_dec,
+        enc_self_mask=None,
+        dec_self_mask=None,
+        dec_enc_mask=None,
+        pre_train=False,
+    ):
+        # x_enc is (batch_size / num gpus, seq_len, enc_in)
+        # x_mark_enc is (batch_size / num gpus, seq_len, date-representation (7forhours)
+        assert len(x_enc.shape) == 3
+        assert x_enc.shape[1] == self.seq_len
+
+        if self.final_mode == "mode3":
+            # This gives the encoder a question input as the last token
+            # TODO: Maybe this should be initialized differently, like to the mean of x_enc, random, mean of dataset?
+            zeros = torch.zeros([x_enc.shape[0], 1, x_enc.shape[2]]).to(x_enc)
+            x_enc = torch.cat([x_enc, zeros], 1)
+            x_mark_enc = torch.cat([x_mark_enc, x_mark_dec], 1)
+            assert enc_self_mask is None
+            enc_self_mask = QuestionMask(
+                x_enc.shape[0], x_enc.shape[1], device=x_enc.device
+            )
+
+        # emb_out is (batch_size / num gpus, seq_len, d_model)
+        emb_out = self.enc_embedding(x_enc, x_mark_enc)
+
+        # enc_out is (batch_size / num gpus, seq_len, d_model) but seq_len will change if distil
+        enc_out, attns = self.encoder(emb_out, attn_mask=enc_self_mask)
+
+        if self.final_mode == "mode1":
+            out = self.final(enc_out.flatten(start_dim=1))
+        elif self.final_mode == "mode2" or self.final_mode == "mode3":
+            out = self.final(enc_out[:, -1, :])
+        else:
+            assert False, f"Forward missing valid final mode {self.final_mode}"
+
+        # The None below is just adding a dummy dimension
+        if self.output_attention:
+            return out[:, None, :], attns
+        else:
+            return out[:, None, :]  # (batch_size, 1, c_out)

old_stuff/Dockerfile

@@ -0,0 +1,8 @@
+FROM continuumio/miniconda3:4.7.12
+
+ADD ./environment.yml ./environment.yml
+
+RUN conda install -n base -c conda-forge mamba && \
+    mamba env update -n base -f ./environment.yml && \
+    conda clean -afy
+

old_stuff/Informer.ipynb

@@ -0,0 +1,698 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "5IM6CZzW_CH0"
+   },
+   "source": [
+    "# Informer Demo"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "b5GFng7v7Eq0"
+   },
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "\n",
+    "# if not 'Informer2020' in sys.path:\n",
+    "#     sys.path += ['Informer2020']"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "rIjZdN5e_SWe"
+   },
+   "source": [
+    "## Experiments: Train and Test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "RPdt-Kwc_RRZ"
+   },
+   "outputs": [],
+   "source": [
+    "from utils.tools import dotdict\n",
+    "from exp.exp_informer import Exp_Informer\n",
+    "import torch\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import os\n",
+    "from utils.ipynb_helpers import args_from_setting, setting_from_args, handle_gpu"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "6mx2dnwY9dWi"
+   },
+   "outputs": [],
+   "source": [
+    "args = dotdict()\n",
+    "args.des = \"full_1h\"\n",
+    "\n",
+    "args.model = \"informer\"  # model of experiment, options: [informer, informerstack, informerlight(TBD)]\n",
+    "\n",
+    "args.data = \"custom\"  # data\n",
+    "args.root_path = \"./data/stock/\"  # root path of data file\n",
+    "\n",
+    "\n",
+    "args.data_path = \"full_1h.csv\"  # data file\n",
+    "args.features = \"MS\"  # forecasting task, options:[M, S, MS]; M:multivariate predict multivariate, S:univariate predict univariate, MS:multivariate predict univariate\n",
+    "args.target = \"XOM_pctchange\"  # target feature in S or MS task\n",
+    "args.freq = \"h\"  # freq for time features encoding, options:[s:secondly, t:minutely, h:hourly, d:daily, b:business days, w:weekly, m:monthly], you can also use more detailed freq like 15min or 3h\n",
+    "args.checkpoints = \"./checkpoints\"  # location of model checkpoints\n",
+    "\n",
+    "args.seq_len = 16  # input sequence length of Informer encoder\n",
+    "args.label_len = 4  # start token length of Informer decoder\n",
+    "args.pred_len = 1  # prediction sequence length\n",
+    "# Informer decoder input: concat[start token series(label_len), zero padding series(pred_len)]\n",
+    "\n",
+    "args.cols = [\n",
+    "    \"XOM_open\",\n",
+    "    \"XOM_high\",\n",
+    "    \"XOM_low\",\n",
+    "    \"XOM_close\",\n",
+    "    \"XOM_volume\",\n",
+    "    \"XOM_pctchange\",\n",
+    "    \"XOM_shortsma\",\n",
+    "]  # [\"XOM_close\", \"BP_close\", \"CVX_close\", \"WTI_close\"]\n",
+    "args.enc_in = 7  # 13 # encoder input size\n",
+    "args.dec_in = 7  # 13 # decoder input size\n",
+    "args.c_out = 1  # output size\n",
+    "args.factor = 5  # probsparse attn factor\n",
+    "args.d_model = 64  # 512 # dimension of model\n",
+    "args.n_heads = 8  # num of heads\n",
+    "args.e_layers = 4  # 2 # num of encoder layers\n",
+    "args.d_layers = 2  # 1 # num of decoder layers\n",
+    "args.d_ff = 2048  # dimension of fcn in model\n",
+    "args.dropout = 0.05  # dropout\n",
+    "args.attn = \"prob\"  # attention used in encoder, options:[prob, full]\n",
+    "args.t_embed = \"timeF\"  # time features encoding, options:[timeF, fixed, learned]\n",
+    "args.activation = \"gelu\"  # activation\n",
+    "args.distil = True  # whether to use distilling in encoder\n",
+    "args.output_attention = False  # whether to output attention in encoder\n",
+    "args.mix = True\n",
+    "args.padding = 0\n",
+    "\n",
+    "args.batch_size = 64\n",
+    "args.learning_rate = 0.00001\n",
+    "args.loss = \"mse\"\n",
+    "args.lradj = \"type1\"\n",
+    "args.use_amp = False  # whether to use automatic mixed precision training\n",
+    "\n",
+    "args.num_workers = 0\n",
+    "args.itr = 1  # number of runs\n",
+    "args.max_epochs = 15\n",
+    "args.patience = 3\n",
+    "\n",
+    "\n",
+    "args.scale = True  # True # True\n",
+    "args.inverse = True  # True # Defaultly False but @Zac thinks it should be True\n",
+    "\n",
+    "\n",
+    "args.date_start = None  # \"2021-01-01\"\n",
+    "args.date_end = None\n",
+    "args.date_test = \"2022-04-01\"  # None\n",
+    "\n",
+    "handle_gpu(args, None)\n",
+    "\n",
+    "# idk what this is for\n",
+    "args.detail_freq = args.freq\n",
+    "args.freq = args.freq[-1:]\n",
+    "\n",
+    "print(\"Args in experiment:\")\n",
+    "print(args)\n",
+    "Exp = Exp_Informer"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Train & Test *args.itr* models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "928tzaA2AA2g",
+    "outputId": "c19f673a-02d1-4f4d-91c3-d0f25e600443"
+   },
+   "outputs": [],
+   "source": [
+    "exp = None\n",
+    "setting = None\n",
+    "for ii in range(args.itr):\n",
+    "    # setting record of experiments\n",
+    "    setting = setting_from_args(args, ii)\n",
+    "\n",
+    "    # set experiments\n",
+    "    exp = Exp(args)\n",
+    "\n",
+    "    # train\n",
+    "    print(f\">>>>>>>start training : {setting}>>>>>>>>>>>>>>>>>>>>>>>>>>\")\n",
+    "    exp.train(setting)\n",
+    "\n",
+    "    # test\n",
+    "    print(f\">>>>>>>testing : {setting}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\")\n",
+    "    exp.test(setting)\n",
+    "\n",
+    "    torch.cuda.empty_cache()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "CDHF-HerAE3u"
+   },
+   "source": [
+    "## Prediction"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "nTkluNNcyMJt",
+    "outputId": "780767fe-6321-4081-e827-6701daeb375b"
+   },
+   "outputs": [],
+   "source": [
+    "# If you already have a trained model, you can set the arguments and model path, then initialize a Experiment and use it to predict\n",
+    "# Prediction is a sequence which is adjacent to the last date of the data, and does not exist in the data\n",
+    "# If you want to get more information about prediction, you can refer to code `exp/exp_informer.py function predict()` and `data/data_loader.py class Dataset_Pred`\n",
+    "\n",
+    "# args = dotdict(model='informer', data='WTH', root_path='./data/ETT/', data_path='WTH.csv', features='M', target='WetBulbCelsius', freq='h', checkpoints='./checkpoints/', seq_len=96, label_len=48, pred_len=24, enc_in=12, dec_in=12, c_out=12, d_model=512, n_heads=8, e_layers=2, d_layers=1, s_layers=[3, 2, 1], d_ff=2048, factor=5, padding=0, distil=True, dropout=0.05, attn='prob', t_embed='timeF', activation='gelu', output_attention=False, do_predict=False, mix=True, cols=None, num_workers=0, itr=2, max_epochs=6, batch_size=32, patience=3, learning_rate=0.0001, des='test', loss='mse', lradj='type1', use_amp=False, inverse=False, use_gpu=True, gpu=0, use_multi_gpu=False, devices='0,1,2,3', detail_freq='h')\n",
+    "\n",
+    "manual = False\n",
+    "\n",
+    "if manual:\n",
+    "    setting = \"informer_custom_ftMS_sl256_ll64_pl16_ei1_di1_co1_iFalse_dm512_nh8_el2_dl1_df2048_atprob_fc5_ebtimeF_dtTrue_mxTrue_exp_0\"\n",
+    "    args = args_from_setting(setting, args)\n",
+    "\n",
+    "    exp = Exp(args)\n",
+    "\n",
+    "path = os.path.join(args.checkpoints, setting, \"checkpoint.pth\")\n",
+    "\n",
+    "exp.predict(setting, True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "KBCPbjGuzAZb",
+    "outputId": "945dc447-88e8-4b08-b7e5-f0a0b486d138"
+   },
+   "outputs": [],
+   "source": [
+    "# the prediction will be saved in ./results/{setting}/real_prediction.npy\n",
+    "\n",
+    "prediction = np.load(f\"./results/{setting}/real_prediction.npy\")\n",
+    "\n",
+    "prediction.shape"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "5yFuVkTV30_j"
+   },
+   "source": [
+    "### More details about Prediction - prediction function"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "Sv9AR_Aw030r"
+   },
+   "outputs": [],
+   "source": [
+    "# here is the detailed code of function predict\n",
+    "\n",
+    "\n",
+    "def predict(exp, setting, load=False):\n",
+    "    pred_data, pred_loader = exp._get_data(flag=\"pred\")\n",
+    "\n",
+    "    if load:\n",
+    "        path = os.path.join(exp.args.checkpoints, setting)\n",
+    "        best_model_path = path + \"/\" + \"checkpoint.pth\"\n",
+    "        exp.model.load_state_dict(torch.load(best_model_path))\n",
+    "\n",
+    "    exp.model.eval()\n",
+    "\n",
+    "    preds = []\n",
+    "\n",
+    "    for i, (batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(pred_loader):\n",
+    "        batch_x = batch_x.float().to(exp.device)\n",
+    "        batch_y = batch_y.float()\n",
+    "        batch_x_mark = batch_x_mark.float().to(exp.device)\n",
+    "        batch_y_mark = batch_y_mark.float().to(exp.device)\n",
+    "\n",
+    "        # decoder input\n",
+    "        if exp.args.padding == 0:\n",
+    "            dec_inp = torch.zeros(\n",
+    "                [batch_y.shape[0], exp.args.pred_len, batch_y.shape[-1]]\n",
+    "            ).float()\n",
+    "        elif exp.args.padding == 1:\n",
+    "            dec_inp = torch.ones(\n",
+    "                [batch_y.shape[0], exp.args.pred_len, batch_y.shape[-1]]\n",
+    "            ).float()\n",
+    "        else:\n",
+    "            dec_inp = torch.zeros(\n",
+    "                [batch_y.shape[0], exp.args.pred_len, batch_y.shape[-1]]\n",
+    "            ).float()\n",
+    "        dec_inp = (\n",
+    "            torch.cat([batch_y[:, : exp.args.label_len, :], dec_inp], dim=1)\n",
+    "            .float()\n",
+    "            .to(exp.device)\n",
+    "        )\n",
+    "        # encoder - decoder\n",
+    "        if exp.args.use_amp:\n",
+    "            with torch.cuda.amp.autocast():\n",
+    "                if exp.args.output_attention:\n",
+    "                    outputs = exp.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)[0]\n",
+    "                else:\n",
+    "                    outputs = exp.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)\n",
+    "        else:\n",
+    "            if exp.args.output_attention:\n",
+    "                outputs = exp.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)[0]\n",
+    "            else:\n",
+    "                outputs = exp.model(batch_x, batch_x_mark, dec_inp, batch_y_mark)\n",
+    "        f_dim = -1 if exp.args.features == \"MS\" else 0\n",
+    "        batch_y = batch_y[:, -exp.args.pred_len :, f_dim:].to(exp.device)\n",
+    "\n",
+    "        pred = outputs.detach().cpu().numpy()  # .squeeze()\n",
+    "\n",
+    "        preds.append(pred)\n",
+    "\n",
+    "    preds = np.array(preds)\n",
+    "    preds = preds.reshape(-1, preds.shape[-2], preds.shape[-1])\n",
+    "\n",
+    "    # result save\n",
+    "    folder_path = \"./results/\" + setting + \"/\"\n",
+    "    if not os.path.exists(folder_path):\n",
+    "        os.makedirs(folder_path)\n",
+    "\n",
+    "    np.save(folder_path + \"real_prediction.npy\", preds)\n",
+    "\n",
+    "    return preds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "tVLWZL2a1pwB",
+    "outputId": "421e9ae1-f024-42b6-c8cb-ed1d38c864cd"
+   },
+   "outputs": [],
+   "source": [
+    "# you can also use this prediction function to get result\n",
+    "prediction = predict(exp, setting, True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 269
+    },
+    "id": "NwtZmQC71uc8",
+    "outputId": "eec9d116-f122-42d9-8e02-c893ff764db0"
+   },
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "plt.figure()\n",
+    "plt.plot(prediction[0, :, -1])\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "EnePVyrW4I14"
+   },
+   "source": [
+    "### More details about Prediction - prediction dataset\n",
+    "\n",
+    "You can give a `root_path` and `data_path` of the data you want to forecast, and set `seq_len`, `label_len`, `pred_len` and other arguments as other Dataset. The difference is that you can set a more detailed freq such as `15min` or `3h` to generate the timestamp of prediction series.\n",
+    "\n",
+    "`Dataset_Pred` only has one sample (including `encoder_input: [1, seq_len, dim]`, `decoder_token: [1, label_len, dim]`, `encoder_input_timestamp: [1, seq_len, date_dim]`, `decoder_input_timstamp: [1, label_len+pred_len, date_dim]`). It will intercept the last sequence of the given data (seq_len data) to forecast the unseen future sequence (pred_len data)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "id": "ZpXhNGp34Hf4"
+   },
+   "outputs": [],
+   "source": [
+    "from data_provider.data_loader import Dataset_Pred\n",
+    "from torch.utils.data import DataLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "j4Rpd1q74T8N"
+   },
+   "outputs": [],
+   "source": [
+    "Data = Dataset_Pred\n",
+    "timeenc = 0 if args.t_embed != \"timeF\" else 1\n",
+    "flag = \"pred\"\n",
+    "shuffle_flag = False\n",
+    "drop_last = False\n",
+    "batch_size = 1\n",
+    "\n",
+    "freq = args.detail_freq\n",
+    "\n",
+    "data_set = Data(args, flag=flag, freq=freq, timeenc=timeenc)\n",
+    "\n",
+    "data_loader = DataLoader(\n",
+    "    data_set,\n",
+    "    batch_size=batch_size,\n",
+    "    shuffle=shuffle_flag,\n",
+    "    num_workers=args.num_workers,\n",
+    "    drop_last=drop_last,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "42C84BfY6UPV",
+    "outputId": "f5ccc428-db92-4708-e104-f5d29aa5adf9"
+   },
+   "outputs": [],
+   "source": [
+    "len(data_set), len(data_loader)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "cNhEP_7sAgqC"
+   },
+   "source": [
+    "## Visualization"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "vMRk8VkQ2Iko",
+    "outputId": "bbf3cd10-7294-472d-e330-21e00f20963a"
+   },
+   "outputs": [],
+   "source": [
+    "# When we finished exp.train(setting) and exp.test(setting), we will get a trained model and the results of test experiment\n",
+    "# The results of test experiment will be saved in ./results/{setting}/pred.npy (prediction of test dataset) and ./results/{setting}/true.npy (groundtruth of test dataset)\n",
+    "\n",
+    "preds = np.load(f\"./results/{setting}/pred.npy\")\n",
+    "trues = np.load(f\"./results/{setting}/true.npy\")\n",
+    "\n",
+    "# [samples, pred_len, dimensions]\n",
+    "preds.shape, trues.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "ZEGhDOmxAeAb"
+   },
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 265
+    },
+    "id": "kyPuOPGAAjl3",
+    "outputId": "8554f6f8-c13a-43e1-b04b-5f27823445d0"
+   },
+   "outputs": [],
+   "source": [
+    "# draw OT prediction\n",
+    "plt.figure()\n",
+    "plt.plot(trues[0, :, -1], label=\"GroundTruth\")\n",
+    "plt.plot(preds[0, :, -1], label=\"Prediction\")\n",
+    "plt.legend()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(trues.shape)\n",
+    "print(preds.shape)\n",
+    "MSE = np.square(np.subtract(trues, preds)).mean()\n",
+    "RMSE = np.sqrt(MSE)\n",
+    "\n",
+    "print(\"against preds\", MSE, RMSE)\n",
+    "\n",
+    "\n",
+    "MSE = np.square(np.subtract(preds, np.zeros(preds.shape))).mean()\n",
+    "RMSE = np.sqrt(MSE)\n",
+    "print(\"against 0s\", MSE, RMSE)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 265
+    },
+    "id": "43MIgWfpMYIB",
+    "outputId": "327f64b7-363c-44f9-c7c8-1f654911068c"
+   },
+   "outputs": [],
+   "source": [
+    "# draw HUFL prediction\n",
+    "plt.figure()\n",
+    "plt.plot(trues[0, :, 0], label=\"GroundTruth\")\n",
+    "plt.plot(preds[0, :, 0], label=\"Prediction\")\n",
+    "plt.legend()\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "hKmqhCfmt0xd"
+   },
+   "outputs": [],
+   "source": [
+    "from data_provider.data_loader import Dataset_Custom\n",
+    "from torch.utils.data import DataLoader\n",
+    "\n",
+    "Data = Dataset_Custom\n",
+    "timeenc = 0 if args.t_embed != \"timeF\" else 1\n",
+    "flag = \"test\"\n",
+    "shuffle_flag = False\n",
+    "drop_last = True\n",
+    "batch_size = 1\n",
+    "data_set = Data(args, flag=flag, freq=freq, timeenc=timeenc)\n",
+    "\n",
+    "data_loader = DataLoader(\n",
+    "    data_set,\n",
+    "    batch_size=batch_size,\n",
+    "    shuffle=shuffle_flag,\n",
+    "    num_workers=args.num_workers,\n",
+    "    drop_last=drop_last,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "iflTTl0quCoK",
+    "outputId": "3708fc91-517e-4c83-e133-059381bde271"
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "args.output_attention = True\n",
+    "\n",
+    "exp = Exp(args)\n",
+    "\n",
+    "model = exp.model\n",
+    "\n",
+    "path = os.path.join(args.checkpoints, setting, \"checkpoint.pth\")\n",
+    "\n",
+    "print(model.load_state_dict(torch.load(path)))\n",
+    "\n",
+    "df = pd.read_csv(os.path.join(args.root_path, args.data_path))\n",
+    "df[args.cols].head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Attention Visualization"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "lDdzqm9HAk2C"
+   },
+   "outputs": [],
+   "source": [
+    "idx = 0\n",
+    "for i, (batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(data_loader):\n",
+    "    if i != idx:\n",
+    "        continue\n",
+    "    batch_x = batch_x.float().to(exp.device)\n",
+    "    batch_y = batch_y.float()\n",
+    "\n",
+    "    batch_x_mark = batch_x_mark.float().to(exp.device)\n",
+    "    batch_y_mark = batch_y_mark.float().to(exp.device)\n",
+    "\n",
+    "    dec_inp = torch.zeros_like(batch_y[:, -args.pred_len :, :]).float()\n",
+    "    dec_inp = (\n",
+    "        torch.cat([batch_y[:, : args.label_len, :], dec_inp], dim=1)\n",
+    "        .float()\n",
+    "        .to(exp.device)\n",
+    "    )\n",
+    "\n",
+    "    outputs, attn = model(batch_x, batch_x_mark, dec_inp, batch_y_mark)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "hWef23vWAmUz",
+    "outputId": "021eca83-e12f-402c-c87e-4fffa643d2f1"
+   },
+   "outputs": [],
+   "source": [
+    "attn[0].shape, attn[1].shape  # , attn[2].shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/",
+     "height": 1000
+    },
+    "id": "iZDH1fZgAnrl",
+    "outputId": "991cae95-04a2-402d-f179-777e962f46fe"
+   },
+   "outputs": [],
+   "source": [
+    "layers = [0, 1]\n",
+    "distil = \"Distil\" if args.distil else \"NoDistil\"\n",
+    "for layer in layers:\n",
+    "    print(\"\\n\\n==========================\")\n",
+    "    print(\"Showing attention layer\", layer)\n",
+    "    print(\"==========================\\n\\n\")\n",
+    "    for h in range(0, args.n_heads):\n",
+    "        plt.figure(figsize=[10, 8])\n",
+    "        plt.title(f\"Informer, {distil}, attn:{args.attn} layer:{layer} head:{h}\")\n",
+    "        A = attn[layer][0, h].detach().cpu().numpy()\n",
+    "        ax = sns.heatmap(A, vmin=0, vmax=A.max() + 0.01)\n",
+    "        plt.show()"
+   ]
+  }
+ ],
+ "metadata": {
+  "accelerator": "GPU",
+  "colab": {
+   "collapsed_sections": [],
+   "provenance": [],
+   "toc_visible": true
+  },
+  "kernelspec": {
+   "display_name": "former",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6 (main, Oct 24 2022, 16:07:47) [GCC 11.2.0]"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "44e5710a47a66ec240c2a0834fd7c20e15c61536e70be6891d892a39679ad994"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

old_stuff/Makefile

@@ -0,0 +1,38 @@
+IMAGE := informer
+ROOT := $(shell dirname $(realpath $(firstword ${MAKEFILE_LIST})))
+PARENT_ROOT := $(shell dirname ${ROOT})
+PORT := 8888
+
+DOCKER_PARAMETERS := \
+	--user $(shell id -u) \
+	-v ${ROOT}:/app \
+	-w /app \
+	-e HOME=/tmp
+
+init:
+	docker build -t ${IMAGE} .
+
+dataset:
+	mkdir -p data/ETT && \
+		wget -O data/ETT/ETTh1.csv https://raw.githubusercontent.com/zhouhaoyi/ETDataset/main/ETT-small/ETTh1.csv && \
+		wget -O data/ETT/ETTh2.csv https://raw.githubusercontent.com/zhouhaoyi/ETDataset/main/ETT-small/ETTh2.csv && \
+		wget -O data/ETT/ETTm1.csv https://raw.githubusercontent.com/zhouhaoyi/ETDataset/main/ETT-small/ETTm1.csv && \
+		wget -O data/ETT/ETTm2.csv https://raw.githubusercontent.com/zhouhaoyi/ETDataset/main/ETT-small/ETTm2.csv && \
+		wget -O data/ETT/ECL.csv "https://drive.google.com/uc?export=download&id=1rUPdR7R2iWFW-LMoDdHoO2g4KgnkpFzP" && \
+		wget -O data/ETT/WTH.csv "https://drive.google.com/uc?export=download&id=1UBRz-aM_57i_KCC-iaSWoKDPTGGv6EaG"
+
+jupyter:
+	docker run -d --rm ${DOCKER_PARAMETERS} -e HOME=/tmp -p ${PORT}:8888 ${IMAGE} \
+		bash -c "jupyter lab --ip=0.0.0.0 --no-browser --NotebookApp.token=''"
+
+run_module: .require-module
+	docker run -i --rm ${DOCKER_PARAMETERS} \
+		${IMAGE} ${module}
+
+bash_docker:
+	docker run -it --rm ${DOCKER_PARAMETERS} ${IMAGE}
+
+.require-module:
+ifndef module
+	$(error module is required)
+endif