LeNEPA trained on a custom CauKer-2M dataset with 5000 points per series

.gitattributes

@@ -1,35 +1,2 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+

LICENSE

@@ -0,0 +1,399 @@
+Attribution-NonCommercial 4.0 International
+
+=======================================================================
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+     Considerations for licensors: Our public licenses are
+     intended for use by those authorized to give the public
+     permission to use material in ways otherwise restricted by
+     copyright and certain other rights. Our licenses are
+     irrevocable. Licensors should read and understand the terms
+     and conditions of the license they choose before applying it.
+     Licensors should also secure all rights necessary before
+     applying our licenses so that the public can reuse the
+     material as expected. Licensors should clearly mark any
+     material not subject to the license. This includes other CC-
+     licensed material, or material used under an exception or
+     limitation to copyright. More considerations for licensors:
+	wiki.creativecommons.org/Considerations_for_licensors
+
+     Considerations for the public: By using one of our public
+     licenses, a licensor grants the public permission to use the
+     licensed material under specified terms and conditions. If
+     the licensor's permission is not necessary for any reason--for
+     example, because of any applicable exception or limitation to
+     copyright--then that use is not regulated by the license. Our
+     licenses grant only permissions under copyright and certain
+     other rights that a licensor has authority to grant. Use of
+     the licensed material may still be restricted for other
+     reasons, including because others have copyright or other
+     rights in the material. A licensor may make special requests,
+     such as asking that all changes be marked or described.
+     Although not required by our licenses, you are encouraged to
+     respect those requests where reasonable. More_considerations
+     for the public: 
+	wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution-NonCommercial 4.0 International Public
+License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial 4.0 International Public License ("Public
+License"). To the extent this Public License may be interpreted as a
+contract, You are granted the Licensed Rights in consideration of Your
+acceptance of these terms and conditions, and the Licensor grants You
+such rights in consideration of benefits the Licensor receives from
+making the Licensed Material available under these terms and
+conditions.
+
+Section 1 -- Definitions.
+
+  a. Adapted Material means material subject to Copyright and Similar
+     Rights that is derived from or based upon the Licensed Material
+     and in which the Licensed Material is translated, altered,
+     arranged, transformed, or otherwise modified in a manner requiring
+     permission under the Copyright and Similar Rights held by the
+     Licensor. For purposes of this Public License, where the Licensed
+     Material is a musical work, performance, or sound recording,
+     Adapted Material is always produced where the Licensed Material is
+     synched in timed relation with a moving image.
+
+  b. Adapter's License means the license You apply to Your Copyright
+     and Similar Rights in Your contributions to Adapted Material in
+     accordance with the terms and conditions of this Public License.
+
+  c. Copyright and Similar Rights means copyright and/or similar rights
+     closely related to copyright including, without limitation,
+     performance, broadcast, sound recording, and Sui Generis Database
+     Rights, without regard to how the rights are labeled or
+     categorized. For purposes of this Public License, the rights
+     specified in Section 2(b)(1)-(2) are not Copyright and Similar
+     Rights.
+  d. Effective Technological Measures means those measures that, in the
+     absence of proper authority, may not be circumvented under laws
+     fulfilling obligations under Article 11 of the WIPO Copyright
+     Treaty adopted on December 20, 1996, and/or similar international
+     agreements.
+
+  e. Exceptions and Limitations means fair use, fair dealing, and/or
+     any other exception or limitation to Copyright and Similar Rights
+     that applies to Your use of the Licensed Material.
+
+  f. Licensed Material means the artistic or literary work, database,
+     or other material to which the Licensor applied this Public
+     License.
+
+  g. Licensed Rights means the rights granted to You subject to the
+     terms and conditions of this Public License, which are limited to
+     all Copyright and Similar Rights that apply to Your use of the
+     Licensed Material and that the Licensor has authority to license.
+
+  h. Licensor means the individual(s) or entity(ies) granting rights
+     under this Public License.
+
+  i. NonCommercial means not primarily intended for or directed towards
+     commercial advantage or monetary compensation. For purposes of
+     this Public License, the exchange of the Licensed Material for
+     other material subject to Copyright and Similar Rights by digital
+     file-sharing or similar means is NonCommercial provided there is
+     no payment of monetary compensation in connection with the
+     exchange.
+
+  j. Share means to provide material to the public by any means or
+     process that requires permission under the Licensed Rights, such
+     as reproduction, public display, public performance, distribution,
+     dissemination, communication, or importation, and to make material
+     available to the public including in ways that members of the
+     public may access the material from a place and at a time
+     individually chosen by them.
+
+  k. Sui Generis Database Rights means rights other than copyright
+     resulting from Directive 96/9/EC of the European Parliament and of
+     the Council of 11 March 1996 on the legal protection of databases,
+     as amended and/or succeeded, as well as other essentially
+     equivalent rights anywhere in the world.
+
+  l. You means the individual or entity exercising the Licensed Rights
+     under this Public License. Your has a corresponding meaning.
+
+Section 2 -- Scope.
+
+  a. License grant.
+
+       1. Subject to the terms and conditions of this Public License,
+          the Licensor hereby grants You a worldwide, royalty-free,
+          non-sublicensable, non-exclusive, irrevocable license to
+          exercise the Licensed Rights in the Licensed Material to:
+
+            a. reproduce and Share the Licensed Material, in whole or
+               in part, for NonCommercial purposes only; and
+
+            b. produce, reproduce, and Share Adapted Material for
+               NonCommercial purposes only.
+
+       2. Exceptions and Limitations. For the avoidance of doubt, where
+          Exceptions and Limitations apply to Your use, this Public
+          License does not apply, and You do not need to comply with
+          its terms and conditions.
+
+       3. Term. The term of this Public License is specified in Section
+          6(a).
+
+       4. Media and formats; technical modifications allowed. The
+          Licensor authorizes You to exercise the Licensed Rights in
+          all media and formats whether now known or hereafter created,
+          and to make technical modifications necessary to do so. The
+          Licensor waives and/or agrees not to assert any right or
+          authority to forbid You from making technical modifications
+          necessary to exercise the Licensed Rights, including
+          technical modifications necessary to circumvent Effective
+          Technological Measures. For purposes of this Public License,
+          simply making modifications authorized by this Section 2(a)
+          (4) never produces Adapted Material.
+
+       5. Downstream recipients.
+
+            a. Offer from the Licensor -- Licensed Material. Every
+               recipient of the Licensed Material automatically
+               receives an offer from the Licensor to exercise the
+               Licensed Rights under the terms and conditions of this
+               Public License.
+
+            b. No downstream restrictions. You may not offer or impose
+               any additional or different terms or conditions on, or
+               apply any Effective Technological Measures to, the
+               Licensed Material if doing so restricts exercise of the
+               Licensed Rights by any recipient of the Licensed
+               Material.
+
+       6. No endorsement. Nothing in this Public License constitutes or
+          may be construed as permission to assert or imply that You
+          are, or that Your use of the Licensed Material is, connected
+          with, or sponsored, endorsed, or granted official status by,
+          the Licensor or others designated to receive attribution as
+          provided in Section 3(a)(1)(A)(i).
+
+  b. Other rights.
+
+       1. Moral rights, such as the right of integrity, are not
+          licensed under this Public License, nor are publicity,
+          privacy, and/or other similar personality rights; however, to
+          the extent possible, the Licensor waives and/or agrees not to
+          assert any such rights held by the Licensor to the limited
+          extent necessary to allow You to exercise the Licensed
+          Rights, but not otherwise.
+
+       2. Patent and trademark rights are not licensed under this
+          Public License.
+
+       3. To the extent possible, the Licensor waives any right to
+          collect royalties from You for the exercise of the Licensed
+          Rights, whether directly or through a collecting society
+          under any voluntary or waivable statutory or compulsory
+          licensing scheme. In all other cases the Licensor expressly
+          reserves any right to collect such royalties, including when
+          the Licensed Material is used other than for NonCommercial
+          purposes.
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+  a. Attribution.
+
+       1. If You Share the Licensed Material (including in modified
+          form), You must:
+
+            a. retain the following if it is supplied by the Licensor
+               with the Licensed Material:
+
+                 i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if
+                    designated);
+
+                ii. a copyright notice;
+
+               iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of
+                    warranties;
+
+                 v. a URI or hyperlink to the Licensed Material to the
+                    extent reasonably practicable;
+
+            b. indicate if You modified the Licensed Material and
+               retain an indication of any previous modifications; and
+
+            c. indicate the Licensed Material is licensed under this
+               Public License, and include the text of, or the URI or
+               hyperlink to, this Public License.
+
+       2. You may satisfy the conditions in Section 3(a)(1) in any
+          reasonable manner based on the medium, means, and context in
+          which You Share the Licensed Material. For example, it may be
+          reasonable to satisfy the conditions by providing a URI or
+          hyperlink to a resource that includes the required
+          information.
+
+       3. If requested by the Licensor, You must remove any of the
+          information required by Section 3(a)(1)(A) to the extent
+          reasonably practicable.
+
+       4. If You Share Adapted Material You produce, the Adapter's
+          License You apply must not prevent recipients of the Adapted
+          Material from complying with this Public License.
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+     to extract, reuse, reproduce, and Share all or a substantial
+     portion of the contents of the database for NonCommercial purposes
+     only;
+
+  b. if You include all or a substantial portion of the database
+     contents in a database in which You have Sui Generis Database
+     Rights, then the database in which You have Sui Generis Database
+     Rights (but not its individual contents) is Adapted Material; and
+
+  c. You must comply with the conditions in Section 3(a) if You Share
+     all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+  c. The disclaimer of warranties and limitation of liability provided
+     above shall be interpreted in a manner that, to the extent
+     possible, most closely approximates an absolute disclaimer and
+     waiver of all liability.
+
+Section 6 -- Term and Termination.
+
+  a. This Public License applies for the term of the Copyright and
+     Similar Rights licensed here. However, if You fail to comply with
+     this Public License, then Your rights under this Public License
+     terminate automatically.
+
+  b. Where Your right to use the Licensed Material has terminated under
+     Section 6(a), it reinstates:
+
+       1. automatically as of the date the violation is cured, provided
+          it is cured within 30 days of Your discovery of the
+          violation; or
+
+       2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any
+     right the Licensor may have to seek remedies for Your violations
+     of this Public License.
+
+  c. For the avoidance of doubt, the Licensor may also offer the
+     Licensed Material under separate terms or conditions or stop
+     distributing the Licensed Material at any time; however, doing so
+     will not terminate this Public License.
+
+  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+     License.
+
+Section 7 -- Other Terms and Conditions.
+
+  a. The Licensor shall not be bound by any additional or different
+     terms or conditions communicated by You unless expressly agreed.
+
+  b. Any arrangements, understandings, or agreements regarding the
+     Licensed Material not stated herein are separate from and
+     independent of the terms and conditions of this Public License.
+
+Section 8 -- Interpretation.
+
+  a. For the avoidance of doubt, this Public License does not, and
+     shall not be interpreted to, reduce, limit, restrict, or impose
+     conditions on any use of the Licensed Material that could lawfully
+     be made without permission under this Public License.
+
+  b. To the extent possible, if any provision of this Public License is
+     deemed unenforceable, it shall be automatically reformed to the
+     minimum extent necessary to make it enforceable. If the provision
+     cannot be reformed, it shall be severed from this Public License
+     without affecting the enforceability of the remaining terms and
+     conditions.
+
+  c. No term or condition of this Public License will be waived and no
+     failure to comply consented to unless expressly agreed to by the
+     Licensor.
+
+  d. Nothing in this Public License constitutes or may be interpreted
+     as a limitation upon, or waiver of, any privileges and immunities
+     that apply to the Licensor or You, including from the legal
+     processes of any jurisdiction or authority.
+
+=======================================================================
+
+Creative Commons is not a party to its public
+licenses. Notwithstanding, Creative Commons may elect to apply one of
+its public licenses to material it publishes and in those instances
+will be considered the “Licensor.” The text of the Creative Commons
+public licenses is dedicated to the public domain under the CC0 Public
+Domain Dedication. Except for the limited purpose of indicating that
+material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the
+public licenses.
+
+Creative Commons may be contacted at creativecommons.org.

README.md

@@ -0,0 +1,111 @@
+---
+license: cc-by-nc-4.0
+library_name: pytorch
+tags:
+  - time-series
+  - synthetic
+  - representation-learning
+  - nepa
+---
+
+# LeNEPA encoder (CauKer2M, encoder-only)
+
+This repository contains an **encoder-only** LeNEPA checkpoint exported to `safetensors` for minimal inference.
+
+What is included:
+
+- `lenepa_encoder.safetensors` - **encoder weights only** (no projector, no training/probe state)
+- `inference.py` - minimal end-to-end inference (no Hydra, no W&B, no repo dependency)
+- `lenepa_encoder_config.json` - fixed IO + architecture contract
+- `provenance.json` - original `.pt` checkpoint path + W&B URL
+
+## IO contract
+
+Inputs:
+
+- `x_waveform`: `torch.float32` with shape `[B, 1, 5000]`
+- sampling frequency: `1`
+- channels: `["c0"]`
+
+Tokenizer:
+
+- causal `conv_patch_embed` with `patch_size=8`
+- per-patch normalization inside the tokenizer
+- MSSE scalar embeddings for patch mean and patch std
+
+Outputs:
+
+- `patch_tokens`: `[B, 625, 256]` (post-final-norm tokens)
+- `embedding`: `[B, 256]` (mean pooled over tokens)
+
+## Preparing inputs
+
+The exported `inference.py` expects `x_waveform.shape == [B, 1, 5000]` and does **not** resample internally.
+
+Two common options:
+
+1. No resampling
+
+Use this when your waveform is already length `5000` or you just if it works better without resampling:
+
+```python
+from pathlib import Path
+
+import torch
+
+from inference import encode_lenepa, load_lenepa_encoder
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = load_lenepa_encoder(weights_path=Path("lenepa_encoder.safetensors"), device=device)
+
+x = torch.randn(2, 1, 5000, device=device, dtype=torch.float32)  # [B, C, L=5000]
+out = encode_lenepa(model=model, x_waveform=x)
+print(out.patch_tokens.shape)
+```
+
+2. Interpolate to `5000`
+
+Validation on UCR shows, that resampling samples that have different length to `5000` generates noticably better classification quality:
+
+```python
+from pathlib import Path
+
+import torch
+from torch.nn import functional as F
+
+from inference import encode_lenepa, load_lenepa_encoder
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = load_lenepa_encoder(weights_path=Path("lenepa_encoder.safetensors"), device=device)
+
+x_raw = torch.randn(2, 1, 137, device=device, dtype=torch.float32)  # [B, C, L_raw]
+x = F.interpolate(x_raw, size=5000, mode="linear", align_corners=False)  # [B, C, 5000]
+out = encode_lenepa(model=model, x_waveform=x)
+print(out.embedding.shape)
+```
+
+## Usage
+
+Smoke test (loads `lenepa_encoder.safetensors` from the current directory and prints output shapes):
+
+```bash
+python inference.py
+```
+
+Programmatic usage:
+
+```python
+from pathlib import Path
+
+import torch
+from torch.nn import functional as F
+
+from inference import encode_lenepa, load_lenepa_encoder
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = load_lenepa_encoder(weights_path=Path("lenepa_encoder.safetensors"), device=device)
+x_raw = torch.randn(2, 1, 137, device=device, dtype=torch.float32)  # [B, C, L_raw]
+x = F.interpolate(x_raw, size=5000, mode="linear", align_corners=False)  # [B, C, 5000]
+out = encode_lenepa(model=model, x_waveform=x)
+print(out.embedding.shape)
+```

inference.py

@@ -0,0 +1,381 @@
+"""Minimal inference for the published LeNEPA encoder checkpoint.
+
+Published IO contract:
+  - x_waveform: torch.float32 [B, 1, 5000], channel order: ["c0"]
+  - outputs:
+      patch_tokens: [B, 625, 256]
+      embedding:    [B, 256]
+
+This code intentionally does NOT:
+  - resample / crop / pad inputs
+  - support other checkpoints or architectures
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import torch
+from safetensors.torch import load_file as safetensors_load
+from torch import nn
+from torch.nn import functional as F
+
+# -----------------------------
+# Published constants (no knobs)
+# -----------------------------
+
+SAMPLING_FREQUENCY = 1
+CHANNELS = ("c0",)
+
+NUM_CHANNELS = 1
+CHANNEL_SIZE = 5000
+PATCH_SIZE = 8
+NUM_PATCHES = 625  # 5000 / 8
+
+DIM = 256
+DEPTH = 8
+NUM_HEADS = 4
+MLP_RATIO = 4.0
+
+PATCH_EMBED_CNN_DIM = 192
+SCALAR_HIDDEN_DIM = 32
+SCALAR_SCALES = (0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0, 10000.0)
+SCALAR_EPSILON = 1.1
+
+QKV_BIAS = True
+BIAS = True
+NORM_EPS = 1e-6
+
+ROPE_BASE = 10_000
+QK_NORM_EPS = 1e-6
+
+
+@dataclass(frozen=True)
+class LeNEPAEncoderOutput:
+    """Outputs of the published LeNEPA encoder."""
+
+    patch_tokens: torch.Tensor  # [B, T=625, D=256]
+    embedding: torch.Tensor  # [B, D=256]
+
+
+class ScalarEncoder(nn.Module):
+    """Affine + LayerNorm scalar encoder used inside the MSSE blocks."""
+
+    def __init__(self, *, k: float, hidden_dim: int, eps: float) -> None:
+        super().__init__()
+        if hidden_dim < 1:
+            raise ValueError(f"hidden_dim must be >= 1, got {hidden_dim}")
+        if eps <= 0:
+            raise ValueError(f"eps must be > 0, got {eps}")
+        self.k = float(k)
+        self.w = nn.Parameter(torch.rand((1, hidden_dim), dtype=torch.float32, requires_grad=True))
+        self.b = nn.Parameter(torch.rand((1, hidden_dim), dtype=torch.float32, requires_grad=True))
+        self.norm = nn.LayerNorm(hidden_dim, eps=eps)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # x: [..., 1]
+        if x.size(-1) != 1:
+            raise ValueError(f"Expected x[..., 1], got x.shape={tuple(x.shape)}")
+        z = x * self.w + self.k * self.b  # [..., H]
+        return self.norm(z)  # [..., H]
+
+
+class MultiScaledScalarEncoder(nn.Module):
+    """Blend per-scale scalar encoders based on the scalar magnitude."""
+
+    def __init__(
+        self,
+        *,
+        scales: tuple[float, ...],
+        hidden_dim: int,
+        epsilon: float,
+        eps: float,
+    ) -> None:
+        super().__init__()
+        if not scales:
+            raise ValueError("scales must be non-empty")
+        if any(scale <= 0 for scale in scales):
+            raise ValueError(f"All scales must be > 0, got scales={scales}")
+        if hidden_dim < 1:
+            raise ValueError(f"hidden_dim must be >= 1, got {hidden_dim}")
+        if epsilon <= 0:
+            raise ValueError(f"epsilon must be > 0, got {epsilon}")
+        if eps <= 0:
+            raise ValueError(f"eps must be > 0, got {eps}")
+
+        scales_t = torch.tensor(scales, dtype=torch.float32)  # [S]
+        self.register_buffer("scales", scales_t, persistent=False)
+        self.epsilon = float(epsilon)
+        self.encoders = nn.ModuleList(
+            [ScalarEncoder(k=float(scale), hidden_dim=hidden_dim, eps=eps) for scale in scales]
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # x: [..., 1]
+        if x.size(-1) != 1:
+            raise ValueError(f"Expected x[..., 1], got x.shape={tuple(x.shape)}")
+
+        x_abs = x.abs().to(dtype=torch.float32)  # [..., 1]
+        scales = self.scales.to(device=x.device)  # [S]
+        inv_scales = (1.0 / scales).reshape(1, -1)  # [1, S]
+        ratio = torch.matmul(x_abs, inv_scales)  # [..., S]
+        alpha = (1.0 / torch.log(ratio + self.epsilon)).abs()  # [..., S]
+        alpha = alpha / alpha.sum(dim=-1, keepdim=True)  # [..., S]
+        alpha = alpha.unsqueeze(-1)  # [..., S, 1]
+
+        encoded_list = [encoder(x) for encoder in self.encoders]  # S * [..., H]
+        encoded = torch.stack(encoded_list, dim=-2)  # [..., S, H]
+        mixed = (encoded.to(dtype=torch.float32) * alpha).sum(dim=-2)  # [..., H]
+        return mixed.to(dtype=x.dtype)  # [..., H]
+
+
+class RotaryEmbedding(nn.Module):
+    """Rotary positional embeddings (RoPE) applied to Q/K."""
+
+    def __init__(self, *, dim: int, base: int) -> None:
+        super().__init__()
+        if dim % 2 != 0:
+            raise ValueError(f"RoPE requires even head_dim, got {dim}")
+        inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))  # [Dh/2]
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self._seq_len_cached: int | None = None
+        self._cos_cached: torch.Tensor | None = None
+        self._sin_cached: torch.Tensor | None = None
+        self._device_cached: torch.device | None = None
+        self._dtype_cached: torch.dtype | None = None
+
+    def _build_cache(self, *, seq_len: int, device: torch.device, dtype: torch.dtype) -> None:
+        positions = torch.arange(seq_len, device=device, dtype=self.inv_freq.dtype)  # [T]
+        freqs = torch.einsum("i,j->ij", positions, self.inv_freq)  # [T, Dh/2]
+        self._cos_cached = freqs.cos().to(dtype)  # [T, Dh/2]
+        self._sin_cached = freqs.sin().to(dtype)  # [T, Dh/2]
+        self._seq_len_cached = seq_len
+        self._device_cached = device
+        self._dtype_cached = dtype
+
+    def _get_cos_sin(
+        self, *, seq_len: int, device: torch.device, dtype: torch.dtype
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        if (
+            self._cos_cached is None
+            or self._sin_cached is None
+            or self._seq_len_cached != seq_len
+            or self._device_cached != device
+            or self._dtype_cached != dtype
+        ):
+            self._build_cache(seq_len=seq_len, device=device, dtype=dtype)
+        if self._cos_cached is None or self._sin_cached is None:
+            raise RuntimeError("RoPE cache was not built; this is a bug")
+        return self._cos_cached, self._sin_cached
+
+    def _apply_rotary(self, x: torch.Tensor, *, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
+        # x: [B, H, T, Dh]
+        B, H, T, Dh = x.shape
+        x_2 = x.view(B, H, T, Dh // 2, 2)  # [B, H, T, Dh/2, 2]
+        x1 = x_2[..., 0]  # [B, H, T, Dh/2]
+        x2 = x_2[..., 1]  # [B, H, T, Dh/2]
+        cos = cos.unsqueeze(0).unsqueeze(0)  # [1, 1, T, Dh/2]
+        sin = sin.unsqueeze(0).unsqueeze(0)  # [1, 1, T, Dh/2]
+        out1 = x1 * cos - x2 * sin
+        out2 = x1 * sin + x2 * cos
+        return torch.stack((out1, out2), dim=-1).flatten(-2)  # [B, H, T, Dh]
+
+    def apply(self, q: torch.Tensor, k: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+        cos, sin = self._get_cos_sin(seq_len=q.size(-2), device=q.device, dtype=q.dtype)  # [T, Dh/2]
+        return self._apply_rotary(q, cos=cos, sin=sin), self._apply_rotary(k, cos=cos, sin=sin)
+
+
+class Attention(nn.Module):
+    """Causal self-attention with RoPE + QK-Norm (no dropout)."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        if DIM % NUM_HEADS != 0:
+            raise ValueError(f"DIM must be divisible by NUM_HEADS, got DIM={DIM} NUM_HEADS={NUM_HEADS}")
+        head_dim = DIM // NUM_HEADS
+        self.num_heads = NUM_HEADS
+        self.rope = RotaryEmbedding(dim=head_dim, base=ROPE_BASE)
+        self.qk_norm = nn.LayerNorm(head_dim, eps=QK_NORM_EPS, elementwise_affine=False)
+        self.qkv = nn.Linear(DIM, DIM * 3, bias=QKV_BIAS)
+        self.proj = nn.Linear(DIM, DIM, bias=BIAS)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # x: [B, T, D]
+        B, T, D = x.shape
+        qkv = (
+            self.qkv(x)  # [B, T, 3*D]
+            .reshape(B, T, 3, self.num_heads, D // self.num_heads)  # [B, T, 3, H, Dh]
+            .permute(2, 0, 3, 1, 4)  # [3, B, H, T, Dh]
+        )
+        q, k, v = qkv[0], qkv[1], qkv[2]  # each [B, H, T, Dh]
+        q, k = self.rope.apply(q, k)  # [B, H, T, Dh] each
+        q = self.qk_norm(q)  # [B, H, T, Dh]
+        k = self.qk_norm(k)  # [B, H, T, Dh]
+        attn = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0, is_causal=True)  # [B, H, T, Dh]
+        out = attn.transpose(1, 2).reshape(B, T, D)  # [B, T, D]
+        return self.proj(out)  # [B, T, D]
+
+
+class GatedMLP(nn.Module):
+    """SwiGLU MLP used in this checkpoint."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        hidden_dim = int((2 / 3) * MLP_RATIO * DIM)
+        if hidden_dim <= 0:
+            raise ValueError(f"hidden_dim must be > 0, got {hidden_dim}")
+        self.fc1 = nn.Linear(DIM, hidden_dim * 2, bias=BIAS)
+        self.fc2 = nn.Linear(hidden_dim, DIM, bias=BIAS)
+        self.act = nn.SiLU()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # x: [B, T, D]
+        gate_and_value = self.fc1(x)  # [B, T, 2H]
+        gate, value = gate_and_value.chunk(2, dim=-1)  # each [B, T, H]
+        return self.fc2(self.act(gate) * value)  # [B, T, D]
+
+
+class Block(nn.Module):
+    """Transformer block: LN -> Attn -> residual -> LN -> MLP -> residual."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.norm1 = nn.LayerNorm(DIM, eps=NORM_EPS)
+        self.attn = Attention()
+        self.norm2 = nn.LayerNorm(DIM, eps=NORM_EPS)
+        self.mlp = GatedMLP()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # x: [B, T, D]
+        x = x + self.attn(self.norm1(x))  # [B, T, D]
+        x = x + self.mlp(self.norm2(x))  # [B, T, D]
+        return x
+
+
+class PatchEmbedding(nn.Module):
+    """Conv patch embedding: Conv1d(C->D_cnn, kernel=stride=patch_size)."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.proj = nn.Conv1d(
+            in_channels=NUM_CHANNELS,
+            out_channels=PATCH_EMBED_CNN_DIM,
+            kernel_size=PATCH_SIZE,
+            stride=PATCH_SIZE,
+            bias=BIAS,
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # x: [B, C, L]
+        z_t = self.proj(x)  # [B, D_cnn, T]
+        return z_t.transpose(1, 2)  # [B, T, D_cnn]
+
+
+class LeNEPAEncoder(nn.Module):
+    """LeNEPA encoder trunk for this exact checkpoint."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        if CHANNEL_SIZE % PATCH_SIZE != 0:
+            raise ValueError("CHANNEL_SIZE must be divisible by PATCH_SIZE")
+        self.nepa_patch_embed_mean_encoder = MultiScaledScalarEncoder(
+            scales=SCALAR_SCALES,
+            hidden_dim=SCALAR_HIDDEN_DIM,
+            epsilon=SCALAR_EPSILON,
+            eps=NORM_EPS,
+        )
+        self.nepa_patch_embed_std_encoder = MultiScaledScalarEncoder(
+            scales=SCALAR_SCALES,
+            hidden_dim=SCALAR_HIDDEN_DIM,
+            epsilon=SCALAR_EPSILON,
+            eps=NORM_EPS,
+        )
+        self.patch_embed = PatchEmbedding()
+        self.blocks = nn.ModuleList([Block() for _ in range(DEPTH)])
+        self.norm = nn.LayerNorm(DIM, eps=NORM_EPS)
+
+    def _tokenize(self, x: torch.Tensor) -> torch.Tensor:
+        # x: [B, C, L]
+        B, C, L = x.shape
+        if C != NUM_CHANNELS or L != CHANNEL_SIZE:
+            raise ValueError(
+                "Input must match the published contract: "
+                f"expected [B, {NUM_CHANNELS}, {CHANNEL_SIZE}], got {tuple(x.shape)}"
+            )
+        if L % PATCH_SIZE != 0:
+            raise ValueError(f"Expected L divisible by PATCH_SIZE, got L={L}, PATCH_SIZE={PATCH_SIZE}")
+
+        T = L // PATCH_SIZE
+        x_f32 = x.to(dtype=torch.float32)  # [B, C, L]
+        x_patches = x_f32.reshape(B, C, T, PATCH_SIZE)  # [B, C, T, P]
+        mean_patch = x_patches.mean(dim=(1, 3))  # [B, T]
+        std_patch = x_patches.std(dim=(1, 3), unbiased=False)  # [B, T]
+        mean_broadcast = mean_patch[:, None, :, None]  # [B, 1, T, 1]
+        std_broadcast = std_patch[:, None, :, None]  # [B, 1, T, 1]
+        x_norm_patches = (x_patches - mean_broadcast) / (std_broadcast + NORM_EPS)  # [B, C, T, P]
+        x_norm = x_norm_patches.reshape(B, C, L).to(dtype=x.dtype)  # [B, C, L]
+
+        z_cnn = self.patch_embed(x_norm)  # [B, T, D_cnn]
+        e_mean = self.nepa_patch_embed_mean_encoder(mean_patch.unsqueeze(-1))  # [B, T, H]
+        e_std = self.nepa_patch_embed_std_encoder(std_patch.unsqueeze(-1))  # [B, T, H]
+        tokens = torch.cat([z_cnn, e_mean.to(dtype=z_cnn.dtype), e_std.to(dtype=z_cnn.dtype)], dim=-1)  # [B, T, D]
+        if tokens.size(-1) != DIM:
+            raise RuntimeError(
+                "Tokenizer produced unexpected dim. "
+                f"Got tokens.shape={tuple(tokens.shape)}, expected last dim={DIM}"
+            )
+        return tokens
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # x: [B, C, L]
+        z = self._tokenize(x)  # [B, T, D]
+        for block in self.blocks:
+            z = block(z)  # [B, T, D]
+        return self.norm(z)  # [B, T, D]
+
+
+@torch.inference_mode()
+def encode_lenepa(*, model: LeNEPAEncoder, x_waveform: torch.Tensor) -> LeNEPAEncoderOutput:
+    """Encode a batch of waveforms."""
+    if x_waveform.dtype != torch.float32:
+        raise ValueError(f"x_waveform must be float32, got {x_waveform.dtype}")
+    if x_waveform.dim() != 3:
+        raise ValueError(f"x_waveform must be [B, C, L], got {tuple(x_waveform.shape)}")
+    model_device = next(model.parameters()).device
+    if x_waveform.device != model_device:
+        raise ValueError(
+            "x_waveform must be on the same device as the model. "
+            f"x_waveform.device={x_waveform.device} model.device={model_device}"
+        )
+
+    patch_tokens = model(x_waveform)  # [B, T, D]
+    embedding = patch_tokens.mean(dim=1)  # [B, D]
+    return LeNEPAEncoderOutput(patch_tokens=patch_tokens, embedding=embedding)
+
+
+def load_lenepa_encoder(*, weights_path: Path, device: torch.device) -> LeNEPAEncoder:
+    """Load the published encoder weights from a safetensors file."""
+    if not weights_path.is_file():
+        raise ValueError(f"weights_path does not exist: {str(weights_path)!r}")
+    state = safetensors_load(str(weights_path))
+    model = LeNEPAEncoder()
+    model.load_state_dict(state, strict=True)
+    model.eval()
+    model.requires_grad_(False)
+    return model.to(device)
+
+
+def _smoke_test() -> None:
+    """Small end-to-end smoke test (random input, prints output shapes)."""
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    here = Path(__file__).resolve().parent
+    model = load_lenepa_encoder(weights_path=here / "lenepa_encoder.safetensors", device=device)
+    x = torch.randn(2, 1, 5000, device=device, dtype=torch.float32)  # [B=2, C=1, L=5000]
+    out = encode_lenepa(model=model, x_waveform=x)
+    print("patch_tokens", tuple(out.patch_tokens.shape))
+    print("embedding", tuple(out.embedding.shape))
+
+
+if __name__ == "__main__":
+    _smoke_test()

lenepa_encoder.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:901d54caf39b637efba0b6c866380f33fa26e2cf7711bc6abe2fead27bbd84c1
+size 25302224

lenepa_encoder_config.json

@@ -0,0 +1,45 @@
+{
+  "bias": true,
+  "channel_size": 5000,
+  "channels": [
+    "c0"
+  ],
+  "depth": 8,
+  "dim": 256,
+  "format": "lenepa_encoder",
+  "format_version": 1,
+  "is_causal": true,
+  "mlp_ratio": 4.0,
+  "nepa_final_norm": "ln",
+  "nepa_patch_embed_cnn_dim": 192,
+  "nepa_patch_embed_scalar_epsilon": 1.1,
+  "nepa_patch_embed_scalar_hidden_dim": 32,
+  "nepa_patch_embed_scalar_scales": [
+    0.0001,
+    0.001,
+    0.01,
+    0.1,
+    1.0,
+    10.0,
+    100.0,
+    1000.0,
+    10000.0
+  ],
+  "nepa_patch_embed_scalar_stats_mode": "patch_norm",
+  "nepa_rep_pooling": "mean",
+  "nepa_static_tokenizer": "conv_patch_embed",
+  "norm_eps": 1e-06,
+  "num_heads": 4,
+  "num_patches": 625,
+  "num_registers": 0,
+  "patch_size": 8,
+  "pos_embed_type": "none",
+  "qk_norm_eps": 1e-06,
+  "qkv_bias": true,
+  "rope_base": 10000,
+  "sampling_frequency": 1,
+  "use_nepa": true,
+  "use_qk_norm": true,
+  "use_rope": true,
+  "use_swiglu": true
+}

provenance.json

@@ -0,0 +1,14 @@
+{
+  "checkpoint_step": 20000,
+  "exported_at_utc": "2026-03-10T15:37:13.781012+00:00",
+  "git_commit": "ab2a718ee2576d9e214c831eb405d36e235164c9",
+  "notes": "encoder-only export (proj.* and sigreg.* removed)",
+  "pretrain_wandb": {
+    "entity": "langotime",
+    "project": "ECG-LeJEPA",
+    "run_id": "6y3c5r13",
+    "run_name": "CAUKER2M_L5000_LENEPA_SIGREGT2p5_L0-8_PD0_PROJ_LR2x_MSSE_PATCHNORM_D256_OPTOYTSBCBAL_s0_UCRinterp5000",
+    "url": "https://wandb.ai/langotime/ECG-LeJEPA/runs/6y3c5r13"
+  },
+  "source_checkpoint_path": "pretrain/CAUKER2M_L5000_LENEPA_SIGREGT2p5_L0-8_PD0_PROJ_LR2x_MSSE_PATCHNORM_D256_OPTOYTSBCBAL_s0_UCRinterp5000/chkpt_20000.pt"
+}

requirements.txt

@@ -0,0 +1,3 @@
+torch
+safetensors
+