Upload 14 files

.gitattributes

@@ -0,0 +1 @@
+emcoder/model.safetensors filter=lfs diff=lfs merge=lfs -text

LICENSE.md

@@ -0,0 +1,402 @@
+Attribution-NonCommercial-NoDerivatives 4.0 International
+
+=======================================================================
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+     Considerations for licensors: Our public licenses are
+     intended for use by those authorized to give the public
+     permission to use material in ways otherwise restricted by
+     copyright and certain other rights. Our licenses are
+     irrevocable. Licensors should read and understand the terms
+     and conditions of the license they choose before applying it.
+     Licensors should also secure all rights necessary before
+     applying our licenses so that the public can reuse the
+     material as expected. Licensors should clearly mark any
+     material not subject to the license. This includes other CC-
+     licensed material, or material used under an exception or
+     limitation to copyright. More considerations for licensors:
+    wiki.creativecommons.org/Considerations_for_licensors
+
+     Considerations for the public: By using one of our public
+     licenses, a licensor grants the public permission to use the
+     licensed material under specified terms and conditions. If
+     the licensor's permission is not necessary for any reason--for
+     example, because of any applicable exception or limitation to
+     copyright--then that use is not regulated by the license. Our
+     licenses grant only permissions under copyright and certain
+     other rights that a licensor has authority to grant. Use of
+     the licensed material may still be restricted for other
+     reasons, including because others have copyright or other
+     rights in the material. A licensor may make special requests,
+     such as asking that all changes be marked or described.
+     Although not required by our licenses, you are encouraged to
+     respect those requests where reasonable. More considerations
+     for the public:
+    wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution-NonCommercial-NoDerivatives 4.0
+International Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial-NoDerivatives 4.0 International Public
+License ("Public License"). To the extent this Public License may be
+interpreted as a contract, You are granted the Licensed Rights in
+consideration of Your acceptance of these terms and conditions, and the
+Licensor grants You such rights in consideration of benefits the
+Licensor receives from making the Licensed Material available under
+these terms and conditions.
+
+
+Section 1 -- Definitions.
+
+  a. Adapted Material means material subject to Copyright and Similar
+     Rights that is derived from or based upon the Licensed Material
+     and in which the Licensed Material is translated, altered,
+     arranged, transformed, or otherwise modified in a manner requiring
+     permission under the Copyright and Similar Rights held by the
+     Licensor. For purposes of this Public License, where the Licensed
+     Material is a musical work, performance, or sound recording,
+     Adapted Material is always produced where the Licensed Material is
+     synched in timed relation with a moving image.
+
+  b. Copyright and Similar Rights means copyright and/or similar rights
+     closely related to copyright including, without limitation,
+     performance, broadcast, sound recording, and Sui Generis Database
+     Rights, without regard to how the rights are labeled or
+     categorized. For purposes of this Public License, the rights
+     specified in Section 2(b)(1)-(2) are not Copyright and Similar
+     Rights.
+
+  c. Effective Technological Measures means those measures that, in the
+     absence of proper authority, may not be circumvented under laws
+     fulfilling obligations under Article 11 of the WIPO Copyright
+     Treaty adopted on December 20, 1996, and/or similar international
+     agreements.
+
+  d. Exceptions and Limitations means fair use, fair dealing, and/or
+     any other exception or limitation to Copyright and Similar Rights
+     that applies to Your use of the Licensed Material.
+
+  e. Licensed Material means the artistic or literary work, database,
+     or other material to which the Licensor applied this Public
+     License.
+
+  f. Licensed Rights means the rights granted to You subject to the
+     terms and conditions of this Public License, which are limited to
+     all Copyright and Similar Rights that apply to Your use of the
+     Licensed Material and that the Licensor has authority to license.
+
+  g. Licensor means the individual(s) or entity(ies) granting rights
+     under this Public License.
+
+  h. NonCommercial means not primarily intended for or directed towards
+     commercial advantage or monetary compensation. For purposes of
+     this Public License, the exchange of the Licensed Material for
+     other material subject to Copyright and Similar Rights by digital
+     file-sharing or similar means is NonCommercial provided there is
+     no payment of monetary compensation in connection with the
+     exchange.
+
+  i. Share means to provide material to the public by any means or
+     process that requires permission under the Licensed Rights, such
+     as reproduction, public display, public performance, distribution,
+     dissemination, communication, or importation, and to make material
+     available to the public including in ways that members of the
+     public may access the material from a place and at a time
+     individually chosen by them.
+
+  j. Sui Generis Database Rights means rights other than copyright
+     resulting from Directive 96/9/EC of the European Parliament and of
+     the Council of 11 March 1996 on the legal protection of databases,
+     as amended and/or succeeded, as well as other essentially
+     equivalent rights anywhere in the world.
+
+  k. You means the individual or entity exercising the Licensed Rights
+     under this Public License. Your has a corresponding meaning.
+
+
+Section 2 -- Scope.
+
+  a. License grant.
+
+       1. Subject to the terms and conditions of this Public License,
+          the Licensor hereby grants You a worldwide, royalty-free,
+          non-sublicensable, non-exclusive, irrevocable license to
+          exercise the Licensed Rights in the Licensed Material to:
+
+            a. reproduce and Share the Licensed Material, in whole or
+               in part, for NonCommercial purposes only; and
+
+            b. produce and reproduce, but not Share, Adapted Material
+               for NonCommercial purposes only.
+
+       2. Exceptions and Limitations. For the avoidance of doubt, where
+          Exceptions and Limitations apply to Your use, this Public
+          License does not apply, and You do not need to comply with
+          its terms and conditions.
+
+       3. Term. The term of this Public License is specified in Section
+          6(a).
+
+       4. Media and formats; technical modifications allowed. The
+          Licensor authorizes You to exercise the Licensed Rights in
+          all media and formats whether now known or hereafter created,
+          and to make technical modifications necessary to do so. The
+          Licensor waives and/or agrees not to assert any right or
+          authority to forbid You from making technical modifications
+          necessary to exercise the Licensed Rights, including
+          technical modifications necessary to circumvent Effective
+          Technological Measures. For purposes of this Public License,
+          simply making modifications authorized by this Section 2(a)
+          (4) never produces Adapted Material.
+
+       5. Downstream recipients.
+
+            a. Offer from the Licensor -- Licensed Material. Every
+               recipient of the Licensed Material automatically
+               receives an offer from the Licensor to exercise the
+               Licensed Rights under the terms and conditions of this
+               Public License.
+
+            b. No downstream restrictions. You may not offer or impose
+               any additional or different terms or conditions on, or
+               apply any Effective Technological Measures to, the
+               Licensed Material if doing so restricts exercise of the
+               Licensed Rights by any recipient of the Licensed
+               Material.
+
+       6. No endorsement. Nothing in this Public License constitutes or
+          may be construed as permission to assert or imply that You
+          are, or that Your use of the Licensed Material is, connected
+          with, or sponsored, endorsed, or granted official status by,
+          the Licensor or others designated to receive attribution as
+          provided in Section 3(a)(1)(A)(i).
+
+  b. Other rights.
+
+       1. Moral rights, such as the right of integrity, are not
+          licensed under this Public License, nor are publicity,
+          privacy, and/or other similar personality rights; however, to
+          the extent possible, the Licensor waives and/or agrees not to
+          assert any such rights held by the Licensor to the limited
+          extent necessary to allow You to exercise the Licensed
+          Rights, but not otherwise.
+
+       2. Patent and trademark rights are not licensed under this
+          Public License.
+
+       3. To the extent possible, the Licensor waives any right to
+          collect royalties from You for the exercise of the Licensed
+          Rights, whether directly or through a collecting society
+          under any voluntary or waivable statutory or compulsory
+          licensing scheme. In all other cases the Licensor expressly
+          reserves any right to collect such royalties, including when
+          the Licensed Material is used other than for NonCommercial
+          purposes.
+
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+  a. Attribution.
+
+       1. If You Share the Licensed Material, You must:
+
+            a. retain the following if it is supplied by the Licensor
+               with the Licensed Material:
+
+                 i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if
+                    designated);
+
+                ii. a copyright notice;
+
+               iii. a notice that refers to this Public License;
+
+                iv. a notice that refers to the disclaimer of
+                    warranties;
+
+                 v. a URI or hyperlink to the Licensed Material to the
+                    extent reasonably practicable;
+
+            b. indicate if You modified the Licensed Material and
+               retain an indication of any previous modifications; and
+
+            c. indicate the Licensed Material is licensed under this
+               Public License, and include the text of, or the URI or
+               hyperlink to, this Public License.
+
+          For the avoidance of doubt, You do not have permission under
+          this Public License to Share Adapted Material.
+
+       2. You may satisfy the conditions in Section 3(a)(1) in any
+          reasonable manner based on the medium, means, and context in
+          which You Share the Licensed Material. For example, it may be
+          reasonable to satisfy the conditions by providing a URI or
+          hyperlink to a resource that includes the required
+          information.
+
+       3. If requested by the Licensor, You must remove any of the
+          information required by Section 3(a)(1)(A) to the extent
+          reasonably practicable.
+
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+  a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+     to extract, reuse, reproduce, and Share all or a substantial
+     portion of the contents of the database for NonCommercial purposes
+     only and provided You do not Share Adapted Material;
+
+  b. if You include all or a substantial portion of the database
+     contents in a database in which You have Sui Generis Database
+     Rights, then the database in which You have Sui Generis Database
+     Rights (but not its individual contents) is Adapted Material; and
+
+  c. You must comply with the conditions in Section 3(a) if You Share
+     all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+  a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+     EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+     AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+     ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+     IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+     WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+     PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+     ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+     KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+     ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+  b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+     TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+     NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+     INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+     COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+     USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+     ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+     DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+     IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+  c. The disclaimer of warranties and limitation of liability provided
+     above shall be interpreted in a manner that, to the extent
+     possible, most closely approximates an absolute disclaimer and
+     waiver of all liability.
+
+
+Section 6 -- Term and Termination.
+
+  a. This Public License applies for the term of the Copyright and
+     Similar Rights licensed here. However, if You fail to comply with
+     this Public License, then Your rights under this Public License
+     terminate automatically.
+
+  b. Where Your right to use the Licensed Material has terminated under
+     Section 6(a), it reinstates:
+
+       1. automatically as of the date the violation is cured, provided
+          it is cured within 30 days of Your discovery of the
+          violation; or
+
+       2. upon express reinstatement by the Licensor.
+
+     For the avoidance of doubt, this Section 6(b) does not affect any
+     right the Licensor may have to seek remedies for Your violations
+     of this Public License.
+
+  c. For the avoidance of doubt, the Licensor may also offer the
+     Licensed Material under separate terms or conditions or stop
+     distributing the Licensed Material at any time; however, doing so
+     will not terminate this Public License.
+
+  d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+     License.
+
+
+Section 7 -- Other Terms and Conditions.
+
+  a. The Licensor shall not be bound by any additional or different
+     terms or conditions communicated by You unless expressly agreed.
+
+  b. Any arrangements, understandings, or agreements regarding the
+     Licensed Material not stated herein are separate from and
+     independent of the terms and conditions of this Public License.
+
+
+Section 8 -- Interpretation.
+
+  a. For the avoidance of doubt, this Public License does not, and
+     shall not be interpreted to, reduce, limit, restrict, or impose
+     conditions on any use of the Licensed Material that could lawfully
+     be made without permission under this Public License.
+
+  b. To the extent possible, if any provision of this Public License is
+     deemed unenforceable, it shall be automatically reformed to the
+     minimum extent necessary to make it enforceable. If the provision
+     cannot be reformed, it shall be severed from this Public License
+     without affecting the enforceability of the remaining terms and
+     conditions.
+
+  c. No term or condition of this Public License will be waived and no
+     failure to comply consented to unless expressly agreed to by the
+     Licensor.
+
+  d. Nothing in this Public License constitutes or may be interpreted
+     as a limitation upon, or waiver of, any privileges and immunities
+     that apply to the Licensor or You, including from the legal
+     processes of any jurisdiction or authority.
+
+=======================================================================
+
+Creative Commons is not a party to its public
+licenses. Notwithstanding, Creative Commons may elect to apply one of
+its public licenses to material it publishes and in those instances
+will be considered the “Licensor.” The text of the Creative Commons
+public licenses is dedicated to the public domain under the CC0 Public
+Domain Dedication. Except for the limited purpose of indicating that
+material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the
+public licenses.
+
+Creative Commons may be contacted at creativecommons.org.

README.md

@@ -0,0 +1,220 @@
+---
+language:
+- en
+license: cc-by-nc-nd-4.0
+library_name: generic
+tags:
+- emotion-recognition
+- bayesian-deep-learning
+- mc-dropout
+- uncertainty-quantification
+- multi-label-classification
+datasets:
+- Skylion007/openwebtext
+- google-research-datasets/go_emotions
+snippet: |
+  from huggingface_hub import snapshot_download
+  from emcoder import EmCoder
+  
+  model_dir = snapshot_download(repo_id="yezdata/EmCoder")
+
+  tokenizer = AutoTokenizer.from_pretrained(model_dir)
+  model = EmCoder.from_pretrained(model_dir)
+metrics:
+- precision
+- recall
+- f1
+model-index:
+- name: EmCoder (v1)
+  results:
+  - task:
+      type: text-classification
+      name: Multi-label Emotion Classification
+    dataset:
+      name: GoEmotions
+      type: go_emotions
+      split: test
+    metrics:
+    - name: Macro F1
+      type: f1
+      value: 0.44
+    - name: Macro Precision
+      type: precision
+      value: 0.408
+    - name: Macro Recall
+      type: recall
+      value: 0.495
+---
+
+# EmCoder
+<blockquote>
+  <b>Probabilistic Emotion Recognition & Uncertainty Quantification</b><br>
+  <b>28 Emotion multi-label classifier trained with MC Dropout methodology</b>
+</blockquote>
+
+
+Unlike standard classifiers, EmCoder quantifies what it doesn't know using Monte Carlo Dropout, making it suitable for high-stakes AI pipelines.<br>
+EmCoder is optimized for **MC Dropout inference**.
+
+
+
+## SOTA benchmark
+### Evaluation on the GoEmotions test split (macro avg metrics)
+EmCoder achieves competitive F1-scores while being ~35% smaller than RoBERTa-base and ~45% smaller than ModernBERT, offering a superior efficiency-to-uncertainty ratio.
+| Model | Precision | Recall | F1-Score | Params |
+| :--- | :--- | :--- | :--- | :--- |
+| **EmCoder (v1)** | **0.408** | **0.495** | **0.440** | **82.1M** |
+| Google BERT (Original) | 0.400 | 0.630 | 0.460 | 110M |
+| RoBERTa-base | 0.575 | 0.396 | 0.450 | 125M |
+| ModernBERT-base | 0.652 | 0.443 | 0.500 | 149M |
+
+
+## How to use
+Since `.safetensors` files only store model weights and not the class logic, you need to use the provided `emcoder.py` to enable **MC Dropout inference**.<br>EmCoder v1.0 requires the `roberta-base` tokenizer for correct token-to-embedding mapping.
+### 1. Setup & Tokenization
+Install dependencies
+```bash
+pip install -r requirements.txt
+```
+Setup EmCoder
+```python
+from transformers import AutoTokenizer
+from huggingface_hub import snapshot_download
+from emcoder import EmCoder # Ensure emcoder.py is in your directory
+
+repo_id = "yezdata/EmCoder"
+model_dir = snapshot_download(repo_id=repo_id)
+print(model_dir)
+
+# Load the same tokenizer used during training
+tokenizer = AutoTokenizer.from_pretrained(model_dir)
+
+# Initialize with same config as training
+model = EmCoder.from_pretrained(model_dir)
+```
+### 2. Bayesian inference
+To obtain probabilistic outputs and uncertainty metrics, use the `mc_forward` method:
+```python
+import torch
+
+# Perform 50 stochastic passes
+N_SAMPLES = 50
+model.eval()
+
+inputs = tokenizer("I am so happy you are here!", return_tensors="pt")
+logits_mc = model.mc_forward(inputs['input_ids'], inputs['attention_mask'], n_samples=N_SAMPLES) # Automatically keeps Dropout active, even when in model.eval
+
+# Bayesian Post-processing
+probs_all = torch.sigmoid(logits_mc) # (n_samples, B, 28)
+
+mean_probs = probs_all.mean(dim=0) # Mean Predicted Probability
+uncertainty = probs_all.std(dim=0) # Epistemic Uncertainty (Standard Deviation)
+
+
+# Formatted Output
+m_probs = mean_probs.squeeze(0)
+u_vals = uncertainty.squeeze(0)
+
+print(f"{'Emotion':<15} | {'Prob':<10} | {'Uncertainty':<10}")
+print("-" * 40)
+
+sorted_indices = torch.argsort(m_probs, descending=True)
+
+for idx in sorted_indices:
+    prob, unc = m_probs[idx].item(), u_vals[idx].item()
+    label = model.config.id2label[idx.item()]
+    
+    if prob > 0.05: # Print only emotions with prob > 5% (optional for clarity)
+        print(f"{label:<15} | {prob:>8.2%} | ±{unc:>8.4f}")
+```
+
+
+## Model Architecture
+![EmCoder Architecture](outputs/architecture.png)
+
+
+### Optimization
+The model is trained using a Weighted Bayesian Binary Cross Entropy loss:
+
+$$
+\mathcal{L}_{Bayesian} = \frac{1}{T} \sum_{t=1}^{T} \text{BCEWithLogits}(z^{(t)}, y; w)
+$$
+
+Where weights $w$ are calculated using a logarithmic class-balancing scale to handle extreme label imbalance:
+
+$$
+w_{c} = \max\left( 0.1, \min\left( 20, 1 + \ln \left( \frac{N_{neg,c} + \epsilon}{N_{pos,c} + \epsilon} \right) \right) \right)
+$$
+
+
+
+## Performance
+**Using threshold of 0.5 for binarizing predictions**
+|                |   precision |   recall |   f1-score |   support |
+|:---------------|------------:|---------:|-----------:|----------:|
+| micro avg      |       0.494 |    0.596 |      0.54  |      6329 |
+| macro avg      |       0.408 |    0.495 |      0.44  |      6329 |
+| weighted avg   |       0.492 |    0.596 |      0.535 |      6329 |
+| samples avg    |       0.525 |    0.616 |      0.544 |      6329 |
+|----------------|-------------|----------|------------|-----------|
+| admiration     |       0.541 |    0.673 |      0.599 |       504 |
+| amusement      |       0.688 |    0.909 |      0.783 |       264 |
+| anger          |       0.419 |    0.47  |      0.443 |       198 |
+| annoyance      |       0.31  |    0.25  |      0.277 |       320 |
+| approval       |       0.304 |    0.271 |      0.287 |       351 |
+| caring         |       0.229 |    0.281 |      0.252 |       135 |
+| confusion      |       0.26  |    0.497 |      0.342 |       153 |
+| curiosity      |       0.432 |    0.764 |      0.552 |       284 |
+| desire         |       0.453 |    0.518 |      0.483 |        83 |
+| disappointment |       0.176 |    0.152 |      0.163 |       151 |
+| disapproval    |       0.279 |    0.404 |      0.33  |       267 |
+| disgust        |       0.447 |    0.545 |      0.491 |       123 |
+| embarrassment  |       0.325 |    0.351 |      0.338 |        37 |
+| excitement     |       0.288 |    0.427 |      0.344 |       103 |
+| fear           |       0.47  |    0.692 |      0.56  |        78 |
+| gratitude      |       0.834 |    0.943 |      0.885 |       352 |
+| grief          |       0     |    0     |      0     |         6 |
+| joy            |       0.445 |    0.652 |      0.529 |       161 |
+| love           |       0.724 |    0.895 |      0.801 |       238 |
+| nervousness    |       0.24  |    0.261 |      0.25  |        23 |
+| optimism       |       0.483 |    0.543 |      0.511 |       186 |
+| pride          |       0.667 |    0.375 |      0.48  |        16 |
+| realization    |       0.226 |    0.166 |      0.191 |       145 |
+| relief         |       0.222 |    0.182 |      0.2   |        11 |
+| remorse        |       0.516 |    0.857 |      0.644 |        56 |
+| sadness        |       0.405 |    0.545 |      0.464 |       156 |
+| surprise       |       0.429 |    0.539 |      0.478 |       141 |
+| neutral        |       0.602 |    0.695 |      0.645 |      1787 |
+
+
+
+**Model uncertainty estimation**
+![epistemic_unc](outputs/epistemic_unc_scatter.png)
+
+**Confusion matrix**
+![multi_label_confusion_matrix](outputs/confusion_matrix.png)
+
+
+
+## Workflow
+![EmCoder Workflow](outputs/workflow.png)
+
+
+### Note
+Note that this model was trained on GoEmotions dataset (social networks domain) and it may not generalize well to other domains.
+
+
+## Citation
+If you use this model, please cite it as follows:
+
+```bibtex
+@software{jez2026emcoder,
+  author = {Václav Jež},
+  title = {EmCoder: Probabilistic Emotion Recognition & Uncertainty Quantification},
+  year = {2026},
+  publisher = {GitHub},
+  journal = {GitHub repository},
+  howpublished = {\url{https://github.com/yezdata/emcoder}},
+  version = {1.0.0}
+}
+```

emcoder.py

@@ -0,0 +1,155 @@
+import torch
+import torch.nn as nn
+from safetensors.torch import load_file
+from pydantic import BaseModel, model_validator, field_validator
+
+
+class ModelConfig(BaseModel):
+    vocab_size: int
+    max_seq_len: int
+    
+    d_model: int
+    n_head: int
+    n_layers: int
+    d_ffn: int
+    
+    dropout: float
+
+    num_labels: int
+    id2label: dict[int, str]
+    label2id: dict[str, int]
+
+    base_encoder_path: str
+
+    @field_validator("id2label", mode="before")
+    @classmethod
+    def coerce_keys_to_int(cls, v):
+        return {int(k): val for k, val in v.items()}
+
+    @model_validator(mode='after')
+    def check_consistency(self):
+        if len(self.id2label) != self.num_labels:
+            raise ValueError("num_labels does not match id2label dictionary len")
+        return self
+
+
+
+
+class EmCoderCore(nn.Module):
+    """The core encoder architecture of EmCoder, without the classification head."""
+    def __init__(self, config: ModelConfig):
+        super().__init__()
+
+        self.token_embedding = nn.Embedding(
+            config.vocab_size,
+            config.d_model
+        )
+        self.pos_embedding = nn.Embedding(
+            config.max_seq_len,
+            config.d_model
+        )
+
+        self.embed_norm = nn.LayerNorm(config.d_model)
+
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model=config.d_model,
+            nhead=config.n_head,
+            dim_feedforward=config.d_ffn,
+            dropout=config.dropout,
+            activation="gelu",
+            norm_first=True,
+            batch_first=True
+        )
+        self.encoder = nn.TransformerEncoder(
+            encoder_layer=encoder_layer,
+            num_layers=config.n_layers
+        )
+        
+        self.final_norm = nn.LayerNorm(config.d_model)
+        self.dropout = nn.Dropout(config.dropout)
+
+    def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
+        """Standard forward pass through the encoder."""
+        seq_len = x.size(1)
+        pos_ids = torch.arange(seq_len, device=x.device).unsqueeze(0)
+
+        x = self.token_embedding(x) + self.pos_embedding(pos_ids)
+
+        x = self.embed_norm(x)
+        x = self.dropout(x)
+
+        padding_mask = (mask == 0)
+
+        encoded = self.encoder(x, src_key_padding_mask=padding_mask)
+        return self.final_norm(encoded)
+
+
+
+class EmCoder(nn.Module):
+    """The full EmCoder model, including the classification head."""
+    def __init__(self, encoder: EmCoderCore, config: ModelConfig):
+        super().__init__()
+
+        self.encoder = encoder
+        self.config = config
+
+        self.classifier = nn.Sequential(
+            nn.Linear(config.d_model, config.d_model),
+            nn.GELU(),
+            nn.Dropout(config.dropout),
+            nn.Linear(config.d_model, config.num_labels)
+        )
+    
+
+    def _set_mc_dropout(self, active: bool = True):
+        for m in self.modules():
+            if isinstance(m, nn.Dropout):
+                m.train(active)
+
+
+    @classmethod
+    def from_pretrained(cls, emcoder_path: str):
+        """Loads the EmCoder model from the specified directory."""
+        # Use model_config.json to initialize same parameterers as in training
+        with open(f"{emcoder_path}/model_config.json", "r") as f:
+            model_config = ModelConfig.model_validate_json(f.read())
+
+
+        encoder = EmCoderCore(model_config)
+        model = cls(encoder, model_config)
+
+        state_dict = load_file(f"{emcoder_path}/model.safetensors")
+        model.load_state_dict(state_dict, strict=True)
+        return model
+
+
+    @staticmethod
+    def _masked_mean_pooling(features: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
+        mask = mask.unsqueeze(-1)  # (B, S, 1)
+        masked_features = features * mask  # (B, S, D)
+        sum_masked_features = masked_features.sum(dim=1)  # (B, D)
+        count_tokens = torch.clamp(mask.sum(dim=1), min=1e-9)  # (B, 1)
+        return sum_masked_features / count_tokens  # (B, D)
+
+
+    def mc_forward(self, x: torch.Tensor, mask: torch.Tensor, n_samples: int) -> torch.Tensor:
+        """Performs Monte Carlo Dropout inference to quantify epistemic uncertainty."""
+        self._set_mc_dropout(active=True)
+
+        B, S = x.shape
+        x_stacked = x.repeat(n_samples, 1) # (n_samples * B, S)
+        mask_stacked = mask.repeat(n_samples, 1)
+
+        features = self.encoder(x_stacked, mask_stacked)
+        pooled = self._masked_mean_pooling(features, mask_stacked)
+        logits = self.classifier(pooled) # (n_samples * B, num_labels)
+
+        return logits.view(n_samples, B, -1)
+
+
+    def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
+        """Standard forward pass without MC Dropout."""
+        features = self.encoder(x, mask)
+
+        pooled = self._masked_mean_pooling(features, mask)
+        return self.classifier(pooled)

emcoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f79307191a44f91b6c9b7e2373062bd655a38efef31a16831e7629d18ce33f50
+size 328565600

emcoder/model_config.json

@@ -0,0 +1,71 @@
+{
+    "vocab_size": 50265,
+    "max_seq_len": 512,
+    "d_model": 768,
+    "n_head": 12,
+    "n_layers": 6,
+    "d_ffn": 3072,
+    "dropout": 0.15,
+    "num_labels": 28,
+    "id2label": {
+        "0": "admiration",
+        "1": "amusement",
+        "2": "anger",
+        "3": "annoyance",
+        "4": "approval",
+        "5": "caring",
+        "6": "confusion",
+        "7": "curiosity",
+        "8": "desire",
+        "9": "disappointment",
+        "10": "disapproval",
+        "11": "disgust",
+        "12": "embarrassment",
+        "13": "excitement",
+        "14": "fear",
+        "15": "gratitude",
+        "16": "grief",
+        "17": "joy",
+        "18": "love",
+        "19": "nervousness",
+        "20": "optimism",
+        "21": "pride",
+        "22": "realization",
+        "23": "relief",
+        "24": "remorse",
+        "25": "sadness",
+        "26": "surprise",
+        "27": "neutral"
+    },
+    "label2id": {
+        "admiration": 0,
+        "amusement": 1,
+        "anger": 2,
+        "annoyance": 3,
+        "approval": 4,
+        "caring": 5,
+        "confusion": 6,
+        "curiosity": 7,
+        "desire": 8,
+        "disappointment": 9,
+        "disapproval": 10,
+        "disgust": 11,
+        "embarrassment": 12,
+        "excitement": 13,
+        "fear": 14,
+        "gratitude": 15,
+        "grief": 16,
+        "joy": 17,
+        "love": 18,
+        "nervousness": 19,
+        "optimism": 20,
+        "pride": 21,
+        "realization": 22,
+        "relief": 23,
+        "remorse": 24,
+        "sadness": 25,
+        "surprise": 26,
+        "neutral": 27
+    },
+    "base_encoder_path": "models/v1/pretrain/checkpoints/epoch_2/step_40000"
+}

emcoder/model_state.json

@@ -0,0 +1,4 @@
+{
+    "train_loss": 0.264223575592041,
+    "eval_loss": 0.2328128303236821
+}

emcoder/train_config.json

@@ -0,0 +1,12 @@
+{
+    "bayesian_train": true,
+    "loss_weights": "log",
+    "tokenized_ds_dir": "data/goemotions_v1_seq512",
+    "encoder_lr": 0.00001,
+    "head_lr": 0.0005,
+    "lr_warmup": 0.05,
+    "weight_decay": 0.01,
+    "batch_size": 32,
+    "gradient_accumulation_steps": 8,
+    "num_epochs": 10
+}

requirements.txt

@@ -0,0 +1,5 @@
+pydantic>=2.13.3
+torch>=2.11.0
+transformers>=5.7.0
+huggingface-hub>=1.13.0
+safetensors>=0.7.0