feat: initial model release

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,180 @@
+RigoBERTa Clinical NON-COMMERCIAL LICENSE AGREEMENT
+
+Release Date: 15-01-2025
+
+By using or distributing any portion or element of the RigoBERTa Clinical
+Language Model, you agree to be bound by this Agreement, acknowledging the
+following terms and conditions.
+
+1\. DEFINITIONS
+
+1.1. Model: Refers to the RigoBERTa Clinical Language Model, owned by ADIC,
+made available under this Agreement.
+
+1.2. Licensor: ADIC.
+
+1.3. Agreement: Means this Non-Commercial License Agreement.
+
+1.4. Acceptable Use Policy: Use of the Model is subject to the
+Acceptable Use Policy defined below. The Licensee agrees to comply with
+it and ensure that users and/or third parties do not use the Model to:
+
+i\) violate or encourage the violation of fundamental rights of third
+parties;
+
+ii\) harm or injure any persons, particularly minors or groups of
+vulnerable, minority, or protected individuals;
+
+iii\) engage in any illegal, invasive, infringing, defamatory, or
+fraudulent activities;
+
+iv\) generate or disseminate false information or content intended to
+harm others;
+
+v\) generate or disseminate personally identifiable information that
+could be used to harm an individual or group of individuals;
+
+vi\) carry out fully automated decision-making that negatively impacts
+individuals' rights;
+
+vii\) engage in, promote, or encourage illegal activities;
+
+viii\) deliberately distribute viruses, worms, trojans, corrupted files,
+hoaxes, or other destructive or deceptive elements;
+
+ix\) interfere with the use of the Model to cause malfunction;
+
+x\) disable, interfere with, or circumvent any aspect of the Model;
+
+xi\) use the Model to access any other ADIC product or service in a way
+that violates the terms of service of such other ADIC product or
+service.
+
+1.5. Derivative Work(s): Means (a) any derivative work from the Model as
+recognized by Spanish and European intellectual property laws, and (b)
+any modifications of the Model and any other model created based on or
+derived from the Model. For clarity, Derivative Works do not include the
+output of any Model.
+
+1.6. Documentation: Means any specifications, manuals, documentation, or
+other written information provided by ADIC related to the Model.
+
+1.7. Licensee: Means you, your employer, or any other person or entity
+(if you are entering into this Agreement on such person or entity's
+behalf), of the age required under applicable laws, rules or regulations
+to provide legal consent and that has legal authority to bind your
+employer or such other person or entity if you are entering in this
+Agreement on their behalf.
+
+1.8. Non-commercial Use: Means exercising any of the rights granted
+under this Agreement for research and/or non-commercial purposes.
+Non-commercial use does not include any production and/or commercial use
+of the Model or any Derivative Work.
+
+2\. LICENSE RIGHTS
+
+a\. Subject to your compliance with this Agreement, the Acceptable Use
+Policy, and the Documentation, ADIC grants you a non-exclusive,
+worldwide, non-transferable, non-sublicensable, revocable, royalty-free,
+and limited license under ADIC's intellectual property rights or other
+proprietary rights embodied in the Model to use, reproduce, distribute,
+and create Derivative Works from the Model, in each case solely for
+research and/or non-commercial uses.
+
+b\. You may not use the Model or Derivative Works to enable third
+parties to use the Model or Derivative Works as part of your hosted
+service or via your APIs, regardless of whether you are adding
+substantial additional functionality thereto or not. Merely distributing
+the Model or Derivative Works for online download without offering any
+related service is not a violation of this section. If you wish to use
+the Model or any Derivative Work for commercial and/or production use or
+make the Model or any Derivative Work available to third parties via
+your hosted service or APIs, please contact ADIC. In case of using the
+Model or any Derivative Work for commercial and/or production purposes,
+the terms of this license will not apply, and different license terms
+and conditions must be accepted and acknowledged.
+
+c\. If you distribute or make the Model or any Derivative Work available
+to a third party, such distribution or availability will remain subject
+to this Agreement, and you must (i) provide a copy of this Agreement to
+such third party and (ii) retain the following attribution notice in a
+"Notice" text file distributed as part of such copies: "*RigoBERTa Clinical
+is a language model owned by ADIC and is distributed under a
+non-commercial research license granted by ADIC.*" If you create a
+Derivative Work from the Model, you must add your own attribution
+notices to the Notice file included with the Model, clearly indicating
+which attributions apply to the Model, and you must state in the Notice
+file that you changed the Model and describe how it was modified.
+
+3\. WARRANTY DISCLAIMER
+
+Unless required by applicable law, the Model and any output or results
+therefrom are provided "as is" without warranties of any kind, either
+express or implied, including, without limitation, any warranties of
+title, non-infringement, merchantability, or fitness for a particular
+purpose. You are solely responsible for determining the appropriateness
+of using or redistributing the Model, Derivative Works, or any output or
+results, and assume any risks associated with your use of the Model,
+Derivative Works, and any output or results.
+
+4\. LIMITATION OF LIABILITY
+
+In no event will ADIC be liable under any theory of liability, whether
+in contract, tort, negligence, products liability or otherwise, arising
+out of this Agreement, for any lost profits or any direct, indirect,
+special, consequential, incidental, or punitive damages, even if ADIC
+has been advised of the possibility of any of the foregoing. ADIC shall
+not be liable to the Licensee for any use of the Model in violation of
+the terms and conditions of this Agreement.
+
+5\. INTELLECTUAL PROPERTY
+
+a\. No trademark licenses are granted under this Agreement, and neither
+ADIC nor the Licensee may use any name or mark owned by or associated
+with the other except as required for reasonable and customary use in
+describing and redistributing the Model or Derivative Works.
+
+b\. All rights, title, and interest in and to the Model, including all
+intellectual property rights, are and will remain the exclusive property
+of ADIC.
+
+c\. Subject to ADIC's ownership of the Model and Derivative Works
+created by or for ADIC, regarding any Derivative Work created by you, as
+between you and ADIC, you shall own such Derivative Works. Similarly,
+any Derivative Work not commissioned by ADIC shall be owned by you.
+
+d\. If you initiate litigation or other proceedings against ADIC
+(including a counterclaim in a lawsuit) alleging that the Model,
+Derivative Works, outputs, or associated products, or any part thereof,
+infringe intellectual property rights or other rights you own or are
+licensable by you, any license granted to you under this Agreement will
+terminate as of the date such litigation or claim is filed or initiated.
+You agree to indemnify and hold ADIC harmless from any third-party
+claims arising out of or related to your use or distribution of the
+Model or Derivative Works in violation of this Agreement.
+
+6\. TERM AND TERMINATION
+
+The term of this Agreement will commence upon your acceptance of this
+Agreement or your access to the Model and will continue in full force
+and effect until terminated in accordance with the terms and conditions
+herein.
+
+This Agreement will automatically terminate if the Licensee breaches any
+of its terms and conditions. Upon termination, the Licensee must cease
+using the Model and delete all copies of it from any device on which it
+may reside. Clauses 3 to 5 shall survive the termination of this
+Agreement.
+
+7\. GENERAL PROVISIONS
+
+7.1. Governing Law: This Agreement shall be governed by and construed in
+accordance with the laws of Spain.
+
+7.2. Entire Agreement: This Agreement constitutes the entire agreement
+between ADIC and the Licensee regarding the use of the Model and
+supersedes all prior agreements and understandings, whether oral or
+written, relating to the Model.
+
+7.3. Modifications: No modification to this Agreement shall be valid
+unless in writing and signed by both ADIC and the Licensee.

README.md

@@ -0,0 +1,119 @@
+---
+library_name: transformers
+license: other
+license_name: rigoclinical-nc
+license_link: https://huggingface.co/IIC/RigoBERTa-Clinical/blob/main/LICENSE
+datasets:
+- IIC/ClinText-SP
+language:
+- es
+pipeline_tag: fill-mask
+---
+
+# RigoBERTa Clinical
+
+**RigoBERTa Clinical** is a state-of-the-art clinical encoder language model for Spanish, developed through domain-adaptive pretraining on the largest publicly available Spanish clinical corpus, **ClinText-SP**. This model significantly improves performance on multiple clinical NLP benchmarks while offering robust language understanding in the clinical domain.
+
+## Model Details
+
+### Model Description
+
+**RigoBERTa Clinical** was built by further pretraining the general-purpose RigoBERTa 2 on a meticulously curated clinical corpus. The pretraining leverages masked language modeling (MLM) to adapt the model’s linguistic knowledge to the Spanish clinical domain.
+
+- **Developed by:** IIC
+- **Model type:** Encoder
+- **Language(s) (NLP):** Spanish
+- **License:** rigoclinical-nc (permissive Non Commercial)
+- **Finetuned from model:** RigoBERTa 2
+
+### Model Sources 
+
+- **Paper:** [ClinText-SP and RigoBERTa Clinical: a new set of open resources for Spanish Clinical NLP](https://arxiv.org/abs/2503.18594)
+
+## Intended Use & Limitations
+
+### Intended Use
+
+**RigoBERTa Clinical** is designed for:
+
+- Clinical text understanding in Spanish.
+- Applications in healthcare NLP tasks such as clinical note classification, entity recognition in clinical texts, and related downstream tasks.
+- Research and development purposes, including benchmarking and further model adaptation.
+
+### Limitations & Caveats
+
+- **Domain Specificity:** Although highly effective for Spanish clinical texts, the model may not generalize to other domains or languages.
+- **Data Biases:** ClinText-SP, while the largest corpus available, may contain biases due to source selection and the inherent limitations of public clinical data.
+- **Operational Cost:** Despite being an encoder-based model with relatively lower computational costs compared to generative LLMs, deployment in resource-constrained settings should be carefully evaluated.
+
+## Training Details
+
+### Training Data: ClinText-SP
+
+ClinText-SP is the largest open Spanish clinical corpus and includes data from various open sources:
+
+- **Volume:** ~26 million tokens, 35,996 samples
+- **Sample Details:** Average of ~700 tokens per sample; contains both long-form clinical cases and shorter, schematic texts,
+- **Sources:** Medical journals, clinical shared tasks, radiological reports, and Wikipedia extracts.
+- **Availability:** [ClinText-SP](https://huggingface.co/datasets/IIC/ClinText-SP) on Hugging Face Datasets
+
+### Training Procedure
+
+#### Preprocessing
+
+- **Tokenizer:** Uses the tokenizer from RigoBERTa 2 to ensure consistency with the base model.
+- **Handling Long Sequences:** Clinical texts exceeding 512 tokens are segmented with a stride of 128 tokens; shorter sequences are padded as necessary.
+- **OOV Handling:** Out-of-vocabulary words are managed using subword tokenization, maintaining robust handling of clinical terminology.
+
+#### Training Details
+
+- **Objective:** Masked Language Modeling (MLM)
+- **Epochs:** 2 full epochs (with the best model selected after ~1.8 epochs, based on downstream performance)
+- **Hyperparameters Grid:**
+  - **Batch Sizes:** 32, 64, 128
+  - **Learning Rates:** Ranges of {5e-6, 1e-5, 2e-5} for batch size 32, {1e-5, 2e-5, 4e-5} for 64, and {1e-5, 4e-5, 8e-5} for 128
+- **Best Settings:** Batch size = 32, Learning rate = 2e-5, ~2800 training steps (~1.8 epochs)
+- **Optimizer:** AdamW with weight decay of 0.1
+- **Hardware:** Trained on a single NVIDIA A100 GPU (80GB memory)
+
+## Evaluation
+
+RigoBERTa Clinical was evaluated on several Spanish clinical NLP tasks including Named Entity Recognition (NER) and multilabel classification. Evaluation metrics (F1 score and micro-averaged F1) indicate that the model outperforms previous clinical and general Spanish language models.
+
+**Key Results:**
+
+- Achieves top performance on datasets such as cantemist, meddocan, and livingner1, among others.
+- Consistently surpasses the performance of models that were trained solely on clinical data, demonstrating the advantage of leveraging general domain knowledge during domain adaptation.
+- Detailed benchmarking results and comparisons are provided in the associated publication.
+
+For a full breakdown of results (including performance on multilingual baselines and other clinical-specific models), please refer to Table 1 and the Nemenyi plot in the original paper.
+
+![Nemenji plot](./data/nemenji.png)
+
+## Citation
+
+If you use RigoBERTa Clinical in your research, please cite the associated paper:
+
+**BibTeX:**
+
+```bibtex
+@misc{subies2025clintextsprigobertaclinicalnew,
+      title={ClinText-SP and RigoBERTa Clinical: a new set of open resources for Spanish Clinical NLP}, 
+      author={Guillem García Subies and Álvaro Barbero Jiménez and Paloma Martínez Fernández},
+      year={2025},
+      eprint={2503.18594},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2503.18594}, 
+}
+```
+
+**APA:**
+
+```
+Subies, G. G., Barbero Jiménez, Á., & Martínez Fernández, P. (2025). ClinText-SP and RigoBERTa Clinical: A new set of open resources for Spanish Clinical NLP. arXiv. https://arxiv.org/abs/2503.18594
+```
+
+## Model Card Authors and Contact
+
+Guillem García Subies: guillem.garcia@iic.uam.es, 100500844@alumnos.uc3m.es

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "/disco/guillem.garcia/models/bs32lr2e5/train_save/checkpoint-2800",
+  "architectures": [
+    "XLMRobertaModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.46.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250002
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae961544b4a9a3f53a0c417cf2492d1e07efc5b6532c2afd424bab5208de4912
+size 2239607176

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a56def25aa40facc030ea8b0b87f3688e4b3c39eb8b45d5702b3a1300fe2a20
+size 17082734

tokenizer_config.json

@@ -0,0 +1,57 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "keep_accents": true,
+  "mask_token": "<mask>",
+  "model_max_len": 512,
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}