| --- |
| license: apache-2.0 |
| language: |
| - eng |
| - deu |
| - fra |
| - pol |
| - por |
| - spa |
| - ita |
| - cmn |
| - nld |
| - afr |
| - als |
| - amh |
| - arb |
| - ars |
| - ary |
| - arz |
| - asm |
| - azj |
| - bel |
| - ben |
| - bew |
| - bod |
| - bos |
| - bul |
| - cat |
| - ces |
| - ckb |
| - cym |
| - dan |
| - div |
| - ekk |
| - ell |
| - epo |
| - eus |
| - fas |
| - fil |
| - fin |
| - gle |
| - glg |
| - gmh |
| - guj |
| - heb |
| - hif |
| - hin |
| - hrv |
| - hun |
| - hye |
| - ind |
| - isl |
| - jpn |
| - kan |
| - kat |
| - kaz |
| - khk |
| - khm |
| - kir |
| - kmr |
| - kor |
| - lao |
| - lat |
| - lit |
| - ltz |
| - lvs |
| - mal |
| - mar |
| - mkd |
| - mlt |
| - mya |
| - nno |
| - nob |
| - npi |
| - nrm |
| - ory |
| - pan |
| - pbt |
| - plt |
| - ron |
| - rus |
| - sin |
| - slk |
| - slv |
| - snd |
| - som |
| - srp |
| - srp |
| - swe |
| - swh |
| - tam |
| - tat |
| - tel |
| - tgk |
| - tha |
| - tur |
| - uig |
| - ukr |
| - urd |
| - uzn |
| - uzn |
| - vie |
| - ydd |
| - zsm |
| --- |
| |
| # FineWeb2-HQ-PlusPlus-Classifier |
|
|
| This repository contains the model weights of the trained deep learning quality classifiers distilled from [FineWeb-edu](https://huggingface.co/HuggingFaceFW/fineweb-edu-classifier) and [DCLM](https://huggingface.co/mlfoundations/fasttext-oh-eli5) for multilingual text quality scoring. The classifier uses [jhu-clsp/mmBERT-base](https://huggingface.co/jhu-clsp/mmBERT-base) embeddings to score the documents and supports English and additional 100 languages. |
|
|
| For more details, see our paper [Adapting English Quality Classifiers for Multilingual LLM Pretraining Data Selection](https://arxiv.org/abs/2610.11585). |
|
|
| ## Quickstart |
|
|
| Classifier uses a simple architecture that takes mean-pooled mmBERT-base embeddings as input. For the DCLM-based classifier, softmax should be applied on the output logit, whereas for the FineWeb-edu-based classifier, the raw logit score should be used. |
|
|
| ```python |
| import torch |
| import torch.nn.functional as F |
| from transformers import AutoModel, AutoTokenizer |
| import huggingface_hub |
| |
| class BinaryClassifier(torch.nn.Module): |
| def __init__(self, embedding_dim=768, hidden_dim=3072): |
| super(BinaryClassifier, self).__init__() |
| self.classifier = torch.nn.Sequential( |
| torch.nn.Linear(embedding_dim, hidden_dim), |
| torch.nn.ReLU(), |
| torch.nn.Linear(hidden_dim, hidden_dim), |
| torch.nn.ReLU(), |
| torch.nn.Linear(hidden_dim, 1), |
| ) |
| |
| def forward(self, X): |
| return self.classifier(X) |
| |
| def to_pt(self, file_name): |
| torch.save(self.state_dict(), file_name) |
| |
| @classmethod |
| def from_pt(cls, file_name, embedding_dim=768, hidden_dim=3072): |
| state_dict = torch.load( |
| file_name, weights_only=True, map_location=torch.device("cpu") |
| ) |
| classifier = BinaryClassifier( |
| embedding_dim=embedding_dim, hidden_dim=hidden_dim |
| ) |
| classifier.load_state_dict(state_dict) |
| classifier.eval() |
| return classifier |
| |
| |
| if __name__ == "__main__": |
| embedding_model_name = "jhu-clsp/mmBERT-base" |
| embedding_tokenizer = AutoTokenizer.from_pretrained(embedding_model_name) |
| embedding_model = AutoModel.from_pretrained( |
| embedding_model_name, |
| dtype=torch.bfloat16, |
| ).cuda() |
| |
| classifiers_dir = huggingface_hub.snapshot_download("epfml/FineWeb2-HQ-PlusPlus-Classifier") |
| mfwedu_model = BinaryClassifier.from_pt(f"{classifiers_dir}/mfwedu.pt").cuda() |
| mdclm_model = BinaryClassifier.from_pt(f"{classifiers_dir}/mdclm.pt").cuda() |
| |
| def score_sample(text, tokenizer, embedding_model, classifier_model, apply_sigmoid=False): |
| inputs = tokenizer([text], return_tensors="pt").to("cuda") |
| embeddings = embedding_model(**inputs).last_hidden_state.float().mean(1) |
| if apply_sigmoid: |
| score = F.sigmoid(classifier_model(embeddings)) |
| else: |
| score = classifier_model(embeddings) |
| return score.item() |
| |
| |
| text_en = "Question: How is bipolar disorder different from unipolar depression or 'regular' depression?\nAnswer: Both bipolar disorder and major depression are typically associated with depressive episodes. So both illnesses are accompanied by depressions. The difference is that in bipolar disorder people also have periods of elevation -- or severe irritability. We call these manic or hypomanic episodes." |
| mfwedu_score = score_sample(text_en, embedding_tokenizer, embedding_model, mfwedu_model, apply_sigmoid=False) |
| mdclm_score = score_sample(text_en, embedding_tokenizer, embedding_model, mdclm_model, apply_sigmoid=True) |
| print(f"{mfwedu_score:0.4f}") # 2.7353 (in [0-5]) |
| print(f"{mdclm_score:0.4f}") # 0.8463 (in [0-1]) |
| |
| |
| text_en = "Custom Wedding Gifts\nPersonalized photo frames, albums & keepsakes. Heirloom quality!\nCustom Engraved Journals\nHandmade in Florence Italy. Dozens of sizes and paper styles!" |
| mfwedu_score = score_sample(text_en, embedding_tokenizer, embedding_model, mfwedu_model, apply_sigmoid=False) |
| mdclm_score = score_sample(text_en, embedding_tokenizer, embedding_model, mdclm_model, apply_sigmoid=True) |
| print(f"{mfwedu_score:0.4f}") # -0.0370 (in [0-5]) |
| print(f"{mdclm_score:0.4f}") # 0.0000 (in [0-1]) |
| ``` |
|
|
| ## Citation information |
| ``` |
| @misc{sabolčec2026adaptingenglishqualityclassifiers, |
| title={Adapting English Quality Classifiers for Multilingual LLM Pretraining Data Selection}, |
| author={Vinko Sabolčec and Bettina Messmer and Yassine Turki and Martin Jaggi}, |
| year={2026}, |
| eprint={2610.11585}, |
| archivePrefix={arXiv}, |
| primaryClass={cs.CL}, |
| url={https://arxiv.org/abs/2610.11585}, |
| } |
| ``` |
