| | --- |
| | license: mit |
| | datasets: |
| | - jhu-clsp/mmbert-decay |
| | - jhu-clsp/mmbert-midtraining |
| | - jhu-clsp/mmbert-pretrain-p1-fineweb2-langs |
| | - jhu-clsp/mmbert-pretrain-p2-fineweb2-remaining |
| | - jhu-clsp/mmbert-pretrain-p3-others |
| | pipeline_tag: fill-mask |
| | library_name: transformers |
| | language: |
| | - aai |
| | - aak |
| | - aau |
| | - aaz |
| | - aba |
| | - abi |
| | - abk |
| | - abn |
| | - abq |
| | - abs |
| | - abt |
| | - abx |
| | - aby |
| | - abz |
| | - aca |
| | - acd |
| | - ace |
| | - acf |
| | - ach |
| | - acm |
| | - acn |
| | - acr |
| | - acu |
| | - ada |
| | - ade |
| | - adh |
| | - adi |
| | - adj |
| | - adl |
| | - ady |
| | - adz |
| | - aeb |
| | - aer |
| | - aeu |
| | - aey |
| | - afr |
| | - agd |
| | - agg |
| | - agm |
| | - agn |
| | - agr |
| | - agt |
| | - agu |
| | - agw |
| | - agx |
| | - aha |
| | - ahk |
| | - aia |
| | - aii |
| | - aim |
| | - ain |
| | - ajg |
| | - aji |
| | - ajz |
| | - akb |
| | - ake |
| | - akh |
| | - akp |
| | - alj |
| | - aln |
| | - alp |
| | - alq |
| | - als |
| | - alt |
| | - aly |
| | - alz |
| | - ame |
| | - amf |
| | - amh |
| | - ami |
| | - amk |
| | - amm |
| | - amn |
| | - amp |
| | - amr |
| | - amu |
| | - amx |
| | - ang |
| | - anm |
| | - ann |
| | - anp |
| | - anv |
| | - any |
| | - aoi |
| | - aoj |
| | - aom |
| | - aoz |
| | - apb |
| | - apc |
| | - ape |
| | - apn |
| | - apr |
| | - apt |
| | - apu |
| | - apw |
| | - apy |
| | - apz |
| | - arb |
| | - are |
| | - arg |
| | - arl |
| | - arn |
| | - arp |
| | - arq |
| | - ars |
| | - ary |
| | - arz |
| | - asg |
| | - asm |
| | - aso |
| | - ast |
| | - ata |
| | - atb |
| | - atd |
| | - atg |
| | - ati |
| | - atj |
| | - atq |
| | - att |
| | - auc |
| | - aui |
| | - auy |
| | - ava |
| | - avk |
| | - avn |
| | - avt |
| | - avu |
| | - awa |
| | - awb |
| | - awx |
| | - ayo |
| | - ayp |
| | - ayr |
| | - azb |
| | - azg |
| | - azj |
| | - azz |
| | - bak |
| | - bam |
| | - ban |
| | - bao |
| | - bar |
| | - bas |
| | - bav |
| | - bba |
| | - bbb |
| | - bbc |
| | - bbj |
| | - bbk |
| | - bbo |
| | - bbr |
| | - bch |
| | - bci |
| | - bcl |
| | - bco |
| | - bcw |
| | - bdd |
| | - bdh |
| | - bdq |
| | - bea |
| | - bef |
| | - bel |
| | - bem |
| | - ben |
| | - beq |
| | - bew |
| | - bex |
| | - bfd |
| | - bfo |
| | - bgr |
| | - bgs |
| | - bgt |
| | - bgz |
| | - bhg |
| | - bhl |
| | - bho |
| | - bhp |
| | - bhw |
| | - bhz |
| | - bib |
| | - big |
| | - bim |
| | - bin |
| | - bis |
| | - biu |
| | - biv |
| | - bjn |
| | - bjp |
| | - bjr |
| | - bjv |
| | - bkd |
| | - bkl |
| | - bkq |
| | - bku |
| | - bkv |
| | - bla |
| | - blh |
| | - blk |
| | - blw |
| | - blz |
| | - bmh |
| | - bmk |
| | - bmq |
| | - bmr |
| | - bmu |
| | - bmv |
| | - bno |
| | - bnp |
| | - boa |
| | - bod |
| | - boj |
| | - bom |
| | - bon |
| | - bos |
| | - bov |
| | - box |
| | - bpr |
| | - bps |
| | - bpy |
| | - bqc |
| | - bqj |
| | - bqp |
| | - bre |
| | - brh |
| | - bru |
| | - brx |
| | - bsc |
| | - bsn |
| | - bsp |
| | - bsq |
| | - bss |
| | - btd |
| | - bth |
| | - bts |
| | - btt |
| | - btx |
| | - bud |
| | - bug |
| | - buk |
| | - bul |
| | - bum |
| | - bus |
| | - bvc |
| | - bvd |
| | - bvr |
| | - bvz |
| | - bwd |
| | - bwi |
| | - bwq |
| | - bwu |
| | - bxh |
| | - bxr |
| | - byr |
| | - byv |
| | - byx |
| | - bzd |
| | - bzh |
| | - bzi |
| | - bzj |
| | - caa |
| | - cab |
| | - cac |
| | - caf |
| | - cag |
| | - cak |
| | - cao |
| | - cap |
| | - caq |
| | - car |
| | - cas |
| | - cat |
| | - cav |
| | - cax |
| | - cbc |
| | - cbi |
| | - cbk |
| | - cbr |
| | - cbs |
| | - cbt |
| | - cbu |
| | - cbv |
| | - cce |
| | - cco |
| | - ccp |
| | - ceb |
| | - ceg |
| | - cek |
| | - ces |
| | - cfm |
| | - cgc |
| | - cgg |
| | - cha |
| | - chd |
| | - che |
| | - chf |
| | - chj |
| | - chk |
| | - cho |
| | - chq |
| | - chr |
| | - chu |
| | - chv |
| | - chw |
| | - chz |
| | - cjk |
| | - cjo |
| | - cjp |
| | - cjs |
| | - cjv |
| | - ckb |
| | - cko |
| | - ckt |
| | - cle |
| | - clu |
| | - cly |
| | - cme |
| | - cmn |
| | - cmo |
| | - cmr |
| | - cnh |
| | - cni |
| | - cnk |
| | - cnl |
| | - cnt |
| | - cnw |
| | - coe |
| | - cof |
| | - cok |
| | - con |
| | - cop |
| | - cor |
| | - cos |
| | - cot |
| | - cou |
| | - cpa |
| | - cpb |
| | - cpc |
| | - cpu |
| | - cpy |
| | - crh |
| | - crj |
| | - crk |
| | - crl |
| | - crm |
| | - crn |
| | - crs |
| | - crt |
| | - crx |
| | - csb |
| | - csk |
| | - cso |
| | - csw |
| | - csy |
| | - cta |
| | - ctd |
| | - cto |
| | - ctp |
| | - ctu |
| | - cub |
| | - cuc |
| | - cui |
| | - cuk |
| | - cul |
| | - cut |
| | - cux |
| | - cwe |
| | - cwt |
| | - cya |
| | - cym |
| | - czt |
| | - daa |
| | - dad |
| | - daf |
| | - dag |
| | - dah |
| | - dak |
| | - dan |
| | - dar |
| | - ddg |
| | - ddn |
| | - ded |
| | - des |
| | - deu |
| | - dga |
| | - dgc |
| | - dgi |
| | - dgr |
| | - dgz |
| | - dhg |
| | - dhm |
| | - dhv |
| | - did |
| | - dig |
| | - dik |
| | - diq |
| | - dis |
| | - diu |
| | - div |
| | - dje |
| | - djk |
| | - djr |
| | - dks |
| | - dln |
| | - dng |
| | - dnj |
| | - dnw |
| | - dob |
| | - doi |
| | - dop |
| | - dos |
| | - dow |
| | - drg |
| | - dru |
| | - dsb |
| | - dtb |
| | - dtp |
| | - dts |
| | - dty |
| | - dua |
| | - due |
| | - dug |
| | - duo |
| | - dur |
| | - dwr |
| | - dww |
| | - dyi |
| | - dyo |
| | - dyu |
| | - dzo |
| | - ebk |
| | - efi |
| | - eka |
| | - ekk |
| | - eko |
| | - ell |
| | - emi |
| | - eml |
| | - emp |
| | - enb |
| | - enl |
| | - enm |
| | - enq |
| | - enx |
| | - epo |
| | - eri |
| | - ese |
| | - esi |
| | - esk |
| | - ess |
| | - esu |
| | - eto |
| | - etr |
| | - etu |
| | - eus |
| | - eve |
| | - ewe |
| | - ewo |
| | - ext |
| | - eza |
| | - faa |
| | - fad |
| | - fai |
| | - fal |
| | - fan |
| | - fao |
| | - far |
| | - fas |
| | - fat |
| | - ffm |
| | - fij |
| | - fil |
| | - fin |
| | - fit |
| | - fkv |
| | - fmu |
| | - fon |
| | - for |
| | - fra |
| | - frd |
| | - fro |
| | - frp |
| | - frr |
| | - fry |
| | - fub |
| | - fud |
| | - fue |
| | - fuf |
| | - fuh |
| | - fuq |
| | - fur |
| | - fuv |
| | - gaa |
| | - gag |
| | - gah |
| | - gai |
| | - gam |
| | - gaw |
| | - gaz |
| | - gbi |
| | - gbo |
| | - gbr |
| | - gcf |
| | - gcr |
| | - gde |
| | - gdg |
| | - gdn |
| | - gdr |
| | - geb |
| | - gej |
| | - gfk |
| | - ghs |
| | - gid |
| | - gil |
| | - giz |
| | - gjn |
| | - gkn |
| | - gla |
| | - gle |
| | - glg |
| | - glk |
| | - glv |
| | - gmh |
| | - gmv |
| | - gna |
| | - gnb |
| | - gnd |
| | - gng |
| | - gnn |
| | - gnw |
| | - goa |
| | - gof |
| | - gog |
| | - goh |
| | - gom |
| | - gor |
| | - gos |
| | - got |
| | - gqr |
| | - grc |
| | - grt |
| | - gso |
| | - gsw |
| | - gub |
| | - guc |
| | - gud |
| | - gug |
| | - guh |
| | - gui |
| | - guj |
| | - guk |
| | - gul |
| | - gum |
| | - gun |
| | - guo |
| | - guq |
| | - gur |
| | - guu |
| | - guw |
| | - gux |
| | - guz |
| | - gvc |
| | - gvf |
| | - gvl |
| | - gvn |
| | - gwi |
| | - gwr |
| | - gya |
| | - gym |
| | - gyr |
| | - hac |
| | - hae |
| | - hag |
| | - hak |
| | - hat |
| | - hav |
| | - haw |
| | - hay |
| | - hbo |
| | - hch |
| | - heb |
| | - heg |
| | - heh |
| | - her |
| | - hif |
| | - hig |
| | - hil |
| | - hin |
| | - hix |
| | - hla |
| | - hmo |
| | - hmr |
| | - hne |
| | - hnj |
| | - hnn |
| | - hns |
| | - hop |
| | - hot |
| | - hra |
| | - hrv |
| | - hrx |
| | - hsb |
| | - hto |
| | - hub |
| | - hui |
| | - hun |
| | - hus |
| | - huu |
| | - huv |
| | - hvn |
| | - hwc |
| | - hye |
| | - hyw |
| | - ian |
| | - iba |
| | - ibg |
| | - ibo |
| | - icr |
| | - ido |
| | - idu |
| | - ifa |
| | - ifb |
| | - ife |
| | - ifk |
| | - ifu |
| | - ify |
| | - ige |
| | - ign |
| | - ike |
| | - ikk |
| | - ikt |
| | - ikw |
| | - ilb |
| | - ile |
| | - ilo |
| | - imo |
| | - ina |
| | - inb |
| | - ind |
| | - inh |
| | - ino |
| | - iou |
| | - ipi |
| | - iqw |
| | - iri |
| | - irk |
| | - iry |
| | - isd |
| | - ish |
| | - isl |
| | - iso |
| | - ita |
| | - itv |
| | - ium |
| | - ivb |
| | - ivv |
| | - iws |
| | - ixl |
| | - izr |
| | - izz |
| | - jaa |
| | - jac |
| | - jae |
| | - jam |
| | - jav |
| | - jbo |
| | - jbu |
| | - jic |
| | - jiv |
| | - jmc |
| | - jpn |
| | - jra |
| | - jun |
| | - jvn |
| | - kaa |
| | - kab |
| | - kac |
| | - kak |
| | - kal |
| | - kam |
| | - kan |
| | - kao |
| | - kaq |
| | - kas |
| | - kat |
| | - kaz |
| | - kbc |
| | - kbd |
| | - kbh |
| | - kbm |
| | - kbo |
| | - kbp |
| | - kbq |
| | - kbr |
| | - kby |
| | - kca |
| | - kcg |
| | - kck |
| | - kdc |
| | - kde |
| | - kdh |
| | - kdi |
| | - kdj |
| | - kdl |
| | - kdr |
| | - kea |
| | - kei |
| | - kek |
| | - ken |
| | - keo |
| | - ker |
| | - kew |
| | - kez |
| | - kff |
| | - kgf |
| | - kgk |
| | - kgp |
| | - kgr |
| | - kha |
| | - khk |
| | - khm |
| | - khs |
| | - khz |
| | - kia |
| | - kij |
| | - kik |
| | - kin |
| | - kir |
| | - kiu |
| | - kix |
| | - kjb |
| | - kje |
| | - kjh |
| | - kjs |
| | - kkc |
| | - kki |
| | - kkj |
| | - kkl |
| | - kle |
| | - klt |
| | - klv |
| | - kmb |
| | - kmg |
| | - kmh |
| | - kmk |
| | - kmm |
| | - kmo |
| | - kmr |
| | - kms |
| | - kmu |
| | - kmy |
| | - knc |
| | - kne |
| | - knf |
| | - kng |
| | - knj |
| | - knk |
| | - kno |
| | - knv |
| | - knx |
| | - kny |
| | - kog |
| | - koi |
| | - koo |
| | - kor |
| | - kos |
| | - kpe |
| | - kpf |
| | - kpg |
| | - kpj |
| | - kpq |
| | - kpr |
| | - kpv |
| | - kpw |
| | - kpx |
| | - kpz |
| | - kqc |
| | - kqe |
| | - kqf |
| | - kql |
| | - kqn |
| | - kqo |
| | - kqp |
| | - kqs |
| | - kqw |
| | - kqy |
| | - krc |
| | - kri |
| | - krj |
| | - krl |
| | - kru |
| | - krx |
| | - ksb |
| | - ksc |
| | - ksd |
| | - ksf |
| | - ksh |
| | - ksj |
| | - ksp |
| | - ksr |
| | - kss |
| | - ksw |
| | - ktb |
| | - ktj |
| | - ktm |
| | - kto |
| | - ktu |
| | - ktz |
| | - kua |
| | - kub |
| | - kud |
| | - kue |
| | - kuj |
| | - kum |
| | - kup |
| | - kus |
| | - kvg |
| | - kvj |
| | - kvn |
| | - kwd |
| | - kwf |
| | - kwi |
| | - kwj |
| | - kwn |
| | - kwy |
| | - kxc |
| | - kxm |
| | - kxw |
| | - kyc |
| | - kyf |
| | - kyg |
| | - kyq |
| | - kyu |
| | - kyz |
| | - kze |
| | - kzf |
| | - kzj |
| | - lac |
| | - lad |
| | - lai |
| | - laj |
| | - lam |
| | - lao |
| | - lap |
| | - lat |
| | - lbb |
| | - lbe |
| | - lbj |
| | - lbk |
| | - lcm |
| | - lcp |
| | - ldi |
| | - ldn |
| | - lee |
| | - lef |
| | - leh |
| | - lem |
| | - leu |
| | - lew |
| | - lex |
| | - lez |
| | - lfn |
| | - lgg |
| | - lgl |
| | - lgm |
| | - lhi |
| | - lhu |
| | - lia |
| | - lid |
| | - lif |
| | - lij |
| | - lim |
| | - lin |
| | - lip |
| | - lis |
| | - lit |
| | - liv |
| | - ljp |
| | - lki |
| | - llb |
| | - lld |
| | - llg |
| | - lln |
| | - lmk |
| | - lmo |
| | - lmp |
| | - lnd |
| | - lob |
| | - loe |
| | - log |
| | - lok |
| | - lol |
| | - lom |
| | - loq |
| | - loz |
| | - lrc |
| | - lsi |
| | - lsm |
| | - ltg |
| | - ltz |
| | - lua |
| | - lub |
| | - luc |
| | - lud |
| | - lue |
| | - lug |
| | - lun |
| | - luo |
| | - lus |
| | - lvs |
| | - lwg |
| | - lwo |
| | - lww |
| | - lzh |
| | - maa |
| | - mad |
| | - maf |
| | - mag |
| | - mah |
| | - mai |
| | - maj |
| | - mak |
| | - mal |
| | - mam |
| | - maq |
| | - mar |
| | - mas |
| | - mau |
| | - mav |
| | - maw |
| | - maz |
| | - mbb |
| | - mbc |
| | - mbd |
| | - mbf |
| | - mbh |
| | - mbi |
| | - mbj |
| | - mbl |
| | - mbs |
| | - mbt |
| | - mca |
| | - mcb |
| | - mcd |
| | - mcf |
| | - mck |
| | - mcn |
| | - mco |
| | - mcp |
| | - mcq |
| | - mcu |
| | - mda |
| | - mdf |
| | - mdy |
| | - med |
| | - mee |
| | - mej |
| | - mek |
| | - men |
| | - meq |
| | - mer |
| | - met |
| | - meu |
| | - mev |
| | - mfe |
| | - mfg |
| | - mfh |
| | - mfi |
| | - mfk |
| | - mfq |
| | - mfy |
| | - mfz |
| | - mgc |
| | - mgh |
| | - mgo |
| | - mgr |
| | - mhi |
| | - mhl |
| | - mhr |
| | - mhw |
| | - mhx |
| | - mhy |
| | - mib |
| | - mic |
| | - mie |
| | - mif |
| | - mig |
| | - mih |
| | - mil |
| | - mim |
| | - min |
| | - mio |
| | - mip |
| | - miq |
| | - mir |
| | - mit |
| | - miy |
| | - miz |
| | - mjc |
| | - mjw |
| | - mkd |
| | - mkl |
| | - mkn |
| | - mks |
| | - mkz |
| | - mlh |
| | - mlp |
| | - mlt |
| | - mlu |
| | - mmn |
| | - mmo |
| | - mmx |
| | - mna |
| | - mnb |
| | - mnf |
| | - mni |
| | - mnk |
| | - mns |
| | - mnw |
| | - mnx |
| | - mny |
| | - moa |
| | - moc |
| | - mog |
| | - moh |
| | - mop |
| | - mor |
| | - mos |
| | - mox |
| | - mpg |
| | - mph |
| | - mpm |
| | - mpp |
| | - mps |
| | - mpt |
| | - mpx |
| | - mqb |
| | - mqj |
| | - mqy |
| | - mrg |
| | - mri |
| | - mrj |
| | - mrq |
| | - mrv |
| | - mrw |
| | - msb |
| | - msc |
| | - mse |
| | - msk |
| | - msy |
| | - mta |
| | - mtg |
| | - mti |
| | - mto |
| | - mtp |
| | - mua |
| | - mug |
| | - muh |
| | - mui |
| | - mup |
| | - mur |
| | - mus |
| | - mux |
| | - muy |
| | - mva |
| | - mvn |
| | - mvp |
| | - mwc |
| | - mwf |
| | - mwl |
| | - mwm |
| | - mwn |
| | - mwp |
| | - mwq |
| | - mwv |
| | - mww |
| | - mxb |
| | - mxp |
| | - mxq |
| | - mxt |
| | - mxv |
| | - mya |
| | - myb |
| | - myk |
| | - myu |
| | - myv |
| | - myw |
| | - myx |
| | - myy |
| | - mza |
| | - mzh |
| | - mzk |
| | - mzl |
| | - mzm |
| | - mzn |
| | - mzw |
| | - mzz |
| | - nab |
| | - naf |
| | - nah |
| | - nak |
| | - nap |
| | - naq |
| | - nas |
| | - nav |
| | - naw |
| | - nba |
| | - nbc |
| | - nbe |
| | - nbl |
| | - nbq |
| | - nbu |
| | - nca |
| | - nch |
| | - ncj |
| | - ncl |
| | - ncq |
| | - nct |
| | - ncu |
| | - ncx |
| | - ndc |
| | - nde |
| | - ndh |
| | - ndi |
| | - ndj |
| | - ndo |
| | - nds |
| | - ndz |
| | - neb |
| | - new |
| | - nfa |
| | - nfr |
| | - ngb |
| | - ngc |
| | - ngl |
| | - ngp |
| | - ngu |
| | - nhd |
| | - nhe |
| | - nhg |
| | - nhi |
| | - nhk |
| | - nho |
| | - nhr |
| | - nhu |
| | - nhw |
| | - nhx |
| | - nhy |
| | - nia |
| | - nif |
| | - nii |
| | - nij |
| | - nim |
| | - nin |
| | - nio |
| | - niu |
| | - niy |
| | - njb |
| | - njm |
| | - njn |
| | - njo |
| | - njz |
| | - nkf |
| | - nko |
| | - nld |
| | - nlg |
| | - nma |
| | - nmf |
| | - nmh |
| | - nmo |
| | - nmw |
| | - nmz |
| | - nnb |
| | - nng |
| | - nnh |
| | - nnl |
| | - nno |
| | - nnp |
| | - nnq |
| | - nnw |
| | - noa |
| | - nob |
| | - nod |
| | - nog |
| | - non |
| | - nop |
| | - not |
| | - nou |
| | - nov |
| | - nph |
| | - npi |
| | - npl |
| | - npo |
| | - npy |
| | - nqo |
| | - nre |
| | - nrf |
| | - nri |
| | - nrm |
| | - nsa |
| | - nse |
| | - nsm |
| | - nsn |
| | - nso |
| | - nss |
| | - nst |
| | - nsu |
| | - ntp |
| | - ntr |
| | - ntu |
| | - nuj |
| | - nus |
| | - nuy |
| | - nvm |
| | - nwb |
| | - nwi |
| | - nwx |
| | - nxd |
| | - nya |
| | - nyf |
| | - nyk |
| | - nyn |
| | - nyo |
| | - nyu |
| | - nyy |
| | - nza |
| | - nzi |
| | - nzm |
| | - obo |
| | - oci |
| | - ogo |
| | - ojb |
| | - oke |
| | - oku |
| | - okv |
| | - old |
| | - olo |
| | - omb |
| | - omw |
| | - ong |
| | - ons |
| | - ood |
| | - opm |
| | - orv |
| | - ory |
| | - oss |
| | - ota |
| | - otd |
| | - ote |
| | - otm |
| | - otn |
| | - oto |
| | - otq |
| | - ots |
| | - otw |
| | - oym |
| | - ozm |
| | - pab |
| | - pad |
| | - pag |
| | - pah |
| | - pam |
| | - pan |
| | - pao |
| | - pap |
| | - pau |
| | - pbb |
| | - pbc |
| | - pbi |
| | - pbt |
| | - pcd |
| | - pck |
| | - pcm |
| | - pdc |
| | - pdt |
| | - pem |
| | - pfe |
| | - pfl |
| | - phm |
| | - pib |
| | - pio |
| | - pir |
| | - pis |
| | - pjt |
| | - pkb |
| | - plg |
| | - pls |
| | - plt |
| | - plu |
| | - plw |
| | - pma |
| | - pmf |
| | - pmq |
| | - pms |
| | - pmx |
| | - pnb |
| | - pne |
| | - pnt |
| | - pny |
| | - poe |
| | - poh |
| | - poi |
| | - pol |
| | - pon |
| | - por |
| | - pos |
| | - pot |
| | - pov |
| | - poy |
| | - ppk |
| | - ppo |
| | - pps |
| | - prf |
| | - prg |
| | - pri |
| | - prq |
| | - pse |
| | - pss |
| | - ptp |
| | - ptu |
| | - pui |
| | - pwg |
| | - pwn |
| | - pww |
| | - pxm |
| | - qub |
| | - quc |
| | - quf |
| | - qug |
| | - quh |
| | - qul |
| | - qup |
| | - qus |
| | - quw |
| | - quy |
| | - quz |
| | - qva |
| | - qvc |
| | - qve |
| | - qvh |
| | - qvi |
| | - qvm |
| | - qvn |
| | - qvo |
| | - qvs |
| | - qvw |
| | - qvz |
| | - qwh |
| | - qxh |
| | - qxl |
| | - qxn |
| | - qxo |
| | - qxr |
| | - rad |
| | - rai |
| | - rap |
| | - rar |
| | - rav |
| | - raw |
| | - rcf |
| | - rej |
| | - rel |
| | - rgu |
| | - rhg |
| | - ria |
| | - rim |
| | - rjs |
| | - rkb |
| | - rmc |
| | - rme |
| | - rml |
| | - rmn |
| | - rmo |
| | - rmq |
| | - rmy |
| | - rnd |
| | - rng |
| | - rnl |
| | - roh |
| | - ron |
| | - roo |
| | - rop |
| | - row |
| | - rro |
| | - rtm |
| | - rub |
| | - rue |
| | - ruf |
| | - rug |
| | - run |
| | - rup |
| | - rus |
| | - rwo |
| | - sab |
| | - sag |
| | - sah |
| | - san |
| | - sas |
| | - sat |
| | - sba |
| | - sbd |
| | - sbe |
| | - sbl |
| | - sbs |
| | - sby |
| | - sck |
| | - scn |
| | - sco |
| | - sda |
| | - sdc |
| | - sdh |
| | - sdo |
| | - sdq |
| | - seh |
| | - ses |
| | - sey |
| | - sfw |
| | - sgb |
| | - sgc |
| | - sgh |
| | - sgs |
| | - sgw |
| | - sgz |
| | - shi |
| | - shk |
| | - shn |
| | - shp |
| | - shu |
| | - sid |
| | - sig |
| | - sil |
| | - sim |
| | - sin |
| | - sja |
| | - sjo |
| | - sju |
| | - skg |
| | - skr |
| | - sld |
| | - slk |
| | - sll |
| | - slv |
| | - sma |
| | - sme |
| | - smj |
| | - smk |
| | - sml |
| | - smn |
| | - smo |
| | - sms |
| | - smt |
| | - sna |
| | - snc |
| | - snd |
| | - snf |
| | - snn |
| | - snp |
| | - snw |
| | - sny |
| | - soe |
| | - som |
| | - sop |
| | - soq |
| | - sot |
| | - soy |
| | - spa |
| | - spl |
| | - spm |
| | - spp |
| | - sps |
| | - spy |
| | - srd |
| | - sri |
| | - srm |
| | - srn |
| | - srp |
| | - srq |
| | - srr |
| | - ssd |
| | - ssg |
| | - ssw |
| | - ssx |
| | - stn |
| | - stp |
| | - stq |
| | - sua |
| | - suc |
| | - sue |
| | - suk |
| | - sun |
| | - sur |
| | - sus |
| | - suz |
| | - swb |
| | - swc |
| | - swe |
| | - swg |
| | - swh |
| | - swk |
| | - swp |
| | - sxb |
| | - sxn |
| | - syb |
| | - syc |
| | - syl |
| | - szl |
| | - szy |
| | - tab |
| | - tac |
| | - tah |
| | - taj |
| | - tam |
| | - tap |
| | - taq |
| | - tar |
| | - tat |
| | - tav |
| | - taw |
| | - tay |
| | - tbc |
| | - tbg |
| | - tbk |
| | - tbl |
| | - tbo |
| | - tbw |
| | - tby |
| | - tbz |
| | - tca |
| | - tcc |
| | - tcf |
| | - tcs |
| | - tcy |
| | - tcz |
| | - ted |
| | - tee |
| | - tel |
| | - tem |
| | - teo |
| | - ter |
| | - tet |
| | - tew |
| | - tfr |
| | - tgk |
| | - tgo |
| | - tgp |
| | - tha |
| | - thk |
| | - thl |
| | - tif |
| | - tig |
| | - tih |
| | - tik |
| | - tim |
| | - tir |
| | - tiv |
| | - tiy |
| | - tke |
| | - tkl |
| | - tkr |
| | - tku |
| | - tlb |
| | - tlf |
| | - tlh |
| | - tlj |
| | - tll |
| | - tly |
| | - tmc |
| | - tmd |
| | - tna |
| | - tnc |
| | - tnk |
| | - tnn |
| | - tnp |
| | - tnr |
| | - tob |
| | - toc |
| | - tod |
| | - tog |
| | - toh |
| | - toi |
| | - toj |
| | - tok |
| | - ton |
| | - too |
| | - top |
| | - tos |
| | - tpa |
| | - tpi |
| | - tpm |
| | - tpp |
| | - tpt |
| | - tpw |
| | - tpz |
| | - tqo |
| | - trc |
| | - trn |
| | - tro |
| | - trp |
| | - trq |
| | - trs |
| | - trv |
| | - tsc |
| | - tsg |
| | - tsn |
| | - tso |
| | - tsw |
| | - tsz |
| | - ttc |
| | - tte |
| | - ttj |
| | - ttq |
| | - tuc |
| | - tue |
| | - tuf |
| | - tui |
| | - tuk |
| | - tul |
| | - tum |
| | - tuo |
| | - tur |
| | - tuv |
| | - tvk |
| | - tvl |
| | - twi |
| | - twu |
| | - twx |
| | - txq |
| | - txu |
| | - tyv |
| | - tzh |
| | - tzj |
| | - tzl |
| | - tzm |
| | - tzo |
| | - ubr |
| | - ubu |
| | - udm |
| | - udu |
| | - uig |
| | - ukr |
| | - umb |
| | - upv |
| | - ura |
| | - urb |
| | - urd |
| | - urh |
| | - uri |
| | - urk |
| | - urt |
| | - urw |
| | - ury |
| | - usa |
| | - usp |
| | - uth |
| | - uvh |
| | - uvl |
| | - uzn |
| | - uzs |
| | - vag |
| | - vap |
| | - var |
| | - vec |
| | - ven |
| | - vep |
| | - vid |
| | - vie |
| | - viv |
| | - vls |
| | - vmk |
| | - vmw |
| | - vmy |
| | - vol |
| | - vot |
| | - vro |
| | - vun |
| | - vut |
| | - waj |
| | - wal |
| | - wap |
| | - war |
| | - wat |
| | - way |
| | - wba |
| | - wbm |
| | - wbp |
| | - wed |
| | - wer |
| | - wes |
| | - wew |
| | - whg |
| | - whk |
| | - wib |
| | - wim |
| | - wiu |
| | - wln |
| | - wls |
| | - wlv |
| | - wlx |
| | - wmt |
| | - wmw |
| | - wnc |
| | - wnu |
| | - wob |
| | - wol |
| | - wos |
| | - wrk |
| | - wrs |
| | - wsg |
| | - wsk |
| | - wuu |
| | - wuv |
| | - wwa |
| | - xal |
| | - xav |
| | - xbi |
| | - xbr |
| | - xed |
| | - xho |
| | - xla |
| | - xmf |
| | - xmm |
| | - xmv |
| | - xnn |
| | - xog |
| | - xon |
| | - xrb |
| | - xsb |
| | - xsi |
| | - xsm |
| | - xsr |
| | - xsu |
| | - xtd |
| | - xtm |
| | - xtn |
| | - xuo |
| | - yaa |
| | - yad |
| | - yal |
| | - yam |
| | - yan |
| | - yao |
| | - yap |
| | - yaq |
| | - yat |
| | - yaz |
| | - ybb |
| | - yby |
| | - ycn |
| | - ydd |
| | - yim |
| | - yka |
| | - yle |
| | - yli |
| | - yml |
| | - yom |
| | - yon |
| | - yor |
| | - yrb |
| | - yre |
| | - yrk |
| | - yrl |
| | - yss |
| | - yua |
| | - yue |
| | - yuj |
| | - yup |
| | - yut |
| | - yuw |
| | - yuz |
| | - yva |
| | - zaa |
| | - zab |
| | - zac |
| | - zad |
| | - zae |
| | - zai |
| | - zam |
| | - zao |
| | - zar |
| | - zas |
| | - zat |
| | - zav |
| | - zaw |
| | - zca |
| | - zdj |
| | - zea |
| | - zgh |
| | - zia |
| | - ziw |
| | - zne |
| | - zom |
| | - zos |
| | - zpa |
| | - zpc |
| | - zpg |
| | - zpi |
| | - zpj |
| | - zpl |
| | - zpm |
| | - zpo |
| | - zpq |
| | - zpt |
| | - zpu |
| | - zpv |
| | - zpz |
| | - zsm |
| | - zsr |
| | - ztq |
| | - zty |
| | - zul |
| | - zyb |
| | - zyp |
| | --- |
| | |
| | # mmBERT: A Modern Multilingual Encoder |
| |
|
| | [](https://opensource.org/licenses/MIT) |
| | [](https://arxiv.org/abs/2509.06888) |
| | [](https://huggingface.co/jhu-clsp/mmBERT-base) |
| | [](https://huggingface.co/collections/jhu-clsp/mmbert-a-modern-multilingual-encoder-68b725831d7c6e3acc435ed4) |
| | [](https://github.com/jhu-clsp/mmBERT) |
| |
|
| | > TL;DR: A state-of-the-art multilingual encoder trained on 3T+ tokens across 1800+ languages, introducing novel techniques for learning low-resource languages during the decay phase. |
| |
|
| | mmBERT is a modern multilingual encoder that significantly outperforms previous generation models like XLM-R on classification, embedding, and retrieval tasks. Built on the ModernBERT architecture with novel multilingual training innovations, mmBERT demonstrates that low-resource languages can be effectively learned during the decay phase of training. It is also significantly faster than any previous multilingual encoder. |
| |
|
| | ## Table of Contents |
| | - [Highlights](#highlights) |
| | - [Quick Start](#quick-start) |
| | - [Model Description](#model-description) |
| | - [Novel Training Innovations](#novel-training-innovations) |
| | - [Model Family](#model-family) |
| | - [Training Data](#training-data) |
| | - [Usage Examples](#usage-examples) |
| | - [Fine-tuning Examples](#fine-tuning-examples) |
| | - [Model Architecture](#model-architecture) |
| | - [Citation](#citation) |
| |
|
| |
|
| | ## Quick Start |
| |
|
| | ### Installation |
| | ```bash |
| | pip install torch>=1.9.0 |
| | pip install transformers>=4.21.0 |
| | ``` |
| |
|
| | ### Usage |
| |
|
| | ```python |
| | from transformers import AutoTokenizer, AutoModel |
| | |
| | tokenizer = AutoTokenizer.from_pretrained("jhu-clsp/mmBERT-base") |
| | model = AutoModel.from_pretrained("jhu-clsp/mmBERT-base") |
| | |
| | inputs = tokenizer("Hello world", return_tensors="pt") |
| | outputs = model(**inputs) |
| | ``` |
| |
|
| | ## Model Description |
| |
|
| | mmBERT represents the first significant advancement over XLM-R for massively multilingual encoder models. Key features include: |
| |
|
| | 1. **Massive Language Coverage** - Trained on over 1800 languages with progressive inclusion strategy |
| | 2. **Modern Architecture** - Built on ModernBERT foundation with Flash Attention 2 and unpadding techniques |
| | 3. **Novel Training Recipe** - Introduces inverse mask scheduling and temperature sampling |
| | 4. **Open Training Data** - Complete 3T+ token dataset publicly available |
| | 5. **Decay Phase Innovation** - Demonstrates effective learning of low-resource languages in final training phase |
| |
|
| | The model uses bidirectional attention with masked language modeling objectives, optimized specifically for multilingual understanding and cross-lingual transfer. |
| |
|
| | ## Novel Training Innovations |
| |
|
| | **Progressive Language Addition**: Start with 60 high-resource languages, expand to 110 mid-resource languages, then include all 1833 languages in decay phase. |
| |
|
| | **Inverse Mask Schedule**: Reduce mask ratio from 30% → 15% → 5% across training phases for progressively refined learning. |
| |
|
| | **Inverse Temperature Sampling**: Adjust multilingual sampling from high-resource bias (τ=0.7) to uniform sampling (τ=0.3). |
| |
|
| | **Model Merging**: Combine English-focused, high-resource, and all-language decay variants using TIES merging. |
| |
|
| | ## Model Family |
| |
|
| | | Model | Total Params | Non-embed Params | Languages | Download | |
| | |:------|:-------------|:------------------|:----------|:---------| |
| | | [mmBERT-small](https://huggingface.co/jhu-clsp/mmBERT-small) | 140M | 42M | 1800+ | [](https://huggingface.co/jhu-clsp/mmBERT-small) | |
| | | [mmBERT-base](https://huggingface.co/jhu-clsp/mmBERT-base) | 307M | 110M | 1800+ | [](https://huggingface.co/jhu-clsp/mmBERT-base) | |
| |
|
| | ## Training Data |
| |
|
| | mmBERT training data is publicly available across different phases: |
| |
|
| | | Phase | Dataset | Tokens | Description | |
| | |:------|:--------|:-------|:------------| |
| | | Pre-training P1 | [mmbert-pretrain-p1](https://huggingface.co/datasets/jhu-clsp/mmbert-pretrain-p1-fineweb2-langs) | 2.3T | 60 languages, foundational training | |
| | | Pre-training P2 | [mmbert-pretrain-p2](https://huggingface.co/datasets/jhu-clsp/mmBERT-pretrain-p2-fineweb2-remaining) | - | Extension data for pre-training phase | |
| | | Pre-training P3 | [mmbert-pretrain-p3](https://huggingface.co/datasets/jhu-clsp/mmBERT-pretrain-p3-others) | - | Final pre-training data | |
| | | Mid-training | [mmbert-midtraining](https://huggingface.co/datasets/jhu-clsp/mmbert-midtraining-data) | 600B | 110 languages, context extension to 8K | |
| | | Decay Phase | [mmbert-decay](https://huggingface.co/datasets/jhu-clsp/mmbert-decay-data) | 100B | 1833 languages, premium quality | |
| |
|
| | **Data Sources**: Filtered DCLM (English), FineWeb2 (multilingual), FineWeb2-HQ (20 high-resource languages), Wikipedia (MegaWika), code repositories (StarCoder, ProLong), academic papers (ArXiv, PeS2o), and community discussions (StackExchange). |
| |
|
| | ## Model Architecture |
| |
|
| | | Parameter | mmBERT-small | mmBERT-base | |
| | |:----------|:-------------|:------------| |
| | | Layers | 22 | 22 | |
| | | Hidden Size | 384 | 768 | |
| | | Intermediate Size | 1152 | 1152 | |
| | | Attention Heads | 6 | 12 | |
| | | Total Parameters | 140M | 307M | |
| | | Non-embedding Parameters | 42M | 110M | |
| | | Max Sequence Length | 8192 | 8192 | |
| | | Vocabulary Size | 256,000 | 256,000 | |
| | | Tokenizer | Gemma 2 | Gemma 2 | |
| |
|
| | ## Usage Examples |
| |
|
| | ### Masked Language Modeling |
| |
|
| | ```python |
| | from transformers import AutoTokenizer, AutoModelForMaskedLM |
| | import torch |
| | |
| | tokenizer = AutoTokenizer.from_pretrained("jhu-clsp/mmBERT-base") |
| | model = AutoModelForMaskedLM.from_pretrained("jhu-clsp/mmBERT-base") |
| | |
| | def predict_masked_token(text): |
| | inputs = tokenizer(text, return_tensors="pt") |
| | with torch.no_grad(): |
| | outputs = model(**inputs) |
| | |
| | mask_indices = torch.where(inputs["input_ids"] == tokenizer.mask_token_id) |
| | predictions = outputs.logits[mask_indices] |
| | top_tokens = torch.topk(predictions, 5, dim=-1) |
| | |
| | return [tokenizer.decode(token) for token in top_tokens.indices[0]] |
| | |
| | # Works across languages |
| | texts = [ |
| | "The capital of France is <mask>.", |
| | "La capital de España es <mask>.", |
| | "Die Hauptstadt von Deutschland ist <mask>." |
| | ] |
| | |
| | for text in texts: |
| | predictions = predict_masked_token(text) |
| | print(f"Text: {text}") |
| | print(f"Predictions: {predictions}") |
| | ``` |
| |
|
| | ### Cross-lingual Embeddings |
| |
|
| | ```python |
| | from transformers import AutoTokenizer, AutoModel |
| | import torch |
| | from sklearn.metrics.pairwise import cosine_similarity |
| | |
| | tokenizer = AutoTokenizer.from_pretrained("jhu-clsp/mmBERT-base") |
| | model = AutoModel.from_pretrained("jhu-clsp/mmBERT-base") |
| | |
| | def get_embeddings(texts): |
| | inputs = tokenizer(texts, padding=True, truncation=True, return_tensors="pt") |
| | |
| | with torch.no_grad(): |
| | outputs = model(**inputs) |
| | embeddings = outputs.last_hidden_state.mean(dim=1) |
| | |
| | return embeddings.numpy() |
| | |
| | multilingual_texts = [ |
| | "Artificial intelligence is transforming technology", |
| | "La inteligencia artificial está transformando la tecnología", |
| | "L'intelligence artificielle transforme la technologie", |
| | "人工智能正在改变技术" |
| | ] |
| | |
| | embeddings = get_embeddings(multilingual_texts) |
| | similarities = cosine_similarity(embeddings) |
| | print("Cross-lingual similarity matrix:") |
| | print(similarities) |
| | ``` |
| |
|
| | ## Fine-tuning Examples |
| |
|
| | ### Dense Retrieval with Sentence Transformers |
| |
|
| | <details> |
| | <summary>Click to expand dense retrieval fine-tuning example</summary> |
| |
|
| | ```python |
| | import argparse |
| | from datasets import load_dataset |
| | from sentence_transformers import ( |
| | SentenceTransformer, |
| | SentenceTransformerTrainer, |
| | SentenceTransformerTrainingArguments, |
| | ) |
| | from sentence_transformers.evaluation import TripletEvaluator |
| | from sentence_transformers.losses import CachedMultipleNegativesRankingLoss |
| | from sentence_transformers.training_args import BatchSamplers |
| | |
| | def main(): |
| | parser = argparse.ArgumentParser() |
| | parser.add_argument("--lr", type=float, default=8e-5) |
| | parser.add_argument("--model_name", type=str, default="jhu-clsp/mmBERT-base") |
| | args = parser.parse_args() |
| | |
| | lr = args.lr |
| | model_name = args.model_name |
| | model_shortname = model_name.split("/")[-1] |
| | |
| | model = SentenceTransformer(model_name) |
| | |
| | dataset = load_dataset( |
| | "sentence-transformers/msmarco-co-condenser-margin-mse-sym-mnrl-mean-v1", |
| | "triplet-hard", |
| | split="train", |
| | ) |
| | dataset_dict = dataset.train_test_split(test_size=1_000, seed=12) |
| | train_dataset = dataset_dict["train"].select(range(1_250_000)) |
| | eval_dataset = dataset_dict["test"] |
| | |
| | loss = CachedMultipleNegativesRankingLoss(model, mini_batch_size=16) |
| | run_name = f"{model_shortname}-DPR-{lr}" |
| | |
| | training_args = SentenceTransformerTrainingArguments( |
| | output_dir=f"output/{model_shortname}/{run_name}", |
| | num_train_epochs=1, |
| | per_device_train_batch_size=512, |
| | per_device_eval_batch_size=512, |
| | warmup_ratio=0.05, |
| | fp16=False, |
| | bf16=True, |
| | batch_sampler=BatchSamplers.NO_DUPLICATES, |
| | learning_rate=lr, |
| | save_strategy="steps", |
| | save_steps=500, |
| | save_total_limit=2, |
| | logging_steps=500, |
| | run_name=run_name, |
| | ) |
| | |
| | dev_evaluator = TripletEvaluator( |
| | anchors=eval_dataset["query"], |
| | positives=eval_dataset["positive"], |
| | negatives=eval_dataset["negative"], |
| | name="msmarco-co-condenser-dev", |
| | ) |
| | dev_evaluator(model) |
| | |
| | trainer = SentenceTransformerTrainer( |
| | model=model, |
| | args=training_args, |
| | train_dataset=train_dataset, |
| | eval_dataset=eval_dataset, |
| | loss=loss, |
| | evaluator=dev_evaluator, |
| | ) |
| | trainer.train() |
| | |
| | model.save_pretrained(f"output/{model_shortname}/{run_name}/final") |
| | model.push_to_hub(run_name, private=False) |
| | |
| | if __name__ == "__main__": |
| | main() |
| | ``` |
| |
|
| | </details> |
| |
|
| | ### Cross-lingual Classification |
| |
|
| | <details> |
| | <summary>Click to expand multilingual classification fine-tuning example</summary> |
| |
|
| | ```python |
| | from transformers import ( |
| | AutoTokenizer, |
| | AutoModelForSequenceClassification, |
| | TrainingArguments, |
| | Trainer |
| | ) |
| | from datasets import load_dataset |
| | import numpy as np |
| | from sklearn.metrics import accuracy_score, f1_score |
| | |
| | def compute_metrics(eval_pred): |
| | predictions, labels = eval_pred |
| | predictions = np.argmax(predictions, axis=1) |
| | return { |
| | 'accuracy': accuracy_score(labels, predictions), |
| | 'f1': f1_score(labels, predictions, average='weighted') |
| | } |
| | |
| | def main(): |
| | model_name = "jhu-clsp/mmBERT-base" |
| | |
| | tokenizer = AutoTokenizer.from_pretrained(model_name) |
| | model = AutoModelForSequenceClassification.from_pretrained( |
| | model_name, |
| | num_labels=3 |
| | ) |
| | |
| | dataset = load_dataset("xnli", "all_languages") |
| | |
| | def tokenize_function(examples): |
| | texts = [f"{p} {tokenizer.sep_token} {h}" |
| | for p, h in zip(examples["premise"], examples["hypothesis"])] |
| | |
| | return tokenizer( |
| | texts, |
| | truncation=True, |
| | padding=True, |
| | max_length=512 |
| | ) |
| | |
| | train_dataset = dataset["train"].map(tokenize_function, batched=True) |
| | eval_dataset = dataset["validation"].map(tokenize_function, batched=True) |
| | |
| | training_args = TrainingArguments( |
| | output_dir="./mmbert-xnli", |
| | learning_rate=3e-5, |
| | per_device_train_batch_size=32, |
| | per_device_eval_batch_size=32, |
| | num_train_epochs=3, |
| | weight_decay=0.01, |
| | evaluation_strategy="epoch", |
| | save_strategy="epoch", |
| | load_best_model_at_end=True, |
| | metric_for_best_model="f1", |
| | greater_is_better=True, |
| | ) |
| | |
| | trainer = Trainer( |
| | model=model, |
| | args=training_args, |
| | train_dataset=train_dataset, |
| | eval_dataset=eval_dataset, |
| | compute_metrics=compute_metrics, |
| | ) |
| | |
| | trainer.train() |
| | |
| | if __name__ == "__main__": |
| | main() |
| | ``` |
| |
|
| | </details> |
| |
|
| | ### Multilingual Reranking |
| |
|
| | <details> |
| | <summary>Click to expand multilingual reranking fine-tuning example</summary> |
| |
|
| | ```python |
| | import logging |
| | from datasets import load_dataset |
| | from sentence_transformers.cross_encoder import ( |
| | CrossEncoder, |
| | CrossEncoderModelCardData, |
| | CrossEncoderTrainer, |
| | CrossEncoderTrainingArguments, |
| | ) |
| | from sentence_transformers.cross_encoder.evaluation import CrossEncoderNanoBEIREvaluator |
| | from sentence_transformers.cross_encoder.losses import BinaryCrossEntropyLoss |
| | from sentence_transformers.util import mine_hard_negatives |
| | from sentence_transformers import SentenceTransformer |
| | import torch |
| | |
| | def main(): |
| | model_name = "jhu-clsp/mmBERT-base" |
| | train_batch_size = 32 |
| | num_epochs = 2 |
| | num_hard_negatives = 7 |
| | |
| | model = CrossEncoder( |
| | model_name, |
| | model_card_data=CrossEncoderModelCardData( |
| | language="multilingual", |
| | license="mit", |
| | ), |
| | ) |
| | |
| | full_dataset = load_dataset("sentence-transformers/gooaq", split="train").select(range(50_000)) |
| | dataset_dict = full_dataset.train_test_split(test_size=1_000, seed=42) |
| | train_dataset = dataset_dict["train"] |
| | eval_dataset = dataset_dict["test"] |
| | |
| | embedding_model = SentenceTransformer("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", device="cpu") |
| | hard_train_dataset = mine_hard_negatives( |
| | train_dataset, |
| | embedding_model, |
| | num_negatives=num_hard_negatives, |
| | margin=0, |
| | range_min=0, |
| | range_max=100, |
| | sampling_strategy="top", |
| | batch_size=2048, |
| | output_format="labeled-pair", |
| | use_faiss=True, |
| | ) |
| | |
| | loss = BinaryCrossEntropyLoss(model=model, pos_weight=torch.tensor(num_hard_negatives)) |
| | |
| | nano_beir_evaluator = CrossEncoderNanoBEIREvaluator( |
| | dataset_names=["msmarco", "nfcorpus", "nq"], |
| | batch_size=train_batch_size, |
| | ) |
| | |
| | args = CrossEncoderTrainingArguments( |
| | output_dir="./mmbert-reranker", |
| | num_train_epochs=num_epochs, |
| | per_device_train_batch_size=train_batch_size, |
| | per_device_eval_batch_size=train_batch_size, |
| | learning_rate=2e-5, |
| | warmup_ratio=0.1, |
| | fp16=False, |
| | bf16=True, |
| | dataloader_num_workers=4, |
| | load_best_model_at_end=True, |
| | metric_for_best_model="eval_msmarco_ndcg@10", |
| | eval_strategy="steps", |
| | eval_steps=1000, |
| | save_strategy="steps", |
| | save_steps=1000, |
| | save_total_limit=2, |
| | logging_steps=200, |
| | seed=42, |
| | ) |
| | |
| | trainer = CrossEncoderTrainer( |
| | model=model, |
| | args=args, |
| | train_dataset=hard_train_dataset, |
| | loss=loss, |
| | evaluator=nano_beir_evaluator, |
| | ) |
| | trainer.train() |
| | |
| | model.save_pretrained("./mmbert-reranker/final") |
| | |
| | if __name__ == "__main__": |
| | main() |
| | ``` |
| |
|
| | </details> |
| |
|
| | ## Training Data |
| |
|
| | mmBERT was trained on a carefully curated 3T+ token multilingual dataset: |
| |
|
| | | Phase | Dataset | Description | |
| | |:------|:--------|:------------| |
| | | [Pre-training P1](https://huggingface.co/datasets/jhu-clsp/mmbert-pretrain-p1-fineweb2-langs) | 2.3T tokens | 60 languages, diverse data mixture | |
| | | [Pre-training P2](https://huggingface.co/datasets/jhu-clsp/mmbert-pretrain-p2-fineweb2-langs) | - | Extension data for pre-training | |
| | | [Pre-training P3](https://huggingface.co/datasets/jhu-clsp/mmbert-pretrain-p3-fineweb2-langs) | - | Final pre-training data | |
| | | [Mid-training](https://huggingface.co/datasets/jhu-clsp/mmbert-midtraining-data) | 600B tokens | 110 languages, context extension | |
| | | [Decay Phase](https://huggingface.co/datasets/jhu-clsp/mmbert-decay-data) | 100B tokens | 1833 languages, premium quality | |
| |
|
| | **Primary Sources:** |
| | - **Filtered DCLM**: High-quality English content |
| | - **FineWeb2**: Broad multilingual web coverage (1800+ languages) |
| | - **FineWeb2-HQ**: Filtered subset of 20 high-resource languages |
| | - **Code**: StarCoder and ProLong repositories |
| | - **Academic**: ArXiv papers and PeS2o scientific content |
| | - **Reference**: Wikipedia (MegaWika) and textbooks |
| | - **Community**: StackExchange discussions |
| |
|
| |
|
| | ## Citation |
| |
|
| | If you use mmBERT in your research, please cite our work: |
| |
|
| | ```bibtex |
| | @misc{marone2025mmbertmodernmultilingualencoder, |
| | title={mmBERT: A Modern Multilingual Encoder with Annealed Language Learning}, |
| | author={Marc Marone and Orion Weller and William Fleshman and Eugene Yang and Dawn Lawrie and Benjamin Van Durme}, |
| | year={2025}, |
| | eprint={2509.06888}, |
| | archivePrefix={arXiv}, |
| | primaryClass={cs.CL}, |
| | url={https://arxiv.org/abs/2509.06888}, |
| | } |
| | ``` |
