| "citation": "@inproceedings{ortiz-suarez-etal-2020-monolingual,\n title = \"A Monolingual Approach to Contextualized Word Embeddings for Mid-Resource Languages\",\n author = \"Ortiz Su{'a}rez, Pedro Javier and\n Romary, Laurent and\n Sagot, Benoit\",\n booktitle = \"Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics\",\n month = jul,\n year = \"2020\",\n address = \"Online\",\n publisher = \"Association for Computational Linguistics\",\n url = \"https://www.aclweb.org/anthology/2020.acl-main.156\",\n pages = \"1703--1714\",\n abstract = \"We use the multilingual OSCAR corpus, extracted from Common Crawl via language classification, filtering and cleaning, to train monolingual contextualized word embeddings (ELMo) for five mid-resource languages. We then compare the performance of OSCAR-based and Wikipedia-based ELMo embeddings for these languages on the part-of-speech tagging and parsing tasks. We show that, despite the noise in the Common-Crawl-based OSCAR data, embeddings trained on OSCAR perform much better than monolingual embeddings trained on Wikipedia. They actually equal or improve the current state of the art in tagging and parsing for all five languages. In particular, they also improve over multilingual Wikipedia-based contextual embeddings (multilingual BERT), which almost always constitutes the previous state of the art, thereby showing that the benefit of a larger, more diverse corpus surpasses the cross-lingual benefit of multilingual embedding architectures.\",\n}\n\n@inproceedings{OrtizSuarezSagotRomary2019,\n author = {Pedro Javier {Ortiz Su{'a}rez} and Benoit Sagot and Laurent Romary},\n title = {Asynchronous pipelines for processing huge corpora on medium to low resource infrastructures},\n series = {Proceedings of the Workshop on Challenges in the Management of Large Corpora (CMLC-7) 2019. Cardiff, 22nd July 2019},\n editor = {Piotr Ba\u0144ski and Adrien Barbaresi and Hanno Biber and Evelyn Breiteneder and Simon Clematide and Marc Kupietz and Harald L{\"u}ngen and Caroline Iliadi},\n publisher = {Leibniz-Institut f{\"u}r Deutsche Sprache},\n address = {Mannheim},\n doi = {10.14618/ids-pub-9021},\n url = {http://nbn-resolving.de/urn:nbn:de:bsz:mh39-90215},\n pages = {9 -- 16},\n year = {2019},\n abstract = {Common Crawl is a considerably large, heterogeneous multilingual corpus comprised of crawled documents from the internet, surpassing 20TB of data and distributed as a set of more than 50 thousand plain text files where each contains many documents written in a wide variety of languages. Even though each document has a metadata block associated to it, this data lacks any information about the language in which each document is written, making it extremely difficult to use Common Crawl for monolingual applications. We propose a general, highly parallel, multithreaded pipeline to clean and classify Common Crawl by language; we specifically design it so that it runs efficiently on medium to low resource infrastructures where I/O speeds are the main constraint. We develop the pipeline so that it can be easily reapplied to any kind of heterogeneous corpus and so that it can be parameterised to a wide range of infrastructures. We also distribute a 6.3TB version of Common Crawl, filtered, classified by language, shuffled at line level in order to avoid copyright issues, and ready to be used for NLP applications.},\n language = {en}\n}\n", |
