Spaces:
Build error
Build error
Update app.py
Browse files
app.py
CHANGED
|
@@ -11,7 +11,8 @@ def load_tok_and_data(lan):
|
|
| 11 |
tokenizer._src_lang = _Tokens[lan]
|
| 12 |
tokenizer.cur_lang_code_id = tokenizer.convert_tokens_to_ids(_Tokens[lan])
|
| 13 |
tokenizer.set_src_lang_special_tokens(_Tokens[lan])
|
| 14 |
-
dataset = load_dataset('Babelscape/SREDFM', lan, split="test", streaming=True)
|
|
|
|
| 15 |
dataset = [example for example in dataset.take(1001)]
|
| 16 |
return (tokenizer, dataset)
|
| 17 |
|
|
|
|
| 11 |
tokenizer._src_lang = _Tokens[lan]
|
| 12 |
tokenizer.cur_lang_code_id = tokenizer.convert_tokens_to_ids(_Tokens[lan])
|
| 13 |
tokenizer.set_src_lang_special_tokens(_Tokens[lan])
|
| 14 |
+
dataset = load_dataset('Babelscape/SREDFM', lan, split="test", streaming=True, trust_remote_code=True)
|
| 15 |
+
|
| 16 |
dataset = [example for example in dataset.take(1001)]
|
| 17 |
return (tokenizer, dataset)
|
| 18 |
|