Thomas Le Menestrel commited on
Commit
dd7513a
·
1 Parent(s): ad72d8e

add tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +0 -1
tokenizer_config.json CHANGED
@@ -18,7 +18,6 @@
18
  "single_word": false
19
  },
20
  "errors": "replace",
21
- "model_max_length": 1024,
22
  "name_or_path": "antoinelouis/belgpt2",
23
  "pad_token": null,
24
  "special_tokens_map_file": "/home/thomaslemenestrel/.cache/huggingface/transformers/e19363d532fabc90192003aa7749a0b8e86d37456a239d24f0adc748cf5796db.3ae9ae72462581d20e36bc528e9c47bb30cd671bb21add40ca0b24a0be9fac22",
 
18
  "single_word": false
19
  },
20
  "errors": "replace",
 
21
  "name_or_path": "antoinelouis/belgpt2",
22
  "pad_token": null,
23
  "special_tokens_map_file": "/home/thomaslemenestrel/.cache/huggingface/transformers/e19363d532fabc90192003aa7749a0b8e86d37456a239d24f0adc748cf5796db.3ae9ae72462581d20e36bc528e9c47bb30cd671bb21add40ca0b24a0be9fac22",