File size: 947 Bytes
cfbc06c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 |
- datapaths: ["chr20_21_22.fa"]
output_dir: "./preprocessed_data"
output_prefix: chr20_21_22_uint8_distinct
train_split: 0.9
valid_split: 0.05
test_split: 0.05
overwrite: True
embed_reverse_complement: true
random_reverse_complement: 0.0
random_lineage_dropout: 0.0
include_sequence_id: false
transcribe: "back_transcribe"
force_uppercase: false
indexed_dataset_dtype: "uint8"
tokenizer_type: "Byte-Level"
vocab_file: null
vocab_size: null
merges_file: null
pretrained_tokenizer_model: null
special_tokens: null
fast_hf_tokenizer: true
append_eod: true
enforce_sample_length: null
ftfy: false
workers: 1
preproc_concurrency: 100000
chunksize: 25
drop_empty_sequences: true
nnn_filter: false # If you split your fasta on NNN (in human these are contigs), then you should set this to true.
seed: 12342 # Not relevant because we are not using random reverse complement or lineage dropout.
|