Fix some bugs

src/run.sh

@@ -9,9 +9,9 @@ export OUTPUT_DIR=/home/m3hrdadfi/code/gpt2-medium-persian
 # export CONFIG_NAME=/home/m3hrdadfi/code/gpt2-medium-persian
 # export TOKENIZER_NAME=/home/m3hrdadfi/code/gpt2-medium-persian
 
-export TRAIN_FILE=/home/m3hrdadfi/data/train.csv
-export VALIDATION_FILE=/home/m3hrdadfi/data/test.csv
-export TEST_FILE=/home/m3hrdadfi/code/data/test.csv
+export TRAIN_FILE=/home/m3hrdadfi/data/train-fixed.csv
+export VALIDATION_FILE=/home/m3hrdadfi/data/test-fixed.csv
+export TEST_FILE=/home/m3hrdadfi/code/data/test-fixed.csv
 # export DATASET_NAME=oscar
 # export DATASET_CONFIG_NAME=unshuffled_deduplicated_fa
 export MAX_SEQUENCE_LENGTH=512

src/run_clm_flax.py

@@ -368,6 +368,7 @@ def main():
     # dataset = dataset.map(normalizer)
     # logger.info(f"Preprocessed dataset kept {len(dataset)} out of {len(raw_dataset)}")
     dataset = raw_dataset
+    logger.info(f"dataset: {dataset}")
 
     # Load pretrained model and tokenizer
 
@@ -421,6 +422,7 @@ def main():
     else:
         column_names = dataset["validation"].column_names
     text_column_name = "text" if "text" in column_names else column_names[0]
+    logger.info(f"text_column_name: {text_column_name}")
 
     # since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function
     tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base")