training

generate_text2sql_dataset_amr.sh

@@ -0,0 +1,38 @@
+set -e
+
+# generate text2sql training dataset with noise_rate 0.2
+python text2sql_data_generator.py \
+    --input_dataset_path "./data/preprocessed_data/preprocessed_train_spider_amr.json" \
+    --output_dataset_path "./data/preprocessed_data/resdsql_train_spider_amr.json" \
+    --topk_table_num 4 \
+    --topk_column_num 5 \
+    --mode "train" \
+    --noise_rate 0.2 \
+    --use_contents \
+    --add_fk_info \
+    --output_skeleton \
+    --target_type "sql"
+
+# predict probability for each schema item in the eval set
+python schema_item_classifier.py \
+    --batch_size 32 \
+    --device "0" \
+    --seed 42 \
+    --save_path "./models/text2sql_schema_item_classifier_semantic" \
+    --dev_filepath "./data/preprocessed_data/preprocessed_dev_amr.json" \
+    --output_filepath "./data/preprocessed_data/dev_with_probs_amr.json" \
+    --use_contents \
+    --add_fk_info \
+    --mode "eval"
+
+# generate text2sql development dataset
+python text2sql_data_generator.py \
+    --input_dataset_path "./data/preprocessed_data/dev_with_probs_amr.json" \
+    --output_dataset_path "./data/preprocessed_data/resdsql_dev_amr.json" \
+    --topk_table_num 4 \
+    --topk_column_num 5 \
+    --mode "eval" \
+    --use_contents \
+    --add_fk_info \
+    --output_skeleton \
+    --target_type "sql"

preprocess.sh

@@ -0,0 +1,19 @@
+set -e
+
+# preprocess train_spider dataset
+python preprocessing.py \
+    --mode "train" \
+    --table_path "./data/spider_amr/tables.json" \
+    --input_dataset_path "./data/spider_amr/train_spider.json" \
+    --output_dataset_path "./data/preprocessed_data/preprocessed_train_spider_amr.json" \
+    --db_path "./database" \
+    --target_type "sql"
+
+# preprocess dev dataset
+python preprocessing.py \
+    --mode "eval" \
+    --table_path "./data/spider_amr/tables.json" \
+    --input_dataset_path "./data/spider_amr/dev.json" \
+    --output_dataset_path "./data/preprocessed_data/preprocessed_dev_amr.json" \
+    --db_path "./database"\
+    --target_type "sql"

train_text2sql_schema_item_classifier.sh

@@ -0,0 +1,21 @@
+set -e
+
+# train schema item classifier
+python -u schema_item_classifier.py \
+    --batch_size 8 \
+    --gradient_descent_step 2 \
+    --device "0" \
+    --learning_rate 1e-5 \
+    --gamma 2.0 \
+    --alpha 0.75 \
+    --epochs 32 \
+    --patience 16 \
+    --seed 42 \
+    --save_path "./models/text2sql_schema_item_classifier_semantic" \
+    --tensorboard_save_path "./tensorboard_log/text2sql_schema_item_classifier_semantic" \
+    --train_filepath "./data/preprocessed_data/preprocessed_train_spider_amr.json" \
+    --dev_filepath "./data/preprocessed_data/preprocessed_dev_amr.json" \
+    --model_name_or_path "roberta-large" \
+    --use_contents \
+    --add_fk_info \
+    --mode "train"

train_text2sql_t5_base.sh

@@ -0,0 +1,31 @@
+set -e
+
+# train text2sql-t5-base model
+python -u text2sql_inputgrande.py \
+    --batch_size 8 \
+    --gradient_descent_step 2 \
+    --device "0" \
+    --learning_rate 1e-4 \
+    --epochs 128 \
+    --seed 42 \
+    --save_path "./models/text2sql-t5-amr" \
+    --tensorboard_save_path "./tensorboard_log/text2sql-t5-amr" \
+    --model_name_or_path "t5-base" \
+    --use_adafactor \
+    --mode train \
+    --train_filepath "./data/preprocessed_data/resdsql_train_spider_amr.json"
+    
+# select the best text2sql-t5-base ckpt
+python -u evaluate_text2sql_ckpts_inputgrande.py \
+    --batch_size 8 \
+    --device "0" \
+    --seed 42 \
+    --save_path "./models/text2sql-t5-amr" \
+    --eval_results_path "./eval_results/text2sql-t5-amr" \
+    --mode eval \
+    --dev_filepath "./data/preprocessed_data/resdsql_dev_amr.json" \
+    --original_dev_filepath "./data/spider_amr/dev.json" \
+    --db_path "./database" \
+    --num_beams 8 \
+    --num_return_sequences 8 \
+    --target_type "sql"