{ "model": { "codellama": { "base_model_id": "codellama/CodeLlama-7b-hf", "quantitize": "int8", "dataset": "Arithmetic_Simple", "data_collator": "DataCollatorForSeq2Seq", "lora_config": { "r": 16, "lora_alpha": 16, "target_modules": [ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj" ], "lora_dropout": 0.05, "bias": "none", "task_type": "CAUSAL_LM" }, "training_args": { "output_dir": "codellama-output", "warmup_steps": 100, "per_device_train_batch_size": 1, "per_device_eval_batch_size": 1, "gradient_accumulation_steps": 4, "max_steps": 10000, "learning_rate": 0.0003, "optim": "adamw_torch", "logging_dir": "codellama-output-logs", "logging_steps": 10, "save_strategy": "steps", "save_steps": 500, "load_best_model_at_end": false, "group_by_length": true, "fp16": true, "evaluation_strategy": "steps", "eval_steps": 1000 }, "tokenizer": { "tokenize_config": { "truncation": true, "max_length": 192, "padding": "max_length" }, "prompt_template": "config/qa_template.txt" } }, "phi-2": { "base_model_id": "microsoft/phi-2", "quantitize": "fp16", "dataset": "Arithmetic_Simple", "data_collator": "DataCollatorForLanguageModeling", "lora_config": { "r": 32, "lora_alpha": 64, "target_modules": [ "q_proj", "k_proj", "v_proj", "dense", "fc1", "fc2" ], "bias": "none", "lora_dropout": 0.05, "task_type": "CAUSAL_LM" }, "training_args": { "output_dir": "phi2-output", "warmup_steps": 500, "per_device_train_batch_size": 1, "per_device_eval_batch_size": 1, "gradient_accumulation_steps": 4, "max_steps": 100000, "learning_rate": 0.0003, "optim": "paged_adamw_8bit", "logging_dir": "phi2-output-logs", "logging_steps": 100, "save_strategy": "steps", "save_steps": 500, "evaluation_strategy": "steps", "eval_steps": 500, "fp16": true }, "tokenizer": { "tokenize_config": { "truncation": true, "max_length": 512, "padding": "max_length" }, "prompt_template": "config/qa_template.txt" } }, "deepseek": { "base_model_id": "deepseek-ai/deepseek-coder-1.3b-instruct", "quantitize": "fp16", "dataset": "ghcode_python", "data_collator": "DataCollatorForLanguageModeling", "lora_config": { "r": 32, "lora_alpha": 64, "target_modules": [ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj" ], "bias": "none", "lora_dropout": 0.05, "task_type": "CAUSAL_LM" }, "lora_large_config": { "r": 128, "lora_alpha": 256, "target_modules": [ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj" ], "bias": "none", "lora_dropout": 0.05, "task_type": "CAUSAL_LM" }, "p_tuning_config": { "num_virtual_tokens": 16, "num_transformer_submodules": 1, "token_dim": 2048, "encoder_hidden_size": 2048, "task_type": "CAUSAL_LM" }, "training_args": { "output_dir": "runs/deepseek-ctrl-gh", "warmup_steps": 0, "per_device_train_batch_size": 8, "per_device_eval_batch_size": 8, "gradient_accumulation_steps": 4, "max_steps": 15000, "learning_rate": 2e-05, "optim": "paged_adamw_8bit", "logging_dir": "runs/deepseek-ctrl-gh/logs", "logging_steps": 100, "save_strategy": "steps", "save_steps": 5000, "evaluation_strategy": "steps", "eval_steps": 5000, "weight_decay": 0.01, "fp16": true }, "tokenizer": { "tokenize_config": { "truncation": true, "max_length": 512, "padding": "max_length" }, "prompt_template": "config/qa_template.txt" } } }, "dataset": { "simple_dataset": { "type": "huggingface", "dataset_purpose": "downstream", "name": "b-mc2/sql-create-context", "train_split": 0.9, "max_train_size": 100, "filling_field": [ "question", "context", "answer" ] }, "testdset": { "type": "local", "dataset_purpose": "downstream", "train_file": "data/Test/TestDataset.json", "val_file": "data/Test/TestDataset.json", "test_file": "data/Test/TestDataset.json", "filling_field": [ "prompted_question", "answer" ] }, "mixture_codegen": { "filling_field": [ "Question", "Answer" ], "dataset_purpose": "downstream" }, "MathQA_Python_loader": { "type": "list-like", "dataset_purpose": "downstream", "train": "data/MathQA_Python_processed/mathqa_python_train_clean_final.json", "val": "data/MathQA_Python_processed/mathqa_python_dev_clean_final.json", "test": "data/MathQA_Python_processed/mathqa_python_test_clean_final.json", "filling_field": [ "Question", "Answer" ] }, "APPS_loader": { "type": "list-like", "dataset_purpose": "downstream", "train": "data/APPS/apps_train.json", "val": "data/APPS/apps_dev.json", "test": "data/APPS/test/apps_test_75.json", "filling_field": [ "Question", "Answer" ] }, "MBPP_loader": { "type": "list-like", "dataset_purpose": "downstream", "train": "data/MBPP/mbpp_train.json", "val": "data/MBPP/mbpp_dev.json", "test": "data/MBPP/mbpp_test.json", "filling_field": [ "Question", "Answer" ] }, "Arithmetic_Simple": { "type": "list-like", "dataset_purpose": "downstream", "attributes": { "subjects": [ 1, 2, 3, 4, 5, 6, 7, 8, 9 ], "lessons": [ "Max_Ops1_Bounds0_100", "Max_Ops1_Bounds0_1000", "Max_Ops2_Bounds0_100", "Max_Ops2_Bounds0_1000", "Max_Ops3_Bounds0_100", "Max_Ops3_Bounds0_1000", "Max_Ops4_Bounds0_100", "Max_Ops4_Bounds0_1000", "Max_Ops5_Bounds0_100", "Max_Ops5_Bounds0_1000" ] }, "train": "data/Arithmetic/Curriculum_Simple", "val": "data/Arithmetic/Curriculum_Simple", "test": "data/Arithmetic/Curriculum_Simple", "filling_field": [ "Question", "Answer" ] }, "Arithmetic_Hard": { "type": "list-like", "dataset_purpose": "downstream", "attributes": { "subjects": [ 1, 2, 3, 4, 5, 6, 7, 8, 9 ], "lessons": [ "Max_Ops1_Bounds-1000_1000", "Max_Ops1_Bounds-100_100", "Max_Ops1_Bounds0_100", "Max_Ops1_Bounds0_1000", "Max_Ops2_Bounds-1000_1000", "Max_Ops2_Bounds-100_100", "Max_Ops2_Bounds0_100", "Max_Ops2_Bounds0_1000", "Max_Ops3_Bounds-1000_1000", "Max_Ops3_Bounds-100_100", "Max_Ops3_Bounds0_100", "Max_Ops3_Bounds0_1000", "Max_Ops4_Bounds-1000_1000", "Max_Ops4_Bounds-100_100", "Max_Ops4_Bounds0_100", "Max_Ops4_Bounds0_1000", "Max_Ops5_Bounds-1000_1000", "Max_Ops5_Bounds-100_100", "Max_Ops5_Bounds0_100", "Max_Ops5_Bounds0_1000", "Max_Ops6_Bounds-1000_1000", "Max_Ops6_Bounds-100_100", "Max_Ops6_Bounds0_100", "Max_Ops6_Bounds0_1000", "Max_Ops7_Bounds-1000_1000", "Max_Ops7_Bounds-100_100", "Max_Ops7_Bounds0_100", "Max_Ops7_Bounds0_1000", "Max_Ops8_Bounds-1000_1000", "Max_Ops8_Bounds-100_100", "Max_Ops8_Bounds0_100", "Max_Ops8_Bounds0_1000", "Max_Ops9_Bounds-1000_1000", "Max_Ops9_Bounds-100_100", "Max_Ops9_Bounds0_100", "Max_Ops9_Bounds0_1000", "Max_Ops10_Bounds-1000_1000", "Max_Ops10_Bounds-100_100", "Max_Ops10_Bounds0_100", "Max_Ops10_Bounds0_1000" ] }, "train": "data/Arithmetic/Curriculum_Hard", "val": "data/Arithmetic/Curriculum_Hard", "test": "data/Arithmetic/Curriculum_Hard", "filling_field": [ "Question", "Answer" ] }, "Arithmetic_Hard_prompt_C11": { "type": "list-like", "dataset_purpose": "downstream", "attributes": { "subjects": [ 1, 2, 3, 4, 5, 6, 7, 8, 9 ], "lessons": [ "Max_Ops1_Bounds-1000_1000", "Max_Ops1_Bounds-100_100", "Max_Ops1_Bounds0_100", "Max_Ops1_Bounds0_1000", "Max_Ops2_Bounds-1000_1000", "Max_Ops2_Bounds-100_100", "Max_Ops2_Bounds0_100", "Max_Ops2_Bounds0_1000", "Max_Ops3_Bounds-1000_1000", "Max_Ops3_Bounds-100_100", "Max_Ops3_Bounds0_100", "Max_Ops3_Bounds0_1000", "Max_Ops4_Bounds-1000_1000", "Max_Ops4_Bounds-100_100", "Max_Ops4_Bounds0_100", "Max_Ops4_Bounds0_1000", "Max_Ops5_Bounds-1000_1000", "Max_Ops5_Bounds-100_100", "Max_Ops5_Bounds0_100", "Max_Ops5_Bounds0_1000", "Max_Ops6_Bounds-1000_1000", "Max_Ops6_Bounds-100_100", "Max_Ops6_Bounds0_100", "Max_Ops6_Bounds0_1000", "Max_Ops7_Bounds-1000_1000", "Max_Ops7_Bounds-100_100", "Max_Ops7_Bounds0_100", "Max_Ops7_Bounds0_1000", "Max_Ops8_Bounds-1000_1000", "Max_Ops8_Bounds-100_100", "Max_Ops8_Bounds0_100", "Max_Ops8_Bounds0_1000", "Max_Ops9_Bounds-1000_1000", "Max_Ops9_Bounds-100_100", "Max_Ops9_Bounds0_100", "Max_Ops9_Bounds0_1000", "Max_Ops10_Bounds-1000_1000", "Max_Ops10_Bounds-100_100", "Max_Ops10_Bounds0_100", "Max_Ops10_Bounds0_1000" ] }, "train": "data/Arithmetic/Curriculum_Hard", "val": "data/Arithmetic/Curriculum_Hard", "test": "data/Arithmetic/Curriculum_Hard", "filling_field": [ "Question", "Answer" ] }, "Arithmetic_Hard_prompt_C12": { "type": "list-like", "dataset_purpose": "downstream", "attributes": { "subjects": [ 1, 2, 3, 4 ], "lessons": [ "Max_Ops1_Bounds-1000_1000", "Max_Ops1_Bounds-100_100", "Max_Ops1_Bounds0_100", "Max_Ops1_Bounds0_1000", "Max_Ops2_Bounds-1000_1000", "Max_Ops2_Bounds-100_100", "Max_Ops2_Bounds0_100", "Max_Ops2_Bounds0_1000", "Max_Ops3_Bounds-1000_1000", "Max_Ops3_Bounds-100_100", "Max_Ops3_Bounds0_100", "Max_Ops3_Bounds0_1000", "Max_Ops4_Bounds-1000_1000", "Max_Ops4_Bounds-100_100", "Max_Ops4_Bounds0_100", "Max_Ops4_Bounds0_1000", "Max_Ops5_Bounds-1000_1000", "Max_Ops5_Bounds-100_100", "Max_Ops5_Bounds0_100", "Max_Ops5_Bounds0_1000", "Max_Ops6_Bounds-1000_1000", "Max_Ops6_Bounds-100_100", "Max_Ops6_Bounds0_100", "Max_Ops6_Bounds0_1000", "Max_Ops7_Bounds-1000_1000", "Max_Ops7_Bounds-100_100", "Max_Ops7_Bounds0_100", "Max_Ops7_Bounds0_1000", "Max_Ops8_Bounds-1000_1000", "Max_Ops8_Bounds-100_100", "Max_Ops8_Bounds0_100", "Max_Ops8_Bounds0_1000", "Max_Ops9_Bounds-1000_1000", "Max_Ops9_Bounds-100_100", "Max_Ops9_Bounds0_100", "Max_Ops9_Bounds0_1000", "Max_Ops10_Bounds-1000_1000", "Max_Ops10_Bounds-100_100", "Max_Ops10_Bounds0_100", "Max_Ops10_Bounds0_1000" ] }, "train": "data/Arithmetic/Curriculum_Hard", "val": "data/Arithmetic/Curriculum_Hard", "test": "data/Arithmetic/Curriculum_Hard", "filling_field": [ "Question", "Answer" ] }, "Arithmetic_XHard": { "type": "list-like", "dataset_purpose": "downstream", "attributes": { "subjects": [ 1, 2, 3, 4, 5, 6, 7, 8, 9 ], "lessons": [ "Max_Ops10_Bounds0_10000.json", "Max_Ops10_Bounds0_1000.json", "Max_Ops10_Bounds-10000_10000.json", "Max_Ops10_Bounds-1000_1000.json", "Max_Ops11_Bounds0_10000.json", "Max_Ops11_Bounds0_1000.json", "Max_Ops11_Bounds-10000_10000.json", "Max_Ops11_Bounds-1000_1000.json", "Max_Ops12_Bounds0_10000.json", "Max_Ops12_Bounds0_1000.json", "Max_Ops12_Bounds-10000_10000.json", "Max_Ops12_Bounds-1000_1000.json", "Max_Ops13_Bounds0_10000.json", "Max_Ops13_Bounds0_1000.json", "Max_Ops13_Bounds-10000_10000.json", "Max_Ops13_Bounds-1000_1000.json", "Max_Ops14_Bounds0_10000.json", "Max_Ops14_Bounds0_1000.json", "Max_Ops14_Bounds-10000_10000.json", "Max_Ops14_Bounds-1000_1000.json", "Max_Ops15_Bounds0_10000.json", "Max_Ops15_Bounds0_1000.json", "Max_Ops15_Bounds-10000_10000.json", "Max_Ops15_Bounds-1000_1000.json", "Max_Ops16_Bounds0_10000.json", "Max_Ops16_Bounds0_1000.json", "Max_Ops16_Bounds-10000_10000.json", "Max_Ops16_Bounds-1000_1000.json", "Max_Ops17_Bounds0_10000.json", "Max_Ops17_Bounds0_1000.json", "Max_Ops17_Bounds-10000_10000.json", "Max_Ops17_Bounds-1000_1000.json", "Max_Ops18_Bounds0_10000.json", "Max_Ops18_Bounds0_1000.json", "Max_Ops18_Bounds-10000_10000.json", "Max_Ops18_Bounds-1000_1000.json", "Max_Ops19_Bounds0_10000.json", "Max_Ops19_Bounds0_1000.json", "Max_Ops19_Bounds-10000_10000.json", "Max_Ops19_Bounds-1000_1000.json", "Max_Ops1_Bounds0_10000.json", "Max_Ops1_Bounds0_1000.json", "Max_Ops1_Bounds-10000_10000.json", "Max_Ops1_Bounds-1000_1000.json", "Max_Ops20_Bounds0_10000.json", "Max_Ops20_Bounds0_1000.json", "Max_Ops20_Bounds-10000_10000.json", "Max_Ops20_Bounds-1000_1000.json", "Max_Ops2_Bounds0_10000.json", "Max_Ops2_Bounds0_1000.json", "Max_Ops2_Bounds-10000_10000.json", "Max_Ops2_Bounds-1000_1000.json", "Max_Ops3_Bounds0_10000.json", "Max_Ops3_Bounds0_1000.json", "Max_Ops3_Bounds-10000_10000.json", "Max_Ops3_Bounds-1000_1000.json", "Max_Ops4_Bounds0_10000.json", "Max_Ops4_Bounds0_1000.json", "Max_Ops4_Bounds-10000_10000.json", "Max_Ops4_Bounds-1000_1000.json", "Max_Ops5_Bounds0_10000.json", "Max_Ops5_Bounds0_1000.json", "Max_Ops5_Bounds-10000_10000.json", "Max_Ops5_Bounds-1000_1000.json", "Max_Ops6_Bounds0_10000.json", "Max_Ops6_Bounds0_1000.json", "Max_Ops6_Bounds-10000_10000.json", "Max_Ops6_Bounds-1000_1000.json", "Max_Ops7_Bounds0_10000.json", "Max_Ops7_Bounds0_1000.json", "Max_Ops7_Bounds-10000_10000.json", "Max_Ops7_Bounds-1000_1000.json", "Max_Ops8_Bounds0_10000.json", "Max_Ops8_Bounds0_1000.json", "Max_Ops8_Bounds-10000_10000.json", "Max_Ops8_Bounds-1000_1000.json", "Max_Ops9_Bounds0_10000.json", "Max_Ops9_Bounds0_1000.json", "Max_Ops9_Bounds-10000_10000.json", "Max_Ops9_Bounds-1000_1000.json" ] }, "train": "data/Arithmetic/Curriculum_XHard", "val": "data/Arithmetic/Curriculum_XHard", "test": "data/Arithmetic/Curriculum_XHard", "filling_field": [ "Question", "Answer" ] }, "GSM8K": { "type": "local", "dataset_purpose": "downstream", "train_file": "data/GSM8K/GSM8K_train.json", "val_file": "data/GSM8K/GSM8K_test.json", "test_file": "data/GSM8K/GSM8K_dev.json", "filling_field": [ "Body", "Question", "Answer" ] }, "APPS": { "type": "local", "dataset_purpose": "downstream", "train_file": "data/APPS/apps_train.json", "val_file": "data/APPS/apps_test.json", "test_file": "data/APPS/apps_dev.json", "filling_field": [ "Body", "Question", "Answer" ] }, "ghcode_python": { "type": "huggingface", "dataset_purpose": "pretrain", "name": "slseanwu/ghcode_python_split_700k", "max_eval_size": 1000, "max_train_size": 160000, "filling_field": [ "code" ] } } }